admin/goyomi
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: use per-source HTTP client instead of global proxy

Browse Source 
- Remove global ProxyEnabled() logic from httpclient
- Each source now explicitly chooses client at import time:
  - flare client: for JS-rendering/cloudflare sources
  - normal httpclient: for REST API sources
- Updated 29 base sources based on Kotlin reference (network.cloudflareClient)
This commit is contained in:
achmad
2026-05-13 09:01:51 +07:00
parent b199bad30d
commit 9a42dd2ab1
31 changed files with 96 additions and 255 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/server/main.go
-12
View File
@@ -6,11 +6,9 @@ import (
"fmt" "fmt"
"log" "log"
"net/http" "net/http"
"os"
"goyomi/internal/config" "goyomi/internal/config"
"goyomi/internal/db" "goyomi/internal/db"
"goyomi/internal/httpclient"
_ "goyomi/internal/registry" _ "goyomi/internal/registry"
_ "goyomi/sources/all/hentaihand" _ "goyomi/sources/all/hentaihand"
_ "goyomi/sources/all/kemono" _ "goyomi/sources/all/kemono"
@@ -46,16 +44,6 @@ func main() {
// Initialize config manager // Initialize config manager
config.InitConfigManager(database.Queries) config.InitConfigManager(database.Queries)
if os.Getenv("FLARESOLVERR_URL") != "" {
fsClient, err := httpclient.NewFlareSolverrClient()
if err != nil {
log.Printf("flaresolverr: client creation failed: %v", err)
} else {
httpclient.SetGlobalFlareSolverr(fsClient)
log.Printf("flaresolverr: client initialized")
}
}
mux := http.NewServeMux() mux := http.NewServeMux()
mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) { mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
fmt.Fprintln(w, "ok") fmt.Fprintln(w, "ok")
internal/httpclient/client.go
+2 -107
View File
@@ -2,56 +2,27 @@ package httpclient
import ( import (
"context" "context"
"fmt"
"io" "io"
"log" "log"
"net/http" "net/http"
"net/http/cookiejar" "net/http/cookiejar"
"os"
"strconv" "strconv"
"strings"
"sync" "sync"
"time" "time"
"goyomi/internal/config"
"golang.org/x/time/rate" "golang.org/x/time/rate"
) )
const defaultUserAgent = "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Mobile Safari/537.36" const defaultUserAgent = "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Mobile Safari/537.36"
var ( var verboseLog bool
globalFSClient *FlareSolverrClient
globalProxy bool
verboseLog bool
)
func SetGlobalFlareSolverr(client *FlareSolverrClient) {
globalFSClient = client
}
func SetGlobalProxy(enabled bool) {
globalProxy = enabled
}
func SetVerboseLog(enabled bool) { func SetVerboseLog(enabled bool) {
verboseLog = enabled verboseLog = enabled
} }
func ProxyEnabled() bool {
envProxy := os.Getenv("FLARESOLVERR_PROXY") == "1"
if globalProxy || envProxy {
return true
}
if config.IsFlareSolverrProxyEnabled() {
return true
}
return false
}
type Client struct { type Client struct {
http *http.Client http *http.Client
fsClient *FlareSolverrClient
proxy bool
rateLimit float64 rateLimit float64
burst int burst int
referer string referer string
@@ -78,29 +49,18 @@ func WithReferer(referer string) Option {
return func(c *Client) { c.referer = referer } return func(c *Client) { c.referer = referer }
} }
func WithFlareSolverr(fs *FlareSolverrClient) Option {
return func(c *Client) { c.fsClient = fs }
}
func WithProxy(enabled bool) Option {
return func(c *Client) { c.proxy = enabled }
}
func WithVerboseLog(enabled bool) Option { func WithVerboseLog(enabled bool) Option {
return func(c *Client) { c.verboseLog = enabled } return func(c *Client) { c.verboseLog = enabled }
} }
func NewClient(opts ...Option) *Client { func NewClient(opts ...Option) *Client {
jar, _ := cookiejar.New(nil) jar, _ := cookiejar.New(nil)
vLog := os.Getenv("VERBOSE_LOG") == "1"
c := &Client{ c := &Client{
http: &http.Client{Timeout: 30 * time.Second, Jar: jar}, http: &http.Client{Timeout: 30 * time.Second, Jar: jar},
fsClient: globalFSClient,
proxy: globalProxy,
rateLimit: 1, rateLimit: 1,
burst: 1, burst: 1,
limiters: map[string]*rate.Limiter{}, limiters: map[string]*rate.Limiter{},
verboseLog: verboseLog || vLog, verboseLog: verboseLog,
} }
for _, o := range opts { for _, o := range opts {
o(c) o(c)
@@ -120,9 +80,6 @@ func (c *Client) limiter(host string) *rate.Limiter {
} }
func (c *Client) Do(req *http.Request) (*http.Response, error) { func (c *Client) Do(req *http.Request) (*http.Response, error) {
if (c.proxy || ProxyEnabled()) && c.fsClient != nil {
return c.doViaFlareSolverr(req)
}
return c.doDirect(req) return c.doDirect(req)
} }
@@ -167,68 +124,6 @@ func (c *Client) doDirect(req *http.Request) (*http.Response, error) {
panic("unreachable") panic("unreachable")
} }
func (c *Client) doViaFlareSolverr(req *http.Request) (*http.Response, error) {
if err := c.limiter(req.URL.Host).Wait(req.Context()); err != nil {
return nil, err
}
url := req.URL.String()
if c.verboseLog {
log.Printf("[httpclient] FLARESOLVERR %s %s", req.Method, url)
}
var html string
var cookies []*http.Cookie
var err error
if req.Method == http.MethodGet {
html, cookies, err = c.fsClient.Get(req.Context(), url)
} else if req.Method == http.MethodPost {
var bodyStr string
if req.Body != nil {
bodyBytes, _ := io.ReadAll(req.Body)
bodyStr = string(bodyBytes)
if c.verboseLog {
log.Printf("[httpclient] FLARESOLVERR POST BODY: %s", bodyStr)
}
}
html, cookies, err = c.fsClient.Post(req.Context(), url, bodyStr)
} else {
return c.doDirect(req)
}
if err != nil {
return nil, fmt.Errorf("flaresolverr: %w", err)
}
if c.verboseLog {
log.Printf("[httpclient] FLARESOLVERR RESPONSE length=%d cookies=%d", len(html), len(cookies))
if len(html) > 0 {
truncate := html
if len(html) > 500 {
truncate = html[:500]
}
log.Printf("[httpclient] FLARESOLVERR RESPONSE HTML (first 500 chars): %s", truncate)
}
}
resp := &http.Response{
StatusCode: http.StatusOK,
Header: http.Header{},
Body: io.NopCloser(strings.NewReader(html)),
}
resp.Header.Set("Content-Type", "text/html; charset=utf-8")
if c.http.Jar != nil {
for _, cookie := range cookies {
c.http.Jar.SetCookies(req.URL, []*http.Cookie{cookie})
}
}
return resp, nil
}
func retryAfter(resp *http.Response) time.Duration { func retryAfter(resp *http.Response) time.Duration {
ra := resp.Header.Get("Retry-After") ra := resp.Header.Get("Retry-After")
if ra == "" { if ra == "" {
sources/base/comicaso/comicaso.go
+3 -3
View File
@@ -12,7 +12,7 @@ import (
"strings" "strings"
"sync" "sync"
"goyomi/internal/httpclient" "goyomi/internal/httpclient/flare"
"goyomi/internal/source" "goyomi/internal/source"
"goyomi/sources/base/util" "goyomi/sources/base/util"
) )
@@ -27,7 +27,7 @@ type Config struct {
type Source struct { type Source struct {
cfg Config cfg Config
client *httpclient.Client client *flare.Client
id int64 id int64
mu sync.Mutex mu sync.Mutex
@@ -68,7 +68,7 @@ type tokenResponseDTO struct {
} }
func New(cfg Config) *Source { func New(cfg Config) *Source {
c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) c := flare.NewClient(flare.WithRateLimit(1, 2))
return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)}
} }
sources/base/comiciviewer/comiciviewer.go
+3 -3
View File
@@ -12,7 +12,7 @@ import (
"github.com/PuerkitoBio/goquery" "github.com/PuerkitoBio/goquery"
"goyomi/internal/httpclient" "goyomi/internal/httpclient/flare"
"goyomi/internal/source" "goyomi/internal/source"
"goyomi/sources/base/util" "goyomi/sources/base/util"
) )
@@ -25,12 +25,12 @@ type Config struct {
type Source struct { type Source struct {
cfg Config cfg Config
client *httpclient.Client client *flare.Client
id int64 id int64
} }
func New(cfg Config) *Source { func New(cfg Config) *Source {
c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) c := flare.NewClient(flare.WithRateLimit(1, 2))
return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)}
} }
sources/base/fuzzydoodle/fuzzydoodle.go
+3 -3
View File
@@ -10,7 +10,7 @@ import (
"github.com/PuerkitoBio/goquery" "github.com/PuerkitoBio/goquery"
"goyomi/internal/httpclient" "goyomi/internal/httpclient/flare"
"goyomi/internal/source" "goyomi/internal/source"
"goyomi/sources/base/util" "goyomi/sources/base/util"
) )
@@ -23,12 +23,12 @@ type Config struct {
type Source struct { type Source struct {
cfg Config cfg Config
client *httpclient.Client client *flare.Client
id int64 id int64
} }
func New(cfg Config) *Source { func New(cfg Config) *Source {
c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) c := flare.NewClient(flare.WithRateLimit(1, 2))
return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)}
} }
sources/base/gattsu/gattsu.go
+4 -9
View File
@@ -11,7 +11,7 @@ import (
"github.com/PuerkitoBio/goquery" "github.com/PuerkitoBio/goquery"
"goyomi/internal/httpclient" "goyomi/internal/httpclient/flare"
"goyomi/internal/source" "goyomi/internal/source"
"goyomi/sources/base/util" "goyomi/sources/base/util"
) )
@@ -24,12 +24,12 @@ type Config struct {
type Source struct { type Source struct {
cfg Config cfg Config
client *httpclient.Client client *flare.Client
id int64 id int64
} }
func New(cfg Config) *Source { func New(cfg Config) *Source {
c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) c := flare.NewClient(flare.WithRateLimit(1, 2))
return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)}
} }
@@ -41,12 +41,7 @@ func (s *Source) SupportsLatest() bool { return true }
func (s *Source) base() string { return strings.TrimRight(s.cfg.BaseURL, "/") } func (s *Source) base() string { return strings.TrimRight(s.cfg.BaseURL, "/") }
func (s *Source) get(ctx context.Context, rawURL string) (*goquery.Document, error) { func (s *Source) get(ctx context.Context, rawURL string) (*goquery.Document, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) resp, err := s.client.Get(ctx, rawURL)
if err != nil {
return nil, err
}
req.Header.Set("Referer", s.cfg.BaseURL+"/")
resp, err := s.client.Do(req)
if err != nil { if err != nil {
return nil, err return nil, err
} }
sources/base/gigaviewer/gigaviewer.go
+3 -3
View File
@@ -10,7 +10,7 @@ import (
"github.com/PuerkitoBio/goquery" "github.com/PuerkitoBio/goquery"
"goyomi/internal/httpclient" "goyomi/internal/httpclient/flare"
"goyomi/internal/source" "goyomi/internal/source"
"goyomi/sources/base/util" "goyomi/sources/base/util"
) )
@@ -23,12 +23,12 @@ type Config struct {
type Source struct { type Source struct {
cfg Config cfg Config
client *httpclient.Client client *flare.Client
id int64 id int64
} }
func New(cfg Config) *Source { func New(cfg Config) *Source {
c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) c := flare.NewClient(flare.WithRateLimit(1, 2))
return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)}
} }
sources/base/gmanga/gmanga.go
+3 -3
View File
@@ -11,7 +11,7 @@ import (
"net/http" "net/http"
"strings" "strings"
"goyomi/internal/httpclient" "goyomi/internal/httpclient/flare"
"goyomi/internal/source" "goyomi/internal/source"
"goyomi/sources/base/util" "goyomi/sources/base/util"
) )
@@ -24,12 +24,12 @@ type Config struct {
type Source struct { type Source struct {
cfg Config cfg Config
client *httpclient.Client client *flare.Client
id int64 id int64
} }
func New(cfg Config) *Source { func New(cfg Config) *Source {
c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) c := flare.NewClient(flare.WithRateLimit(1, 2))
return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)}
} }
sources/base/goda/goda.go
+3 -3
View File
@@ -12,7 +12,7 @@ import (
"github.com/PuerkitoBio/goquery" "github.com/PuerkitoBio/goquery"
"goyomi/internal/httpclient" "goyomi/internal/httpclient/flare"
"goyomi/internal/source" "goyomi/internal/source"
) )
@@ -24,12 +24,12 @@ type Config struct {
type Source struct { type Source struct {
cfg Config cfg Config
client *httpclient.Client client *flare.Client
id int64 id int64
} }
func New(cfg Config) *Source { func New(cfg Config) *Source {
c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) c := flare.NewClient(flare.WithRateLimit(1, 2))
return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)}
} }
sources/base/hotcomics/hotcomics.go
+3 -3
View File
@@ -11,7 +11,7 @@ import (
"github.com/PuerkitoBio/goquery" "github.com/PuerkitoBio/goquery"
"goyomi/internal/httpclient" "goyomi/internal/httpclient/flare"
"goyomi/internal/source" "goyomi/internal/source"
"goyomi/sources/base/util" "goyomi/sources/base/util"
) )
@@ -24,12 +24,12 @@ type Config struct {
type Source struct { type Source struct {
cfg Config cfg Config
client *httpclient.Client client *flare.Client
id int64 id int64
} }
func New(cfg Config) *Source { func New(cfg Config) *Source {
c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) c := flare.NewClient(flare.WithRateLimit(1, 2))
return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)}
} }
sources/base/iken/iken.go
+3 -3
View File
@@ -10,7 +10,7 @@ import (
"net/http" "net/http"
"strings" "strings"
"goyomi/internal/httpclient" "goyomi/internal/httpclient/flare"
"goyomi/internal/source" "goyomi/internal/source"
"goyomi/sources/base/util" "goyomi/sources/base/util"
) )
@@ -26,7 +26,7 @@ type Config struct {
type Source struct { type Source struct {
cfg Config cfg Config
client *httpclient.Client client *flare.Client
id int64 id int64
} }
@@ -34,7 +34,7 @@ func New(cfg Config) *Source {
if cfg.APIURL == "" { if cfg.APIURL == "" {
cfg.APIURL = cfg.BaseURL cfg.APIURL = cfg.BaseURL
} }
c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) c := flare.NewClient(flare.WithRateLimit(1, 2))
return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)}
} }
sources/base/kemono/kemono.go
+3 -3
View File
@@ -11,7 +11,7 @@ import (
"net/http" "net/http"
"strings" "strings"
"goyomi/internal/httpclient" "goyomi/internal/httpclient/flare"
"goyomi/internal/source" "goyomi/internal/source"
"goyomi/sources/base/util" "goyomi/sources/base/util"
) )
@@ -47,12 +47,12 @@ type postDTO struct {
type Source struct { type Source struct {
cfg Config cfg Config
client *httpclient.Client client *flare.Client
id int64 id int64
} }
func New(cfg Config) *Source { func New(cfg Config) *Source {
c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) c := flare.NewClient(flare.WithRateLimit(1, 2))
return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)}
} }
sources/base/keyoapp/keyoapp.go
+3 -3
View File
@@ -12,7 +12,7 @@ import (
"github.com/PuerkitoBio/goquery" "github.com/PuerkitoBio/goquery"
"goyomi/internal/httpclient" "goyomi/internal/httpclient/flare"
"goyomi/internal/source" "goyomi/internal/source"
"goyomi/sources/base/util" "goyomi/sources/base/util"
) )
@@ -25,12 +25,12 @@ type Config struct {
type Source struct { type Source struct {
cfg Config cfg Config
client *httpclient.Client client *flare.Client
id int64 id int64
} }
func New(cfg Config) *Source { func New(cfg Config) *Source {
c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) c := flare.NewClient(flare.WithRateLimit(1, 2))
return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)}
} }
sources/base/manga18/manga18.go
+3 -3
View File
@@ -12,7 +12,7 @@ import (
"github.com/PuerkitoBio/goquery" "github.com/PuerkitoBio/goquery"
"goyomi/internal/httpclient" "goyomi/internal/httpclient/flare"
"goyomi/internal/source" "goyomi/internal/source"
"goyomi/sources/base/util" "goyomi/sources/base/util"
) )
@@ -25,12 +25,12 @@ type Config struct {
type Source struct { type Source struct {
cfg Config cfg Config
client *httpclient.Client client *flare.Client
id int64 id int64
} }
func New(cfg Config) *Source { func New(cfg Config) *Source {
c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) c := flare.NewClient(flare.WithRateLimit(1, 2))
return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)}
} }
sources/base/mangataro/mangataro.go
+5 -19
View File
@@ -3,7 +3,6 @@
package mangataro package mangataro
import ( import (
"bytes"
"context" "context"
"crypto/md5" "crypto/md5"
"encoding/json" "encoding/json"
@@ -14,7 +13,7 @@ import (
"strings" "strings"
"time" "time"
"goyomi/internal/httpclient" "goyomi/internal/httpclient/flare"
"goyomi/internal/source" "goyomi/internal/source"
"goyomi/sources/base/util" "goyomi/sources/base/util"
) )
@@ -27,12 +26,12 @@ type Config struct {
type Source struct { type Source struct {
cfg Config cfg Config
client *httpclient.Client client *flare.Client
id int64 id int64
} }
func New(cfg Config) *Source { func New(cfg Config) *Source {
c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) c := flare.NewClient(flare.WithRateLimit(1, 2))
return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)}
} }
@@ -44,13 +43,7 @@ func (s *Source) SupportsLatest() bool { return true }
func (s *Source) base() string { return strings.TrimRight(s.cfg.BaseURL, "/") } func (s *Source) base() string { return strings.TrimRight(s.cfg.BaseURL, "/") }
func (s *Source) doGet(ctx context.Context, rawURL string, out any) error { func (s *Source) doGet(ctx context.Context, rawURL string, out any) error {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) resp, err := s.client.Get(ctx, rawURL)
if err != nil {
return err
}
req.Header.Set("Referer", s.cfg.BaseURL+"/")
req.Header.Set("Accept", "application/json")
resp, err := s.client.Do(req)
if err != nil { if err != nil {
return err return err
} }
@@ -67,14 +60,7 @@ func (s *Source) doPost(ctx context.Context, rawURL string, payload any, out any
if err != nil { if err != nil {
return err return err
} }
req, err := http.NewRequestWithContext(ctx, http.MethodPost, rawURL, bytes.NewReader(body)) resp, err := s.client.Post(ctx, rawURL, "application/json", strings.NewReader(string(body)))
if err != nil {
return err
}
req.Header.Set("Referer", s.cfg.BaseURL+"/")
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Accept", "application/json")
resp, err := s.client.Do(req)
if err != nil { if err != nil {
return err return err
} }
sources/base/manhwaz/manhwaz.go
+4 -10
View File
@@ -10,7 +10,7 @@ import (
"github.com/PuerkitoBio/goquery" "github.com/PuerkitoBio/goquery"
"goyomi/internal/httpclient" "goyomi/internal/httpclient/flare"
"goyomi/internal/source" "goyomi/internal/source"
"goyomi/sources/base/util" "goyomi/sources/base/util"
) )
@@ -26,7 +26,7 @@ type Config struct {
type Source struct { type Source struct {
cfg Config cfg Config
client *httpclient.Client client *flare.Client
id int64 id int64
} }
@@ -40,7 +40,7 @@ func New(cfg Config) *Source {
if cfg.StatusHeading == "" { if cfg.StatusHeading == "" {
cfg.StatusHeading = "status" cfg.StatusHeading = "status"
} }
c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) c := flare.NewClient(flare.WithRateLimit(1, 2))
return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)}
} }
@@ -52,13 +52,7 @@ func (s *Source) SupportsLatest() bool { return true }
func (s *Source) base() string { return strings.TrimRight(s.cfg.BaseURL, "/") } func (s *Source) base() string { return strings.TrimRight(s.cfg.BaseURL, "/") }
func (s *Source) get(ctx context.Context, rawURL string) (*goquery.Document, error) { func (s *Source) get(ctx context.Context, rawURL string) (*goquery.Document, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) resp, err := s.client.Get(ctx, rawURL)
if err != nil {
return nil, err
}
req.Header.Set("Referer", s.cfg.BaseURL+"/")
req.Header.Set("Origin", s.cfg.BaseURL)
resp, err := s.client.Do(req)
if err != nil { if err != nil {
return nil, err return nil, err
} }
sources/base/masonry/masonry.go
+3 -3
View File
@@ -10,7 +10,7 @@ import (
"github.com/PuerkitoBio/goquery" "github.com/PuerkitoBio/goquery"
"goyomi/internal/httpclient" "goyomi/internal/httpclient/flare"
"goyomi/internal/source" "goyomi/internal/source"
"goyomi/sources/base/util" "goyomi/sources/base/util"
) )
@@ -23,12 +23,12 @@ type Config struct {
type Source struct { type Source struct {
cfg Config cfg Config
client *httpclient.Client client *flare.Client
id int64 id int64
} }
func New(cfg Config) *Source { func New(cfg Config) *Source {
c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) c := flare.NewClient(flare.WithRateLimit(1, 2))
return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)}
} }
sources/base/mmlook/mmlook.go
+3 -3
View File
@@ -11,7 +11,7 @@ import (
"github.com/PuerkitoBio/goquery" "github.com/PuerkitoBio/goquery"
"goyomi/internal/httpclient" "goyomi/internal/httpclient/flare"
"goyomi/internal/source" "goyomi/internal/source"
"goyomi/sources/base/util" "goyomi/sources/base/util"
) )
@@ -26,7 +26,7 @@ type Config struct {
type Source struct { type Source struct {
cfg Config cfg Config
client *httpclient.Client client *flare.Client
id int64 id int64
} }
@@ -37,7 +37,7 @@ func New(cfg Config) *Source {
if cfg.Lang == "" { if cfg.Lang == "" {
cfg.Lang = "zh" cfg.Lang = "zh"
} }
c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) c := flare.NewClient(flare.WithRateLimit(1, 2))
return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)}
} }
sources/base/mmrcms/mmrcms.go
+3 -3
View File
@@ -10,7 +10,7 @@ import (
"net/http" "net/http"
"strings" "strings"
"goyomi/internal/httpclient" "goyomi/internal/httpclient/flare"
"goyomi/internal/source" "goyomi/internal/source"
"goyomi/sources/base/util" "goyomi/sources/base/util"
) )
@@ -30,12 +30,12 @@ type mangaDTO struct {
type Source struct { type Source struct {
cfg Config cfg Config
client *httpclient.Client client *flare.Client
id int64 id int64
} }
func New(cfg Config) *Source { func New(cfg Config) *Source {
c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) c := flare.NewClient(flare.WithRateLimit(1, 2))
return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)}
} }
sources/base/paprika/paprika.go
+4 -9
View File
@@ -10,7 +10,7 @@ import (
"github.com/PuerkitoBio/goquery" "github.com/PuerkitoBio/goquery"
"goyomi/internal/httpclient" "goyomi/internal/httpclient/flare"
"goyomi/internal/source" "goyomi/internal/source"
"goyomi/sources/base/util" "goyomi/sources/base/util"
) )
@@ -23,12 +23,12 @@ type Config struct {
type Source struct { type Source struct {
cfg Config cfg Config
client *httpclient.Client client *flare.Client
id int64 id int64
} }
func New(cfg Config) *Source { func New(cfg Config) *Source {
c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) c := flare.NewClient(flare.WithRateLimit(1, 2))
return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)}
} }
@@ -40,12 +40,7 @@ func (s *Source) SupportsLatest() bool { return true }
func (s *Source) base() string { return strings.TrimRight(s.cfg.BaseURL, "/") } func (s *Source) base() string { return strings.TrimRight(s.cfg.BaseURL, "/") }
func (s *Source) get(ctx context.Context, rawURL string) (*goquery.Document, error) { func (s *Source) get(ctx context.Context, rawURL string) (*goquery.Document, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) resp, err := s.client.Get(ctx, rawURL)
if err != nil {
return nil, err
}
req.Header.Set("Referer", s.cfg.BaseURL+"/")
resp, err := s.client.Do(req)
if err != nil { if err != nil {
return nil, err return nil, err
} }
sources/base/peachscan/peachscan.go
+3 -3
View File
@@ -12,7 +12,7 @@ import (
"github.com/PuerkitoBio/goquery" "github.com/PuerkitoBio/goquery"
"goyomi/internal/httpclient" "goyomi/internal/httpclient/flare"
"goyomi/internal/source" "goyomi/internal/source"
"goyomi/sources/base/util" "goyomi/sources/base/util"
) )
@@ -27,12 +27,12 @@ type Config struct {
type Source struct { type Source struct {
cfg Config cfg Config
client *httpclient.Client client *flare.Client
id int64 id int64
} }
func New(cfg Config) *Source { func New(cfg Config) *Source {
c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) c := flare.NewClient(flare.WithRateLimit(1, 2))
return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)}
} }
sources/base/pizzareader/pizzareader.go
+3 -3
View File
@@ -11,7 +11,7 @@ import (
"sort" "sort"
"strings" "strings"
"goyomi/internal/httpclient" "goyomi/internal/httpclient/flare"
"goyomi/internal/source" "goyomi/internal/source"
"goyomi/sources/base/util" "goyomi/sources/base/util"
) )
@@ -25,7 +25,7 @@ type Config struct {
type Source struct { type Source struct {
cfg Config cfg Config
client *httpclient.Client client *flare.Client
id int64 id int64
} }
@@ -33,7 +33,7 @@ func New(cfg Config) *Source {
if cfg.APIPath == "" { if cfg.APIPath == "" {
cfg.APIPath = "/api" cfg.APIPath = "/api"
} }
c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) c := flare.NewClient(flare.WithRateLimit(1, 2))
return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)}
} }
sources/base/raijinscans/raijinscans.go
+3 -3
View File
@@ -11,7 +11,7 @@ import (
"github.com/PuerkitoBio/goquery" "github.com/PuerkitoBio/goquery"
"goyomi/internal/httpclient" "goyomi/internal/httpclient/flare"
"goyomi/internal/source" "goyomi/internal/source"
"goyomi/sources/base/util" "goyomi/sources/base/util"
) )
@@ -24,12 +24,12 @@ type Config struct {
type Source struct { type Source struct {
cfg Config cfg Config
client *httpclient.Client client *flare.Client
id int64 id int64
} }
func New(cfg Config) *Source { func New(cfg Config) *Source {
c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) c := flare.NewClient(flare.WithRateLimit(1, 2))
return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)}
} }
sources/base/senkuro/senkuro.go
+4 -11
View File
@@ -2,15 +2,13 @@
package senkuro package senkuro
import ( import (
"bytes"
"context" "context"
"encoding/json" "encoding/json"
"fmt" "fmt"
"io" "io"
"net/http"
"strings" "strings"
"goyomi/internal/httpclient" "goyomi/internal/httpclient/flare"
"goyomi/internal/source" "goyomi/internal/source"
"goyomi/sources/base/util" "goyomi/sources/base/util"
) )
@@ -24,7 +22,7 @@ type Config struct {
type Source struct { type Source struct {
cfg Config cfg Config
client *httpclient.Client client *flare.Client
id int64 id int64
} }
@@ -32,7 +30,7 @@ func New(cfg Config) *Source {
if cfg.APIURL == "" { if cfg.APIURL == "" {
cfg.APIURL = strings.TrimRight(cfg.BaseURL, "/") + "/api/graphql" cfg.APIURL = strings.TrimRight(cfg.BaseURL, "/") + "/api/graphql"
} }
c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) c := flare.NewClient(flare.WithRateLimit(3, 1))
return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)}
} }
@@ -48,12 +46,7 @@ type gqlRequest struct {
func (s *Source) gql(ctx context.Context, query string, vars map[string]any, out any) error { func (s *Source) gql(ctx context.Context, query string, vars map[string]any, out any) error {
body, _ := json.Marshal(gqlRequest{Query: query, Variables: vars}) body, _ := json.Marshal(gqlRequest{Query: query, Variables: vars})
req, err := http.NewRequestWithContext(ctx, http.MethodPost, s.cfg.APIURL, bytes.NewReader(body)) resp, err := s.client.Post(ctx, s.cfg.APIURL, "application/json", strings.NewReader(string(body)))
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/json")
resp, err := s.client.Do(req)
if err != nil { if err != nil {
return err return err
} }
sources/base/sinmh/sinmh.go
+3 -3
View File
@@ -13,7 +13,7 @@ import (
"github.com/PuerkitoBio/goquery" "github.com/PuerkitoBio/goquery"
"goyomi/internal/httpclient" "goyomi/internal/httpclient/flare"
"goyomi/internal/source" "goyomi/internal/source"
) )
@@ -26,7 +26,7 @@ type Config struct {
type Source struct { type Source struct {
cfg Config cfg Config
client *httpclient.Client client *flare.Client
id int64 id int64
imageHost string imageHost string
categories []Category categories []Category
@@ -43,7 +43,7 @@ func New(cfg Config) *Source {
if cfg.MobileURL == "" { if cfg.MobileURL == "" {
cfg.MobileURL = strings.Replace(cfg.BaseURL, "www.", "m.", 1) cfg.MobileURL = strings.Replace(cfg.BaseURL, "www.", "m.", 1)
} }
c := httpclient.NewClient(httpclient.WithRateLimit(2, 1)) c := flare.NewClient(flare.WithRateLimit(2, 1))
s := &Source{ s := &Source{
cfg: cfg, cfg: cfg,
client: c, client: c,
sources/base/spicytheme/spicytheme.go
+3 -3
View File
@@ -9,7 +9,7 @@ import (
"net/url" "net/url"
"time" "time"
"goyomi/internal/httpclient" "goyomi/internal/httpclient/flare"
"goyomi/internal/source" "goyomi/internal/source"
) )
@@ -22,12 +22,12 @@ type Config struct {
type Source struct { type Source struct {
cfg Config cfg Config
client *httpclient.Client client *flare.Client
id int64 id int64
} }
func New(cfg Config) *Source { func New(cfg Config) *Source {
c := httpclient.NewClient() c := flare.NewClient(flare.WithRateLimit(1, 2))
return &Source{ return &Source{
cfg: cfg, cfg: cfg,
client: c, client: c,
sources/base/stalkercms/stalkercms.go
+3 -3
View File
@@ -12,7 +12,7 @@ import (
"github.com/PuerkitoBio/goquery" "github.com/PuerkitoBio/goquery"
"goyomi/internal/httpclient" "goyomi/internal/httpclient/flare"
"goyomi/internal/source" "goyomi/internal/source"
) )
@@ -26,7 +26,7 @@ type Config struct {
type Source struct { type Source struct {
cfg Config cfg Config
client *httpclient.Client client *flare.Client
id int64 id int64
} }
@@ -37,7 +37,7 @@ func New(cfg Config) *Source {
if cfg.Lang == "" { if cfg.Lang == "" {
cfg.Lang = "pt-BR" cfg.Lang = "pt-BR"
} }
c := httpclient.NewClient(httpclient.WithRateLimit(2, 1)) c := flare.NewClient(flare.WithRateLimit(2, 1))
return &Source{ return &Source{
cfg: cfg, cfg: cfg,
client: c, client: c,
sources/base/uzaymanga/uzaymanga.go
+3 -3
View File
@@ -13,7 +13,7 @@ import (
"github.com/PuerkitoBio/goquery" "github.com/PuerkitoBio/goquery"
"goyomi/internal/httpclient" "goyomi/internal/httpclient/flare"
"goyomi/internal/source" "goyomi/internal/source"
) )
@@ -26,12 +26,12 @@ type Config struct {
type Source struct { type Source struct {
cfg Config cfg Config
client *httpclient.Client client *flare.Client
id int64 id int64
} }
func New(cfg Config) *Source { func New(cfg Config) *Source {
c := httpclient.NewClient(httpclient.WithRateLimit(3, 1)) c := flare.NewClient(flare.WithRateLimit(3, 1))
return &Source{ return &Source{
cfg: cfg, cfg: cfg,
client: c, client: c,
sources/base/wpcomics/wpcomics.go
+4 -9
View File
@@ -10,7 +10,7 @@ import (
"github.com/PuerkitoBio/goquery" "github.com/PuerkitoBio/goquery"
"goyomi/internal/httpclient" "goyomi/internal/httpclient/flare"
"goyomi/internal/source" "goyomi/internal/source"
"goyomi/sources/base/util" "goyomi/sources/base/util"
) )
@@ -24,7 +24,7 @@ type Config struct {
type Source struct { type Source struct {
cfg Config cfg Config
client *httpclient.Client client *flare.Client
id int64 id int64
} }
@@ -32,7 +32,7 @@ func New(cfg Config) *Source {
if cfg.PopularPath == "" { if cfg.PopularPath == "" {
cfg.PopularPath = "hot" cfg.PopularPath = "hot"
} }
c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) c := flare.NewClient(flare.WithRateLimit(1, 2))
return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)}
} }
@@ -42,12 +42,7 @@ func (s *Source) Lang() string { return s.cfg.Lang }
func (s *Source) SupportsLatest() bool { return true } func (s *Source) SupportsLatest() bool { return true }
func (s *Source) get(ctx context.Context, rawURL string) (*goquery.Document, error) { func (s *Source) get(ctx context.Context, rawURL string) (*goquery.Document, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) resp, err := s.client.Get(ctx, rawURL)
if err != nil {
return nil, err
}
req.Header.Set("Referer", s.cfg.BaseURL+"/")
resp, err := s.client.Do(req)
if err != nil { if err != nil {
return nil, err return nil, err
} }
sources/base/yuyu/yuyu.go
+3 -3
View File
@@ -12,7 +12,7 @@ import (
"github.com/PuerkitoBio/goquery" "github.com/PuerkitoBio/goquery"
"goyomi/internal/httpclient" "goyomi/internal/httpclient/flare"
"goyomi/internal/source" "goyomi/internal/source"
) )
@@ -24,12 +24,12 @@ type Config struct {
type Source struct { type Source struct {
cfg Config cfg Config
client *httpclient.Client client *flare.Client
id int64 id int64
} }
func New(cfg Config) *Source { func New(cfg Config) *Source {
c := httpclient.NewClient() c := flare.NewClient(flare.WithRateLimit(1, 2))
return &Source{ return &Source{
cfg: cfg, cfg: cfg,
client: c, client: c,
sources/base/zeistmanga/zeistmanga.go
+3 -3
View File
@@ -12,7 +12,7 @@ import (
"github.com/PuerkitoBio/goquery" "github.com/PuerkitoBio/goquery"
"goyomi/internal/httpclient" "goyomi/internal/httpclient/flare"
"goyomi/internal/source" "goyomi/internal/source"
) )
@@ -24,12 +24,12 @@ type Config struct {
type Source struct { type Source struct {
cfg Config cfg Config
client *httpclient.Client client *flare.Client
id int64 id int64
} }
func New(cfg Config) *Source { func New(cfg Config) *Source {
c := httpclient.NewClient() c := flare.NewClient(flare.WithRateLimit(1, 2))
return &Source{ return &Source{
cfg: cfg, cfg: cfg,
client: c, client: c,