From 9a42dd2ab1fc62b18ef0a48948c196204c60637a Mon Sep 17 00:00:00 2001 From: achmad Date: Wed, 13 May 2026 09:01:51 +0700 Subject: [PATCH] refactor: use per-source HTTP client instead of global proxy - Remove global ProxyEnabled() logic from httpclient - Each source now explicitly chooses client at import time: - flare client: for JS-rendering/cloudflare sources - normal httpclient: for REST API sources - Updated 29 base sources based on Kotlin reference (network.cloudflareClient) --- cmd/server/main.go | 12 --- internal/httpclient/client.go | 109 +--------------------- sources/base/comicaso/comicaso.go | 6 +- sources/base/comiciviewer/comiciviewer.go | 6 +- sources/base/fuzzydoodle/fuzzydoodle.go | 6 +- sources/base/gattsu/gattsu.go | 13 +-- sources/base/gigaviewer/gigaviewer.go | 6 +- sources/base/gmanga/gmanga.go | 6 +- sources/base/goda/goda.go | 6 +- sources/base/hotcomics/hotcomics.go | 6 +- sources/base/iken/iken.go | 6 +- sources/base/kemono/kemono.go | 6 +- sources/base/keyoapp/keyoapp.go | 6 +- sources/base/manga18/manga18.go | 6 +- sources/base/mangataro/mangataro.go | 24 +---- sources/base/manhwaz/manhwaz.go | 14 +-- sources/base/masonry/masonry.go | 6 +- sources/base/mmlook/mmlook.go | 6 +- sources/base/mmrcms/mmrcms.go | 6 +- sources/base/paprika/paprika.go | 13 +-- sources/base/peachscan/peachscan.go | 6 +- sources/base/pizzareader/pizzareader.go | 6 +- sources/base/raijinscans/raijinscans.go | 6 +- sources/base/senkuro/senkuro.go | 15 +-- sources/base/sinmh/sinmh.go | 6 +- sources/base/spicytheme/spicytheme.go | 6 +- sources/base/stalkercms/stalkercms.go | 6 +- sources/base/uzaymanga/uzaymanga.go | 6 +- sources/base/wpcomics/wpcomics.go | 13 +-- sources/base/yuyu/yuyu.go | 6 +- sources/base/zeistmanga/zeistmanga.go | 6 +- 31 files changed, 96 insertions(+), 255 deletions(-) diff --git a/cmd/server/main.go b/cmd/server/main.go index 9f0dc30..819f9c3 100755 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -6,11 +6,9 @@ import ( "fmt" "log" "net/http" - "os" "goyomi/internal/config" "goyomi/internal/db" - "goyomi/internal/httpclient" _ "goyomi/internal/registry" _ "goyomi/sources/all/hentaihand" _ "goyomi/sources/all/kemono" @@ -46,16 +44,6 @@ func main() { // Initialize config manager config.InitConfigManager(database.Queries) - if os.Getenv("FLARESOLVERR_URL") != "" { - fsClient, err := httpclient.NewFlareSolverrClient() - if err != nil { - log.Printf("flaresolverr: client creation failed: %v", err) - } else { - httpclient.SetGlobalFlareSolverr(fsClient) - log.Printf("flaresolverr: client initialized") - } - } - mux := http.NewServeMux() mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) { fmt.Fprintln(w, "ok") diff --git a/internal/httpclient/client.go b/internal/httpclient/client.go index b343040..12ff4d0 100755 --- a/internal/httpclient/client.go +++ b/internal/httpclient/client.go @@ -2,56 +2,27 @@ package httpclient import ( "context" - "fmt" "io" "log" "net/http" "net/http/cookiejar" - "os" "strconv" - "strings" "sync" "time" - "goyomi/internal/config" "golang.org/x/time/rate" ) const defaultUserAgent = "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Mobile Safari/537.36" -var ( - globalFSClient *FlareSolverrClient - globalProxy bool - verboseLog bool -) - -func SetGlobalFlareSolverr(client *FlareSolverrClient) { - globalFSClient = client -} - -func SetGlobalProxy(enabled bool) { - globalProxy = enabled -} +var verboseLog bool func SetVerboseLog(enabled bool) { verboseLog = enabled } -func ProxyEnabled() bool { - envProxy := os.Getenv("FLARESOLVERR_PROXY") == "1" - if globalProxy || envProxy { - return true - } - if config.IsFlareSolverrProxyEnabled() { - return true - } - return false -} - type Client struct { http *http.Client - fsClient *FlareSolverrClient - proxy bool rateLimit float64 burst int referer string @@ -78,29 +49,18 @@ func WithReferer(referer string) Option { return func(c *Client) { c.referer = referer } } -func WithFlareSolverr(fs *FlareSolverrClient) Option { - return func(c *Client) { c.fsClient = fs } -} - -func WithProxy(enabled bool) Option { - return func(c *Client) { c.proxy = enabled } -} - func WithVerboseLog(enabled bool) Option { return func(c *Client) { c.verboseLog = enabled } } func NewClient(opts ...Option) *Client { jar, _ := cookiejar.New(nil) - vLog := os.Getenv("VERBOSE_LOG") == "1" c := &Client{ http: &http.Client{Timeout: 30 * time.Second, Jar: jar}, - fsClient: globalFSClient, - proxy: globalProxy, rateLimit: 1, burst: 1, limiters: map[string]*rate.Limiter{}, - verboseLog: verboseLog || vLog, + verboseLog: verboseLog, } for _, o := range opts { o(c) @@ -120,9 +80,6 @@ func (c *Client) limiter(host string) *rate.Limiter { } func (c *Client) Do(req *http.Request) (*http.Response, error) { - if (c.proxy || ProxyEnabled()) && c.fsClient != nil { - return c.doViaFlareSolverr(req) - } return c.doDirect(req) } @@ -167,68 +124,6 @@ func (c *Client) doDirect(req *http.Request) (*http.Response, error) { panic("unreachable") } -func (c *Client) doViaFlareSolverr(req *http.Request) (*http.Response, error) { - if err := c.limiter(req.URL.Host).Wait(req.Context()); err != nil { - return nil, err - } - - url := req.URL.String() - - if c.verboseLog { - log.Printf("[httpclient] FLARESOLVERR %s %s", req.Method, url) - } - - var html string - var cookies []*http.Cookie - var err error - - if req.Method == http.MethodGet { - html, cookies, err = c.fsClient.Get(req.Context(), url) - } else if req.Method == http.MethodPost { - var bodyStr string - if req.Body != nil { - bodyBytes, _ := io.ReadAll(req.Body) - bodyStr = string(bodyBytes) - if c.verboseLog { - log.Printf("[httpclient] FLARESOLVERR POST BODY: %s", bodyStr) - } - } - html, cookies, err = c.fsClient.Post(req.Context(), url, bodyStr) - } else { - return c.doDirect(req) - } - - if err != nil { - return nil, fmt.Errorf("flaresolverr: %w", err) - } - - if c.verboseLog { - log.Printf("[httpclient] FLARESOLVERR RESPONSE length=%d cookies=%d", len(html), len(cookies)) - if len(html) > 0 { - truncate := html - if len(html) > 500 { - truncate = html[:500] - } - log.Printf("[httpclient] FLARESOLVERR RESPONSE HTML (first 500 chars): %s", truncate) - } - } - - resp := &http.Response{ - StatusCode: http.StatusOK, - Header: http.Header{}, - Body: io.NopCloser(strings.NewReader(html)), - } - resp.Header.Set("Content-Type", "text/html; charset=utf-8") - - if c.http.Jar != nil { - for _, cookie := range cookies { - c.http.Jar.SetCookies(req.URL, []*http.Cookie{cookie}) - } - } - - return resp, nil -} - func retryAfter(resp *http.Response) time.Duration { ra := resp.Header.Get("Retry-After") if ra == "" { diff --git a/sources/base/comicaso/comicaso.go b/sources/base/comicaso/comicaso.go index a28c2b6..60c7a71 100755 --- a/sources/base/comicaso/comicaso.go +++ b/sources/base/comicaso/comicaso.go @@ -12,7 +12,7 @@ import ( "strings" "sync" - "goyomi/internal/httpclient" + "goyomi/internal/httpclient/flare" "goyomi/internal/source" "goyomi/sources/base/util" ) @@ -27,7 +27,7 @@ type Config struct { type Source struct { cfg Config - client *httpclient.Client + client *flare.Client id int64 mu sync.Mutex @@ -68,7 +68,7 @@ type tokenResponseDTO struct { } func New(cfg Config) *Source { - c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) + c := flare.NewClient(flare.WithRateLimit(1, 2)) return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} } diff --git a/sources/base/comiciviewer/comiciviewer.go b/sources/base/comiciviewer/comiciviewer.go index 9b3daad..de7e401 100755 --- a/sources/base/comiciviewer/comiciviewer.go +++ b/sources/base/comiciviewer/comiciviewer.go @@ -12,7 +12,7 @@ import ( "github.com/PuerkitoBio/goquery" - "goyomi/internal/httpclient" + "goyomi/internal/httpclient/flare" "goyomi/internal/source" "goyomi/sources/base/util" ) @@ -25,12 +25,12 @@ type Config struct { type Source struct { cfg Config - client *httpclient.Client + client *flare.Client id int64 } func New(cfg Config) *Source { - c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) + c := flare.NewClient(flare.WithRateLimit(1, 2)) return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} } diff --git a/sources/base/fuzzydoodle/fuzzydoodle.go b/sources/base/fuzzydoodle/fuzzydoodle.go index 4761762..82602b0 100755 --- a/sources/base/fuzzydoodle/fuzzydoodle.go +++ b/sources/base/fuzzydoodle/fuzzydoodle.go @@ -10,7 +10,7 @@ import ( "github.com/PuerkitoBio/goquery" - "goyomi/internal/httpclient" + "goyomi/internal/httpclient/flare" "goyomi/internal/source" "goyomi/sources/base/util" ) @@ -23,12 +23,12 @@ type Config struct { type Source struct { cfg Config - client *httpclient.Client + client *flare.Client id int64 } func New(cfg Config) *Source { - c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) + c := flare.NewClient(flare.WithRateLimit(1, 2)) return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} } diff --git a/sources/base/gattsu/gattsu.go b/sources/base/gattsu/gattsu.go index 132e29e..537c0e8 100755 --- a/sources/base/gattsu/gattsu.go +++ b/sources/base/gattsu/gattsu.go @@ -11,7 +11,7 @@ import ( "github.com/PuerkitoBio/goquery" - "goyomi/internal/httpclient" + "goyomi/internal/httpclient/flare" "goyomi/internal/source" "goyomi/sources/base/util" ) @@ -24,12 +24,12 @@ type Config struct { type Source struct { cfg Config - client *httpclient.Client + client *flare.Client id int64 } func New(cfg Config) *Source { - c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) + c := flare.NewClient(flare.WithRateLimit(1, 2)) return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} } @@ -41,12 +41,7 @@ func (s *Source) SupportsLatest() bool { return true } func (s *Source) base() string { return strings.TrimRight(s.cfg.BaseURL, "/") } func (s *Source) get(ctx context.Context, rawURL string) (*goquery.Document, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) - if err != nil { - return nil, err - } - req.Header.Set("Referer", s.cfg.BaseURL+"/") - resp, err := s.client.Do(req) + resp, err := s.client.Get(ctx, rawURL) if err != nil { return nil, err } diff --git a/sources/base/gigaviewer/gigaviewer.go b/sources/base/gigaviewer/gigaviewer.go index 7e69e4e..42a1885 100755 --- a/sources/base/gigaviewer/gigaviewer.go +++ b/sources/base/gigaviewer/gigaviewer.go @@ -10,7 +10,7 @@ import ( "github.com/PuerkitoBio/goquery" - "goyomi/internal/httpclient" + "goyomi/internal/httpclient/flare" "goyomi/internal/source" "goyomi/sources/base/util" ) @@ -23,12 +23,12 @@ type Config struct { type Source struct { cfg Config - client *httpclient.Client + client *flare.Client id int64 } func New(cfg Config) *Source { - c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) + c := flare.NewClient(flare.WithRateLimit(1, 2)) return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} } diff --git a/sources/base/gmanga/gmanga.go b/sources/base/gmanga/gmanga.go index e5a5b79..4c73bec 100755 --- a/sources/base/gmanga/gmanga.go +++ b/sources/base/gmanga/gmanga.go @@ -11,7 +11,7 @@ import ( "net/http" "strings" - "goyomi/internal/httpclient" + "goyomi/internal/httpclient/flare" "goyomi/internal/source" "goyomi/sources/base/util" ) @@ -24,12 +24,12 @@ type Config struct { type Source struct { cfg Config - client *httpclient.Client + client *flare.Client id int64 } func New(cfg Config) *Source { - c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) + c := flare.NewClient(flare.WithRateLimit(1, 2)) return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} } diff --git a/sources/base/goda/goda.go b/sources/base/goda/goda.go index 0dd3dc0..f883915 100755 --- a/sources/base/goda/goda.go +++ b/sources/base/goda/goda.go @@ -12,7 +12,7 @@ import ( "github.com/PuerkitoBio/goquery" - "goyomi/internal/httpclient" + "goyomi/internal/httpclient/flare" "goyomi/internal/source" ) @@ -24,12 +24,12 @@ type Config struct { type Source struct { cfg Config - client *httpclient.Client + client *flare.Client id int64 } func New(cfg Config) *Source { - c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) + c := flare.NewClient(flare.WithRateLimit(1, 2)) return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} } diff --git a/sources/base/hotcomics/hotcomics.go b/sources/base/hotcomics/hotcomics.go index 9bae447..62e5598 100755 --- a/sources/base/hotcomics/hotcomics.go +++ b/sources/base/hotcomics/hotcomics.go @@ -11,7 +11,7 @@ import ( "github.com/PuerkitoBio/goquery" - "goyomi/internal/httpclient" + "goyomi/internal/httpclient/flare" "goyomi/internal/source" "goyomi/sources/base/util" ) @@ -24,12 +24,12 @@ type Config struct { type Source struct { cfg Config - client *httpclient.Client + client *flare.Client id int64 } func New(cfg Config) *Source { - c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) + c := flare.NewClient(flare.WithRateLimit(1, 2)) return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} } diff --git a/sources/base/iken/iken.go b/sources/base/iken/iken.go index 2999f97..af6b25e 100755 --- a/sources/base/iken/iken.go +++ b/sources/base/iken/iken.go @@ -10,7 +10,7 @@ import ( "net/http" "strings" - "goyomi/internal/httpclient" + "goyomi/internal/httpclient/flare" "goyomi/internal/source" "goyomi/sources/base/util" ) @@ -26,7 +26,7 @@ type Config struct { type Source struct { cfg Config - client *httpclient.Client + client *flare.Client id int64 } @@ -34,7 +34,7 @@ func New(cfg Config) *Source { if cfg.APIURL == "" { cfg.APIURL = cfg.BaseURL } - c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) + c := flare.NewClient(flare.WithRateLimit(1, 2)) return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} } diff --git a/sources/base/kemono/kemono.go b/sources/base/kemono/kemono.go index 20f632f..1c1e97f 100755 --- a/sources/base/kemono/kemono.go +++ b/sources/base/kemono/kemono.go @@ -11,7 +11,7 @@ import ( "net/http" "strings" - "goyomi/internal/httpclient" + "goyomi/internal/httpclient/flare" "goyomi/internal/source" "goyomi/sources/base/util" ) @@ -47,12 +47,12 @@ type postDTO struct { type Source struct { cfg Config - client *httpclient.Client + client *flare.Client id int64 } func New(cfg Config) *Source { - c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) + c := flare.NewClient(flare.WithRateLimit(1, 2)) return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} } diff --git a/sources/base/keyoapp/keyoapp.go b/sources/base/keyoapp/keyoapp.go index 8c31525..817f02b 100755 --- a/sources/base/keyoapp/keyoapp.go +++ b/sources/base/keyoapp/keyoapp.go @@ -12,7 +12,7 @@ import ( "github.com/PuerkitoBio/goquery" - "goyomi/internal/httpclient" + "goyomi/internal/httpclient/flare" "goyomi/internal/source" "goyomi/sources/base/util" ) @@ -25,12 +25,12 @@ type Config struct { type Source struct { cfg Config - client *httpclient.Client + client *flare.Client id int64 } func New(cfg Config) *Source { - c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) + c := flare.NewClient(flare.WithRateLimit(1, 2)) return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} } diff --git a/sources/base/manga18/manga18.go b/sources/base/manga18/manga18.go index dcc3949..f5d0f37 100755 --- a/sources/base/manga18/manga18.go +++ b/sources/base/manga18/manga18.go @@ -12,7 +12,7 @@ import ( "github.com/PuerkitoBio/goquery" - "goyomi/internal/httpclient" + "goyomi/internal/httpclient/flare" "goyomi/internal/source" "goyomi/sources/base/util" ) @@ -25,12 +25,12 @@ type Config struct { type Source struct { cfg Config - client *httpclient.Client + client *flare.Client id int64 } func New(cfg Config) *Source { - c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) + c := flare.NewClient(flare.WithRateLimit(1, 2)) return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} } diff --git a/sources/base/mangataro/mangataro.go b/sources/base/mangataro/mangataro.go index e73b0eb..5d29d5f 100755 --- a/sources/base/mangataro/mangataro.go +++ b/sources/base/mangataro/mangataro.go @@ -3,7 +3,6 @@ package mangataro import ( - "bytes" "context" "crypto/md5" "encoding/json" @@ -14,7 +13,7 @@ import ( "strings" "time" - "goyomi/internal/httpclient" + "goyomi/internal/httpclient/flare" "goyomi/internal/source" "goyomi/sources/base/util" ) @@ -27,12 +26,12 @@ type Config struct { type Source struct { cfg Config - client *httpclient.Client + client *flare.Client id int64 } func New(cfg Config) *Source { - c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) + c := flare.NewClient(flare.WithRateLimit(1, 2)) return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} } @@ -44,13 +43,7 @@ func (s *Source) SupportsLatest() bool { return true } func (s *Source) base() string { return strings.TrimRight(s.cfg.BaseURL, "/") } func (s *Source) doGet(ctx context.Context, rawURL string, out any) error { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) - if err != nil { - return err - } - req.Header.Set("Referer", s.cfg.BaseURL+"/") - req.Header.Set("Accept", "application/json") - resp, err := s.client.Do(req) + resp, err := s.client.Get(ctx, rawURL) if err != nil { return err } @@ -67,14 +60,7 @@ func (s *Source) doPost(ctx context.Context, rawURL string, payload any, out any if err != nil { return err } - req, err := http.NewRequestWithContext(ctx, http.MethodPost, rawURL, bytes.NewReader(body)) - if err != nil { - return err - } - req.Header.Set("Referer", s.cfg.BaseURL+"/") - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Accept", "application/json") - resp, err := s.client.Do(req) + resp, err := s.client.Post(ctx, rawURL, "application/json", strings.NewReader(string(body))) if err != nil { return err } diff --git a/sources/base/manhwaz/manhwaz.go b/sources/base/manhwaz/manhwaz.go index 0e1329c..a369f10 100755 --- a/sources/base/manhwaz/manhwaz.go +++ b/sources/base/manhwaz/manhwaz.go @@ -10,7 +10,7 @@ import ( "github.com/PuerkitoBio/goquery" - "goyomi/internal/httpclient" + "goyomi/internal/httpclient/flare" "goyomi/internal/source" "goyomi/sources/base/util" ) @@ -26,7 +26,7 @@ type Config struct { type Source struct { cfg Config - client *httpclient.Client + client *flare.Client id int64 } @@ -40,7 +40,7 @@ func New(cfg Config) *Source { if cfg.StatusHeading == "" { cfg.StatusHeading = "status" } - c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) + c := flare.NewClient(flare.WithRateLimit(1, 2)) return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} } @@ -52,13 +52,7 @@ func (s *Source) SupportsLatest() bool { return true } func (s *Source) base() string { return strings.TrimRight(s.cfg.BaseURL, "/") } func (s *Source) get(ctx context.Context, rawURL string) (*goquery.Document, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) - if err != nil { - return nil, err - } - req.Header.Set("Referer", s.cfg.BaseURL+"/") - req.Header.Set("Origin", s.cfg.BaseURL) - resp, err := s.client.Do(req) + resp, err := s.client.Get(ctx, rawURL) if err != nil { return nil, err } diff --git a/sources/base/masonry/masonry.go b/sources/base/masonry/masonry.go index 67232cc..0d48c52 100755 --- a/sources/base/masonry/masonry.go +++ b/sources/base/masonry/masonry.go @@ -10,7 +10,7 @@ import ( "github.com/PuerkitoBio/goquery" - "goyomi/internal/httpclient" + "goyomi/internal/httpclient/flare" "goyomi/internal/source" "goyomi/sources/base/util" ) @@ -23,12 +23,12 @@ type Config struct { type Source struct { cfg Config - client *httpclient.Client + client *flare.Client id int64 } func New(cfg Config) *Source { - c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) + c := flare.NewClient(flare.WithRateLimit(1, 2)) return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} } diff --git a/sources/base/mmlook/mmlook.go b/sources/base/mmlook/mmlook.go index aad3681..b9d8314 100755 --- a/sources/base/mmlook/mmlook.go +++ b/sources/base/mmlook/mmlook.go @@ -11,7 +11,7 @@ import ( "github.com/PuerkitoBio/goquery" - "goyomi/internal/httpclient" + "goyomi/internal/httpclient/flare" "goyomi/internal/source" "goyomi/sources/base/util" ) @@ -26,7 +26,7 @@ type Config struct { type Source struct { cfg Config - client *httpclient.Client + client *flare.Client id int64 } @@ -37,7 +37,7 @@ func New(cfg Config) *Source { if cfg.Lang == "" { cfg.Lang = "zh" } - c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) + c := flare.NewClient(flare.WithRateLimit(1, 2)) return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} } diff --git a/sources/base/mmrcms/mmrcms.go b/sources/base/mmrcms/mmrcms.go index 5722531..e8aaaff 100755 --- a/sources/base/mmrcms/mmrcms.go +++ b/sources/base/mmrcms/mmrcms.go @@ -10,7 +10,7 @@ import ( "net/http" "strings" - "goyomi/internal/httpclient" + "goyomi/internal/httpclient/flare" "goyomi/internal/source" "goyomi/sources/base/util" ) @@ -30,12 +30,12 @@ type mangaDTO struct { type Source struct { cfg Config - client *httpclient.Client + client *flare.Client id int64 } func New(cfg Config) *Source { - c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) + c := flare.NewClient(flare.WithRateLimit(1, 2)) return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} } diff --git a/sources/base/paprika/paprika.go b/sources/base/paprika/paprika.go index b2e0189..edfa5f3 100755 --- a/sources/base/paprika/paprika.go +++ b/sources/base/paprika/paprika.go @@ -10,7 +10,7 @@ import ( "github.com/PuerkitoBio/goquery" - "goyomi/internal/httpclient" + "goyomi/internal/httpclient/flare" "goyomi/internal/source" "goyomi/sources/base/util" ) @@ -23,12 +23,12 @@ type Config struct { type Source struct { cfg Config - client *httpclient.Client + client *flare.Client id int64 } func New(cfg Config) *Source { - c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) + c := flare.NewClient(flare.WithRateLimit(1, 2)) return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} } @@ -40,12 +40,7 @@ func (s *Source) SupportsLatest() bool { return true } func (s *Source) base() string { return strings.TrimRight(s.cfg.BaseURL, "/") } func (s *Source) get(ctx context.Context, rawURL string) (*goquery.Document, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) - if err != nil { - return nil, err - } - req.Header.Set("Referer", s.cfg.BaseURL+"/") - resp, err := s.client.Do(req) + resp, err := s.client.Get(ctx, rawURL) if err != nil { return nil, err } diff --git a/sources/base/peachscan/peachscan.go b/sources/base/peachscan/peachscan.go index cfa7370..c202af0 100755 --- a/sources/base/peachscan/peachscan.go +++ b/sources/base/peachscan/peachscan.go @@ -12,7 +12,7 @@ import ( "github.com/PuerkitoBio/goquery" - "goyomi/internal/httpclient" + "goyomi/internal/httpclient/flare" "goyomi/internal/source" "goyomi/sources/base/util" ) @@ -27,12 +27,12 @@ type Config struct { type Source struct { cfg Config - client *httpclient.Client + client *flare.Client id int64 } func New(cfg Config) *Source { - c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) + c := flare.NewClient(flare.WithRateLimit(1, 2)) return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} } diff --git a/sources/base/pizzareader/pizzareader.go b/sources/base/pizzareader/pizzareader.go index bad7b47..7082e95 100755 --- a/sources/base/pizzareader/pizzareader.go +++ b/sources/base/pizzareader/pizzareader.go @@ -11,7 +11,7 @@ import ( "sort" "strings" - "goyomi/internal/httpclient" + "goyomi/internal/httpclient/flare" "goyomi/internal/source" "goyomi/sources/base/util" ) @@ -25,7 +25,7 @@ type Config struct { type Source struct { cfg Config - client *httpclient.Client + client *flare.Client id int64 } @@ -33,7 +33,7 @@ func New(cfg Config) *Source { if cfg.APIPath == "" { cfg.APIPath = "/api" } - c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) + c := flare.NewClient(flare.WithRateLimit(1, 2)) return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} } diff --git a/sources/base/raijinscans/raijinscans.go b/sources/base/raijinscans/raijinscans.go index f7fe873..ad1d41e 100755 --- a/sources/base/raijinscans/raijinscans.go +++ b/sources/base/raijinscans/raijinscans.go @@ -11,7 +11,7 @@ import ( "github.com/PuerkitoBio/goquery" - "goyomi/internal/httpclient" + "goyomi/internal/httpclient/flare" "goyomi/internal/source" "goyomi/sources/base/util" ) @@ -24,12 +24,12 @@ type Config struct { type Source struct { cfg Config - client *httpclient.Client + client *flare.Client id int64 } func New(cfg Config) *Source { - c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) + c := flare.NewClient(flare.WithRateLimit(1, 2)) return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} } diff --git a/sources/base/senkuro/senkuro.go b/sources/base/senkuro/senkuro.go index 08c85a1..274a74a 100755 --- a/sources/base/senkuro/senkuro.go +++ b/sources/base/senkuro/senkuro.go @@ -2,15 +2,13 @@ package senkuro import ( - "bytes" "context" "encoding/json" "fmt" "io" - "net/http" "strings" - "goyomi/internal/httpclient" + "goyomi/internal/httpclient/flare" "goyomi/internal/source" "goyomi/sources/base/util" ) @@ -24,7 +22,7 @@ type Config struct { type Source struct { cfg Config - client *httpclient.Client + client *flare.Client id int64 } @@ -32,7 +30,7 @@ func New(cfg Config) *Source { if cfg.APIURL == "" { cfg.APIURL = strings.TrimRight(cfg.BaseURL, "/") + "/api/graphql" } - c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) + c := flare.NewClient(flare.WithRateLimit(3, 1)) return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} } @@ -48,12 +46,7 @@ type gqlRequest struct { func (s *Source) gql(ctx context.Context, query string, vars map[string]any, out any) error { body, _ := json.Marshal(gqlRequest{Query: query, Variables: vars}) - req, err := http.NewRequestWithContext(ctx, http.MethodPost, s.cfg.APIURL, bytes.NewReader(body)) - if err != nil { - return err - } - req.Header.Set("Content-Type", "application/json") - resp, err := s.client.Do(req) + resp, err := s.client.Post(ctx, s.cfg.APIURL, "application/json", strings.NewReader(string(body))) if err != nil { return err } diff --git a/sources/base/sinmh/sinmh.go b/sources/base/sinmh/sinmh.go index b392ec2..b0bd054 100755 --- a/sources/base/sinmh/sinmh.go +++ b/sources/base/sinmh/sinmh.go @@ -13,7 +13,7 @@ import ( "github.com/PuerkitoBio/goquery" - "goyomi/internal/httpclient" + "goyomi/internal/httpclient/flare" "goyomi/internal/source" ) @@ -26,7 +26,7 @@ type Config struct { type Source struct { cfg Config - client *httpclient.Client + client *flare.Client id int64 imageHost string categories []Category @@ -43,7 +43,7 @@ func New(cfg Config) *Source { if cfg.MobileURL == "" { cfg.MobileURL = strings.Replace(cfg.BaseURL, "www.", "m.", 1) } - c := httpclient.NewClient(httpclient.WithRateLimit(2, 1)) + c := flare.NewClient(flare.WithRateLimit(2, 1)) s := &Source{ cfg: cfg, client: c, diff --git a/sources/base/spicytheme/spicytheme.go b/sources/base/spicytheme/spicytheme.go index 3682439..55724aa 100755 --- a/sources/base/spicytheme/spicytheme.go +++ b/sources/base/spicytheme/spicytheme.go @@ -9,7 +9,7 @@ import ( "net/url" "time" - "goyomi/internal/httpclient" + "goyomi/internal/httpclient/flare" "goyomi/internal/source" ) @@ -22,12 +22,12 @@ type Config struct { type Source struct { cfg Config - client *httpclient.Client + client *flare.Client id int64 } func New(cfg Config) *Source { - c := httpclient.NewClient() + c := flare.NewClient(flare.WithRateLimit(1, 2)) return &Source{ cfg: cfg, client: c, diff --git a/sources/base/stalkercms/stalkercms.go b/sources/base/stalkercms/stalkercms.go index a7706aa..bbd6418 100755 --- a/sources/base/stalkercms/stalkercms.go +++ b/sources/base/stalkercms/stalkercms.go @@ -12,7 +12,7 @@ import ( "github.com/PuerkitoBio/goquery" - "goyomi/internal/httpclient" + "goyomi/internal/httpclient/flare" "goyomi/internal/source" ) @@ -26,7 +26,7 @@ type Config struct { type Source struct { cfg Config - client *httpclient.Client + client *flare.Client id int64 } @@ -37,7 +37,7 @@ func New(cfg Config) *Source { if cfg.Lang == "" { cfg.Lang = "pt-BR" } - c := httpclient.NewClient(httpclient.WithRateLimit(2, 1)) + c := flare.NewClient(flare.WithRateLimit(2, 1)) return &Source{ cfg: cfg, client: c, diff --git a/sources/base/uzaymanga/uzaymanga.go b/sources/base/uzaymanga/uzaymanga.go index 33248c4..06621f9 100755 --- a/sources/base/uzaymanga/uzaymanga.go +++ b/sources/base/uzaymanga/uzaymanga.go @@ -13,7 +13,7 @@ import ( "github.com/PuerkitoBio/goquery" - "goyomi/internal/httpclient" + "goyomi/internal/httpclient/flare" "goyomi/internal/source" ) @@ -26,12 +26,12 @@ type Config struct { type Source struct { cfg Config - client *httpclient.Client + client *flare.Client id int64 } func New(cfg Config) *Source { - c := httpclient.NewClient(httpclient.WithRateLimit(3, 1)) + c := flare.NewClient(flare.WithRateLimit(3, 1)) return &Source{ cfg: cfg, client: c, diff --git a/sources/base/wpcomics/wpcomics.go b/sources/base/wpcomics/wpcomics.go index 32789b6..f292dcf 100755 --- a/sources/base/wpcomics/wpcomics.go +++ b/sources/base/wpcomics/wpcomics.go @@ -10,7 +10,7 @@ import ( "github.com/PuerkitoBio/goquery" - "goyomi/internal/httpclient" + "goyomi/internal/httpclient/flare" "goyomi/internal/source" "goyomi/sources/base/util" ) @@ -24,7 +24,7 @@ type Config struct { type Source struct { cfg Config - client *httpclient.Client + client *flare.Client id int64 } @@ -32,7 +32,7 @@ func New(cfg Config) *Source { if cfg.PopularPath == "" { cfg.PopularPath = "hot" } - c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) + c := flare.NewClient(flare.WithRateLimit(1, 2)) return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} } @@ -42,12 +42,7 @@ func (s *Source) Lang() string { return s.cfg.Lang } func (s *Source) SupportsLatest() bool { return true } func (s *Source) get(ctx context.Context, rawURL string) (*goquery.Document, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) - if err != nil { - return nil, err - } - req.Header.Set("Referer", s.cfg.BaseURL+"/") - resp, err := s.client.Do(req) + resp, err := s.client.Get(ctx, rawURL) if err != nil { return nil, err } diff --git a/sources/base/yuyu/yuyu.go b/sources/base/yuyu/yuyu.go index 82fe543..79ae32c 100755 --- a/sources/base/yuyu/yuyu.go +++ b/sources/base/yuyu/yuyu.go @@ -12,7 +12,7 @@ import ( "github.com/PuerkitoBio/goquery" - "goyomi/internal/httpclient" + "goyomi/internal/httpclient/flare" "goyomi/internal/source" ) @@ -24,12 +24,12 @@ type Config struct { type Source struct { cfg Config - client *httpclient.Client + client *flare.Client id int64 } func New(cfg Config) *Source { - c := httpclient.NewClient() + c := flare.NewClient(flare.WithRateLimit(1, 2)) return &Source{ cfg: cfg, client: c, diff --git a/sources/base/zeistmanga/zeistmanga.go b/sources/base/zeistmanga/zeistmanga.go index 3a81792..f63a475 100755 --- a/sources/base/zeistmanga/zeistmanga.go +++ b/sources/base/zeistmanga/zeistmanga.go @@ -12,7 +12,7 @@ import ( "github.com/PuerkitoBio/goquery" - "goyomi/internal/httpclient" + "goyomi/internal/httpclient/flare" "goyomi/internal/source" ) @@ -24,12 +24,12 @@ type Config struct { type Source struct { cfg Config - client *httpclient.Client + client *flare.Client id int64 } func New(cfg Config) *Source { - c := httpclient.NewClient() + c := flare.NewClient(flare.WithRateLimit(1, 2)) return &Source{ cfg: cfg, client: c,