diff --git a/internal/httpclient/flare/flare.go b/internal/httpclient/flare/flare.go new file mode 100644 index 0000000..fc4196b --- /dev/null +++ b/internal/httpclient/flare/flare.go @@ -0,0 +1,184 @@ +package flare + +import ( + "context" + "fmt" + "io" + "net/http" + "net/http/cookiejar" + "net/url" + "strings" + "sync" + "time" + + "goyomi/internal/httpclient" + "golang.org/x/time/rate" +) + +const defaultUserAgent = "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Mobile Safari/537.36" + +type Client struct { + fsClient *httpclient.FlareSolverrClient + rateLimit float64 + burst int + referer string + + mu sync.Mutex + limiters map[string]*rate.Limiter +} + +type Response struct { + *http.Response + Body io.ReadCloser +} + +func (r *Response) Close() error { + return r.Body.Close() +} + +type Option func(*Client) + +func WithRateLimit(rps float64, burst int) Option { + return func(c *Client) { + c.rateLimit = rps + c.burst = burst + } +} + +func WithReferer(referer string) Option { + return func(c *Client) { c.referer = referer } +} + +func WithTimeout(d time.Duration) Option { + return func(c *Client) {} +} + +func NewClient(opts ...Option) *Client { + c := &Client{ + limiters: make(map[string]*rate.Limiter), + } + for _, opt := range opts { + opt(c) + } + if c.rateLimit == 0 { + c.rateLimit = 1 + } + if c.burst == 0 { + c.burst = 1 + } + fsClient, err := httpclient.NewFlareSolverrClient() + if err == nil { + c.fsClient = fsClient + } + return c +} + +func (c *Client) SetFlareSolverrClient(fs *httpclient.FlareSolverrClient) { + c.fsClient = fs +} + +func (c *Client) doRequest(ctx context.Context, method string, rawURL string, body string) (*Response, error) { + if c.fsClient == nil { + return nil, fmt.Errorf("FlareSolverr client not configured") + } + c.mu.Lock() + limiter, ok := c.limiters[rawURL] + if !ok { + limiter = rate.NewLimiter(rate.Limit(c.rateLimit), c.burst) + c.limiters[rawURL] = limiter + } + c.mu.Unlock() + if err := limiter.Wait(ctx); err != nil { + return nil, err + } + + var html string + var cookies []*http.Cookie + var err error + + if method == http.MethodGet { + html, cookies, err = c.fsClient.Get(ctx, rawURL) + } else { + html, cookies, err = c.fsClient.Post(ctx, rawURL, body) + } + if err != nil { + return nil, err + } + + jar, _ := cookiejar.New(nil) + if len(cookies) > 0 { + httpURL, _ := http.NewRequest(http.MethodGet, rawURL, nil) + jar.SetCookies(httpURL.URL, cookies) + } + + fakeResp := &http.Response{ + StatusCode: http.StatusOK, + Header: http.Header{}, + Body: io.NopCloser(strings.NewReader(html)), + Request: &http.Request{URL: &url.URL{Path: rawURL}}, + } + return &Response{Response: fakeResp, Body: fakeResp.Body}, nil +} + +func (c *Client) Get(ctx context.Context, rawURL string) (*Response, error) { + return c.doRequest(ctx, http.MethodGet, rawURL, "") +} + +func (c *Client) Post(ctx context.Context, rawURL string, bodyType string, body io.Reader) (*Response, error) { + bodyStr, _ := io.ReadAll(body) + return c.doRequest(ctx, http.MethodPost, rawURL, string(bodyStr)) +} + +func (c *Client) GetHTML(ctx context.Context, url string) (io.ReadCloser, error) { + resp, err := c.Get(ctx, url) + if err != nil { + return nil, err + } + if resp.StatusCode != http.StatusOK { + resp.Body.Close() + return nil, fmt.Errorf("HTTP %d for %s", resp.StatusCode, url) + } + return resp.Body, nil +} + +func (c *Client) PostHTML(ctx context.Context, url string, body string) (io.ReadCloser, error) { + resp, err := c.Post(ctx, url, "application/x-www-form-urlencoded", strings.NewReader(body)) + if err != nil { + return nil, err + } + if resp.StatusCode != http.StatusOK { + resp.Body.Close() + return nil, fmt.Errorf("HTTP %d for %s", resp.StatusCode, url) + } + return resp.Body, nil +} + +func (c *Client) GetBytes(ctx context.Context, url string) ([]byte, error) { + resp, err := c.Get(ctx, url) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("HTTP %d for %s", resp.StatusCode, url) + } + return io.ReadAll(resp.Body) +} + +func (c *Client) SetProxy(proxyURL string) error { + return nil +} + +func (c *Client) Do(req *http.Request) (*http.Response, error) { + url := req.URL.String() + var body string + if req.Body != nil { + b, _ := io.ReadAll(req.Body) + body = string(b) + } + resp, err := c.doRequest(req.Context(), req.Method, url, body) + if err != nil { + return nil, err + } + return resp.Response, nil +} \ No newline at end of file diff --git a/sources/base/libgroup/libgroup.go b/sources/base/libgroup/libgroup.go index 6922cc4..a6d6f5c 100755 --- a/sources/base/libgroup/libgroup.go +++ b/sources/base/libgroup/libgroup.go @@ -10,7 +10,7 @@ import ( "net/http" "strings" - "goyomi/internal/httpclient" + "goyomi/internal/httpclient/flare" "goyomi/internal/source" "goyomi/sources/base/util" ) @@ -18,17 +18,17 @@ import ( const defaultAPIDomain = "https://api.lib.social" type Config struct { - Name string - BaseURL string - Lang string - SiteID int - APIDomain string // defaults to "https://api.lib.social" + Name string + BaseURL string + Lang string + SiteID int + APIDomain string // defaults to "https://api.lib.social" BearerToken string // optional; set after WebView acquisition } type Source struct { cfg Config - client *httpclient.Client + client *flare.Client id int64 } @@ -36,7 +36,8 @@ func New(cfg Config) *Source { if cfg.APIDomain == "" { cfg.APIDomain = defaultAPIDomain } - c := httpclient.NewClient(httpclient.WithRateLimit(1, 1)) + opts := []flare.Option{flare.WithRateLimit(1, 1)} + c := flare.NewClient(opts...) return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} } diff --git a/sources/base/liliana/liliana.go b/sources/base/liliana/liliana.go index b80112d..687be13 100755 --- a/sources/base/liliana/liliana.go +++ b/sources/base/liliana/liliana.go @@ -11,7 +11,7 @@ import ( "github.com/PuerkitoBio/goquery" - "goyomi/internal/httpclient" + "goyomi/internal/httpclient/flare" "goyomi/internal/source" "goyomi/sources/base/util" ) @@ -24,12 +24,14 @@ type Config struct { type Source struct { cfg Config - client *httpclient.Client + client *flare.Client id int64 } func New(cfg Config) *Source { - c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) + opts := []flare.Option{flare.WithRateLimit(1, 2)} + c := flare.NewClient(opts...) + return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} } diff --git a/sources/base/madara/madara.go b/sources/base/madara/madara.go index 145eb3e..16b6267 100755 --- a/sources/base/madara/madara.go +++ b/sources/base/madara/madara.go @@ -13,16 +13,16 @@ import ( "github.com/PuerkitoBio/goquery" - "goyomi/internal/httpclient" + "goyomi/internal/httpclient/flare" "goyomi/internal/source" "goyomi/sources/base/util" ) // Config holds per-source configuration and overridable CSS selectors. type Config struct { - Name string - BaseURL string - Lang string + Name string + BaseURL string + Lang string // MangaSubString is the URL path segment for manga listings (default "manga"). MangaSubString string @@ -98,13 +98,14 @@ func (c *Config) setDefaults() { // Source implements source.CatalogueSource for Madara-based sites. type Source struct { cfg Config - client *httpclient.Client + client *flare.Client id int64 } func New(cfg Config) *Source { cfg.setDefaults() - c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) + opts := []flare.Option{flare.WithRateLimit(1, 2)} + c := flare.NewClient(opts...) return &Source{ cfg: cfg, client: c, diff --git a/sources/base/mangareader/mangareader.go b/sources/base/mangareader/mangareader.go index 9a48ab9..c4d2f63 100755 --- a/sources/base/mangareader/mangareader.go +++ b/sources/base/mangareader/mangareader.go @@ -10,22 +10,22 @@ import ( "github.com/PuerkitoBio/goquery" - "goyomi/internal/httpclient" + "goyomi/internal/httpclient/flare" "goyomi/internal/source" "goyomi/sources/base/util" ) type Config struct { - Name string - BaseURL string - Lang string - TypeParam string // "comic", "manga", "manhwa", etc. - ChapterListID string // ID of chapter list container, default "en-chapters" + Name string + BaseURL string + Lang string + TypeParam string // "comic", "manga", "manhwa", etc. + ChapterListID string // ID of chapter list container, default "en-chapters" } type Source struct { cfg Config - client *httpclient.Client + client *flare.Client id int64 } @@ -33,7 +33,8 @@ func New(cfg Config) *Source { if cfg.ChapterListID == "" { cfg.ChapterListID = "en-chapters" } - c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) + opts := []flare.Option{flare.WithRateLimit(1, 2)} + c := flare.NewClient(opts...) return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} } diff --git a/sources/base/mangathemesia/mangathemesia.go b/sources/base/mangathemesia/mangathemesia.go index d9f65d9..4347ad1 100755 --- a/sources/base/mangathemesia/mangathemesia.go +++ b/sources/base/mangathemesia/mangathemesia.go @@ -13,7 +13,7 @@ import ( "github.com/PuerkitoBio/goquery" - "goyomi/internal/httpclient" + "goyomi/internal/httpclient/flare" "goyomi/internal/source" "goyomi/sources/base/util" ) @@ -73,13 +73,14 @@ func (c *Config) setDefaults() { // Source implements source.CatalogueSource for MangaThemesia sites. type Source struct { cfg Config - client *httpclient.Client + client *flare.Client id int64 } func New(cfg Config) *Source { cfg.setDefaults() - c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) + opts := []flare.Option{flare.WithRateLimit(1, 2)} + c := flare.NewClient(opts...) return &Source{ cfg: cfg, client: c, diff --git a/sources/base/mangaworld/mangaworld.go b/sources/base/mangaworld/mangaworld.go index b0e8e42..976b5a7 100755 --- a/sources/base/mangaworld/mangaworld.go +++ b/sources/base/mangaworld/mangaworld.go @@ -10,7 +10,7 @@ import ( "github.com/PuerkitoBio/goquery" - "goyomi/internal/httpclient" + "goyomi/internal/httpclient/flare" "goyomi/internal/source" "goyomi/sources/base/util" ) @@ -23,12 +23,13 @@ type Config struct { type Source struct { cfg Config - client *httpclient.Client + client *flare.Client id int64 } func New(cfg Config) *Source { - c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) + opts := []flare.Option{flare.WithRateLimit(1, 2)} + c := flare.NewClient(opts...) return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} } diff --git a/sources/base/zmanga/zmanga.go b/sources/base/zmanga/zmanga.go index f9cb516..9aa7de8 100755 --- a/sources/base/zmanga/zmanga.go +++ b/sources/base/zmanga/zmanga.go @@ -10,7 +10,7 @@ import ( "github.com/PuerkitoBio/goquery" - "goyomi/internal/httpclient" + "goyomi/internal/httpclient/flare" "goyomi/internal/source" "goyomi/sources/base/util" ) @@ -23,12 +23,13 @@ type Config struct { type Source struct { cfg Config - client *httpclient.Client + client *flare.Client id int64 } func New(cfg Config) *Source { - c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) + opts := []flare.Option{flare.WithRateLimit(1, 2)} + c := flare.NewClient(opts...) return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} }