package httpclient import ( "context" "fmt" "io" "log" "net/http" "net/http/cookiejar" "strconv" "strings" "sync" "time" "goyomi/internal/config" "golang.org/x/time/rate" ) const defaultUserAgent = "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Mobile Safari/537.36" var ( globalFSClient *FlareSolverrClient globalProxy bool verboseLog bool ) func SetGlobalFlareSolverr(client *FlareSolverrClient) { globalFSClient = client } func SetGlobalProxy(enabled bool) { globalProxy = enabled } func SetVerboseLog(enabled bool) { verboseLog = enabled } func ProxyEnabled() bool { if globalProxy { return true } if config.IsFlareSolverrProxyEnabled() { return true } return false } type Client struct { http *http.Client fsClient *FlareSolverrClient proxy bool rateLimit float64 burst int referer string verboseLog bool mu sync.Mutex limiters map[string]*rate.Limiter } type Option func(*Client) func WithRateLimit(rps float64, burst int) Option { return func(c *Client) { c.rateLimit = rps c.burst = burst } } func WithTimeout(d time.Duration) Option { return func(c *Client) { c.http.Timeout = d } } func WithReferer(referer string) Option { return func(c *Client) { c.referer = referer } } func WithFlareSolverr(fs *FlareSolverrClient) Option { return func(c *Client) { c.fsClient = fs } } func WithProxy(enabled bool) Option { return func(c *Client) { c.proxy = enabled } } func WithVerboseLog(enabled bool) Option { return func(c *Client) { c.verboseLog = enabled } } func NewClient(opts ...Option) *Client { jar, _ := cookiejar.New(nil) c := &Client{ http: &http.Client{Timeout: 30 * time.Second, Jar: jar}, fsClient: globalFSClient, proxy: globalProxy, rateLimit: 1, burst: 1, limiters: map[string]*rate.Limiter{}, verboseLog: verboseLog, } for _, o := range opts { o(c) } return c } func (c *Client) limiter(host string) *rate.Limiter { c.mu.Lock() defer c.mu.Unlock() l, ok := c.limiters[host] if !ok { l = rate.NewLimiter(rate.Limit(c.rateLimit), c.burst) c.limiters[host] = l } return l } func (c *Client) Do(req *http.Request) (*http.Response, error) { if (c.proxy || ProxyEnabled()) && c.fsClient != nil { return c.doViaFlareSolverr(req) } return c.doDirect(req) } func (c *Client) doDirect(req *http.Request) (*http.Response, error) { if err := c.limiter(req.URL.Host).Wait(req.Context()); err != nil { return nil, err } if c.referer != "" && req.Header.Get("Referer") == "" { req.Header.Set("Referer", c.referer) } if req.Header.Get("User-Agent") == "" { req.Header.Set("User-Agent", defaultUserAgent) } if c.verboseLog { log.Printf("[httpclient] DIRECT GET %s", req.URL.String()) } const maxRetries = 3 for attempt := 0; attempt <= maxRetries; attempt++ { resp, err := c.http.Do(req) if err != nil { return nil, err } if c.verboseLog { log.Printf("[httpclient] DIRECT RESPONSE %s status=%d", req.URL.String(), resp.StatusCode) } if resp.StatusCode != http.StatusTooManyRequests { return resp, nil } resp.Body.Close() if attempt == maxRetries { return resp, nil } sleep := retryAfter(resp) select { case <-req.Context().Done(): return nil, req.Context().Err() case <-time.After(sleep): } } panic("unreachable") } func (c *Client) doViaFlareSolverr(req *http.Request) (*http.Response, error) { if err := c.limiter(req.URL.Host).Wait(req.Context()); err != nil { return nil, err } url := req.URL.String() if c.verboseLog { log.Printf("[httpclient] FLARESOLVERR %s %s", req.Method, url) } var html string var cookies []*http.Cookie var err error if req.Method == http.MethodGet { html, cookies, err = c.fsClient.Get(req.Context(), url) } else if req.Method == http.MethodPost { var bodyStr string if req.Body != nil { bodyBytes, _ := io.ReadAll(req.Body) bodyStr = string(bodyBytes) if c.verboseLog { log.Printf("[httpclient] FLARESOLVERR POST BODY: %s", bodyStr) } } html, cookies, err = c.fsClient.Post(req.Context(), url, bodyStr) } else { return c.doDirect(req) } if err != nil { return nil, fmt.Errorf("flaresolverr: %w", err) } if c.verboseLog { log.Printf("[httpclient] FLARESOLVERR RESPONSE length=%d cookies=%d", len(html), len(cookies)) if len(html) > 0 { truncate := html if len(html) > 500 { truncate = html[:500] } log.Printf("[httpclient] FLARESOLVERR RESPONSE HTML (first 500 chars): %s", truncate) } } resp := &http.Response{ StatusCode: http.StatusOK, Header: http.Header{}, Body: io.NopCloser(strings.NewReader(html)), } resp.Header.Set("Content-Type", "text/html; charset=utf-8") if c.http.Jar != nil { for _, cookie := range cookies { c.http.Jar.SetCookies(req.URL, []*http.Cookie{cookie}) } } return resp, nil } func retryAfter(resp *http.Response) time.Duration { ra := resp.Header.Get("Retry-After") if ra == "" { return 5 * time.Second } if secs, err := strconv.ParseFloat(ra, 64); err == nil { return time.Duration(secs * float64(time.Second)) } if t, err := http.ParseTime(ra); err == nil { d := time.Until(t) if d > 0 { return d } } return 5 * time.Second } func (c *Client) Get(ctx context.Context, url string, headers map[string]string) (*http.Response, error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { return nil, err } for k, v := range headers { req.Header.Set(k, v) } return c.Do(req) } func (c *Client) Post(ctx context.Context, url string, body io.Reader, contentType string, headers map[string]string) (*http.Response, error) { req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, body) if err != nil { return nil, err } req.Header.Set("Content-Type", contentType) for k, v := range headers { req.Header.Set(k, v) } return c.Do(req) } // HTTPClient returns the underlying *http.Client (for passing to graphql helper etc.) func (c *Client) HTTPClient() *http.Client { return c.http }