goyomi/internal/httpclient/client.go

package httpclient

import (
	"context"
	"fmt"
	"io"
	"log"
	"net/http"
	"net/http/cookiejar"
	"strconv"
	"strings"
	"sync"
	"time"

	"goyomi/internal/config"
	"golang.org/x/time/rate"
)

const defaultUserAgent = "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Mobile Safari/537.36"

var (
	globalFSClient *FlareSolverrClient
	globalProxy    bool
	verboseLog     bool
)

func SetGlobalFlareSolverr(client *FlareSolverrClient) {
	globalFSClient = client
}

func SetGlobalProxy(enabled bool) {
	globalProxy = enabled
}

func SetVerboseLog(enabled bool) {
	verboseLog = enabled
}

func ProxyEnabled() bool {
	if globalProxy {
		return true
	}
	if config.IsFlareSolverrProxyEnabled() {
		return true
	}
	return false
}

type Client struct {
	http       *http.Client
	fsClient   *FlareSolverrClient
	proxy      bool
	rateLimit  float64
	burst      int
	referer    string
	verboseLog bool

	mu       sync.Mutex
	limiters map[string]*rate.Limiter
}

type Option func(*Client)

func WithRateLimit(rps float64, burst int) Option {
	return func(c *Client) {
		c.rateLimit = rps
		c.burst = burst
	}
}

func WithTimeout(d time.Duration) Option {
	return func(c *Client) { c.http.Timeout = d }
}

func WithReferer(referer string) Option {
	return func(c *Client) { c.referer = referer }
}

func WithFlareSolverr(fs *FlareSolverrClient) Option {
	return func(c *Client) { c.fsClient = fs }
}

func WithProxy(enabled bool) Option {
	return func(c *Client) { c.proxy = enabled }
}

func WithVerboseLog(enabled bool) Option {
	return func(c *Client) { c.verboseLog = enabled }
}

func NewClient(opts ...Option) *Client {
	jar, _ := cookiejar.New(nil)
	c := &Client{
		http:       &http.Client{Timeout: 30 * time.Second, Jar: jar},
		fsClient:   globalFSClient,
		proxy:      globalProxy,
		rateLimit:  1,
		burst:      1,
		limiters:   map[string]*rate.Limiter{},
		verboseLog: verboseLog,
	}
	for _, o := range opts {
		o(c)
	}
	return c
}

func (c *Client) limiter(host string) *rate.Limiter {
	c.mu.Lock()
	defer c.mu.Unlock()
	l, ok := c.limiters[host]
	if !ok {
		l = rate.NewLimiter(rate.Limit(c.rateLimit), c.burst)
		c.limiters[host] = l
	}
	return l
}

func (c *Client) Do(req *http.Request) (*http.Response, error) {
	if (c.proxy || ProxyEnabled()) && c.fsClient != nil {
		return c.doViaFlareSolverr(req)
	}
	return c.doDirect(req)
}

func (c *Client) doDirect(req *http.Request) (*http.Response, error) {
	if err := c.limiter(req.URL.Host).Wait(req.Context()); err != nil {
		return nil, err
	}
	if c.referer != "" && req.Header.Get("Referer") == "" {
		req.Header.Set("Referer", c.referer)
	}
	if req.Header.Get("User-Agent") == "" {
		req.Header.Set("User-Agent", defaultUserAgent)
	}

	if c.verboseLog {
		log.Printf("[httpclient] DIRECT GET %s", req.URL.String())
	}

	const maxRetries = 3
	for attempt := 0; attempt <= maxRetries; attempt++ {
		resp, err := c.http.Do(req)
		if err != nil {
			return nil, err
		}
		if c.verboseLog {
			log.Printf("[httpclient] DIRECT RESPONSE %s status=%d", req.URL.String(), resp.StatusCode)
		}
		if resp.StatusCode != http.StatusTooManyRequests {
			return resp, nil
		}
		resp.Body.Close()
		if attempt == maxRetries {
			return resp, nil
		}
		sleep := retryAfter(resp)
		select {
		case <-req.Context().Done():
			return nil, req.Context().Err()
		case <-time.After(sleep):
		}
	}
	panic("unreachable")
}

func (c *Client) doViaFlareSolverr(req *http.Request) (*http.Response, error) {
	if err := c.limiter(req.URL.Host).Wait(req.Context()); err != nil {
		return nil, err
	}

	url := req.URL.String()

	if c.verboseLog {
		log.Printf("[httpclient] FLARESOLVERR %s %s", req.Method, url)
	}

	var html string
	var cookies []*http.Cookie
	var err error

	if req.Method == http.MethodGet {
		html, cookies, err = c.fsClient.Get(req.Context(), url)
	} else if req.Method == http.MethodPost {
		var bodyStr string
		if req.Body != nil {
			bodyBytes, _ := io.ReadAll(req.Body)
			bodyStr = string(bodyBytes)
			if c.verboseLog {
				log.Printf("[httpclient] FLARESOLVERR POST BODY: %s", bodyStr)
			}
		}
		html, cookies, err = c.fsClient.Post(req.Context(), url, bodyStr)
	} else {
		return c.doDirect(req)
	}

	if err != nil {
		return nil, fmt.Errorf("flaresolverr: %w", err)
	}

	if c.verboseLog {
		log.Printf("[httpclient] FLARESOLVERR RESPONSE length=%d cookies=%d", len(html), len(cookies))
		if len(html) > 0 {
			truncate := html
			if len(html) > 500 {
				truncate = html[:500]
			}
			log.Printf("[httpclient] FLARESOLVERR RESPONSE HTML (first 500 chars): %s", truncate)
		}
	}

	resp := &http.Response{
		StatusCode: http.StatusOK,
		Header:     http.Header{},
		Body:       io.NopCloser(strings.NewReader(html)),
	}
	resp.Header.Set("Content-Type", "text/html; charset=utf-8")

	if c.http.Jar != nil {
		for _, cookie := range cookies {
			c.http.Jar.SetCookies(req.URL, []*http.Cookie{cookie})
		}
	}

	return resp, nil
}

func retryAfter(resp *http.Response) time.Duration {
	ra := resp.Header.Get("Retry-After")
	if ra == "" {
		return 5 * time.Second
	}
	if secs, err := strconv.ParseFloat(ra, 64); err == nil {
		return time.Duration(secs * float64(time.Second))
	}
	if t, err := http.ParseTime(ra); err == nil {
		d := time.Until(t)
		if d > 0 {
			return d
		}
	}
	return 5 * time.Second
}

func (c *Client) Get(ctx context.Context, url string, headers map[string]string) (*http.Response, error) {
	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
	if err != nil {
		return nil, err
	}
	for k, v := range headers {
		req.Header.Set(k, v)
	}
	return c.Do(req)
}

func (c *Client) Post(ctx context.Context, url string, body io.Reader, contentType string, headers map[string]string) (*http.Response, error) {
	req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, body)
	if err != nil {
		return nil, err
	}
	req.Header.Set("Content-Type", contentType)
	for k, v := range headers {
		req.Header.Set(k, v)
	}
	return c.Do(req)
}

// HTTPClient returns the underlying *http.Client (for passing to graphql helper etc.)
func (c *Client) HTTPClient() *http.Client { return c.http }