feat: add FlareSolverr proxy support with DB-backed config
- Add config table for storing FlareSolverr proxy setting
- Add HTTP endpoints to get/set proxy status (GET/POST /api/config/flaresolverr)
- Refactor httpclient to support proxy mode (requests go through FlareSolverr)
- Add verbose logging for debugging
- Add POST support to FlareSolverr client
Usage:
GET /api/config/flaresolverr - returns {flaresolverr_proxy: bool}
POST /api/config/flaresolverr - body: {enabled: true/false}
This commit is contained in:
+134
-11
@@ -2,23 +2,58 @@ package httpclient
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/http/cookiejar"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"goyomi/internal/config"
|
||||
"golang.org/x/time/rate"
|
||||
)
|
||||
|
||||
const defaultUserAgent = "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Mobile Safari/537.36"
|
||||
|
||||
var (
|
||||
globalFSClient *FlareSolverrClient
|
||||
globalProxy bool
|
||||
verboseLog bool
|
||||
)
|
||||
|
||||
func SetGlobalFlareSolverr(client *FlareSolverrClient) {
|
||||
globalFSClient = client
|
||||
}
|
||||
|
||||
func SetGlobalProxy(enabled bool) {
|
||||
globalProxy = enabled
|
||||
}
|
||||
|
||||
func SetVerboseLog(enabled bool) {
|
||||
verboseLog = enabled
|
||||
}
|
||||
|
||||
func ProxyEnabled() bool {
|
||||
if globalProxy {
|
||||
return true
|
||||
}
|
||||
if config.IsFlareSolverrProxyEnabled() {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
type Client struct {
|
||||
http *http.Client
|
||||
rateLimit float64
|
||||
burst int
|
||||
referer string
|
||||
http *http.Client
|
||||
fsClient *FlareSolverrClient
|
||||
proxy bool
|
||||
rateLimit float64
|
||||
burst int
|
||||
referer string
|
||||
verboseLog bool
|
||||
|
||||
mu sync.Mutex
|
||||
limiters map[string]*rate.Limiter
|
||||
@@ -41,16 +76,28 @@ func WithReferer(referer string) Option {
|
||||
return func(c *Client) { c.referer = referer }
|
||||
}
|
||||
|
||||
func WithFlareSolverr(fs *FlareSolverrClient) Option {
|
||||
return func(c *Client) { c.fsClient = fs }
|
||||
}
|
||||
|
||||
func WithProxy(enabled bool) Option {
|
||||
return func(c *Client) { c.proxy = enabled }
|
||||
}
|
||||
|
||||
func WithVerboseLog(enabled bool) Option {
|
||||
return func(c *Client) { c.verboseLog = enabled }
|
||||
}
|
||||
|
||||
func NewClient(opts ...Option) *Client {
|
||||
jar, _ := cookiejar.New(nil)
|
||||
c := &Client{
|
||||
http: &http.Client{
|
||||
Timeout: 30 * time.Second,
|
||||
Jar: jar,
|
||||
},
|
||||
rateLimit: 1,
|
||||
burst: 1,
|
||||
limiters: map[string]*rate.Limiter{},
|
||||
http: &http.Client{Timeout: 30 * time.Second, Jar: jar},
|
||||
fsClient: globalFSClient,
|
||||
proxy: globalProxy,
|
||||
rateLimit: 1,
|
||||
burst: 1,
|
||||
limiters: map[string]*rate.Limiter{},
|
||||
verboseLog: verboseLog,
|
||||
}
|
||||
for _, o := range opts {
|
||||
o(c)
|
||||
@@ -70,6 +117,13 @@ func (c *Client) limiter(host string) *rate.Limiter {
|
||||
}
|
||||
|
||||
func (c *Client) Do(req *http.Request) (*http.Response, error) {
|
||||
if (c.proxy || ProxyEnabled()) && c.fsClient != nil {
|
||||
return c.doViaFlareSolverr(req)
|
||||
}
|
||||
return c.doDirect(req)
|
||||
}
|
||||
|
||||
func (c *Client) doDirect(req *http.Request) (*http.Response, error) {
|
||||
if err := c.limiter(req.URL.Host).Wait(req.Context()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -80,12 +134,19 @@ func (c *Client) Do(req *http.Request) (*http.Response, error) {
|
||||
req.Header.Set("User-Agent", defaultUserAgent)
|
||||
}
|
||||
|
||||
if c.verboseLog {
|
||||
log.Printf("[httpclient] DIRECT GET %s", req.URL.String())
|
||||
}
|
||||
|
||||
const maxRetries = 3
|
||||
for attempt := 0; attempt <= maxRetries; attempt++ {
|
||||
resp, err := c.http.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if c.verboseLog {
|
||||
log.Printf("[httpclient] DIRECT RESPONSE %s status=%d", req.URL.String(), resp.StatusCode)
|
||||
}
|
||||
if resp.StatusCode != http.StatusTooManyRequests {
|
||||
return resp, nil
|
||||
}
|
||||
@@ -103,6 +164,68 @@ func (c *Client) Do(req *http.Request) (*http.Response, error) {
|
||||
panic("unreachable")
|
||||
}
|
||||
|
||||
func (c *Client) doViaFlareSolverr(req *http.Request) (*http.Response, error) {
|
||||
if err := c.limiter(req.URL.Host).Wait(req.Context()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
url := req.URL.String()
|
||||
|
||||
if c.verboseLog {
|
||||
log.Printf("[httpclient] FLARESOLVERR %s %s", req.Method, url)
|
||||
}
|
||||
|
||||
var html string
|
||||
var cookies []*http.Cookie
|
||||
var err error
|
||||
|
||||
if req.Method == http.MethodGet {
|
||||
html, cookies, err = c.fsClient.Get(req.Context(), url)
|
||||
} else if req.Method == http.MethodPost {
|
||||
var bodyStr string
|
||||
if req.Body != nil {
|
||||
bodyBytes, _ := io.ReadAll(req.Body)
|
||||
bodyStr = string(bodyBytes)
|
||||
if c.verboseLog {
|
||||
log.Printf("[httpclient] FLARESOLVERR POST BODY: %s", bodyStr)
|
||||
}
|
||||
}
|
||||
html, cookies, err = c.fsClient.Post(req.Context(), url, bodyStr)
|
||||
} else {
|
||||
return c.doDirect(req)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("flaresolverr: %w", err)
|
||||
}
|
||||
|
||||
if c.verboseLog {
|
||||
log.Printf("[httpclient] FLARESOLVERR RESPONSE length=%d cookies=%d", len(html), len(cookies))
|
||||
if len(html) > 0 {
|
||||
truncate := html
|
||||
if len(html) > 500 {
|
||||
truncate = html[:500]
|
||||
}
|
||||
log.Printf("[httpclient] FLARESOLVERR RESPONSE HTML (first 500 chars): %s", truncate)
|
||||
}
|
||||
}
|
||||
|
||||
resp := &http.Response{
|
||||
StatusCode: http.StatusOK,
|
||||
Header: http.Header{},
|
||||
Body: io.NopCloser(strings.NewReader(html)),
|
||||
}
|
||||
resp.Header.Set("Content-Type", "text/html; charset=utf-8")
|
||||
|
||||
if c.http.Jar != nil {
|
||||
for _, cookie := range cookies {
|
||||
c.http.Jar.SetCookies(req.URL, []*http.Cookie{cookie})
|
||||
}
|
||||
}
|
||||
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func retryAfter(resp *http.Response) time.Duration {
|
||||
ra := resp.Header.Get("Retry-After")
|
||||
if ra == "" {
|
||||
|
||||
Reference in New Issue
Block a user