admin/goyomi
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: separate httpclient packages for regular and FlareSolverr sources

Browse Source 
- Add internal/httpclient/flare package for Cloudflare-protected sites
- Update 7 bases (madara, zmanga, mangaworld, mangathemesia, mangareader,
  libgroup, liliana) to use flare client
- Remove unused internal/config/source.go
This commit is contained in:
Achmad
2026-05-11 10:48:05 +00:00
parent 308d66bd36
commit b199bad30d
8 changed files with 226 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/httpclient/flare/flare.go
+184
View File
@@ -0,0 +1,184 @@
package flare
import (
"context"
"fmt"
"io"
"net/http"
"net/http/cookiejar"
"net/url"
"strings"
"sync"
"time"
"goyomi/internal/httpclient"
"golang.org/x/time/rate"
)
const defaultUserAgent = "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Mobile Safari/537.36"
type Client struct {
fsClient *httpclient.FlareSolverrClient
rateLimit float64
burst int
referer string
mu sync.Mutex
limiters map[string]*rate.Limiter
}
type Response struct {
*http.Response
Body io.ReadCloser
}
func (r *Response) Close() error {
return r.Body.Close()
}
type Option func(*Client)
func WithRateLimit(rps float64, burst int) Option {
return func(c *Client) {
c.rateLimit = rps
c.burst = burst
}
}
func WithReferer(referer string) Option {
return func(c *Client) { c.referer = referer }
}
func WithTimeout(d time.Duration) Option {
return func(c *Client) {}
}
func NewClient(opts ...Option) *Client {
c := &Client{
limiters: make(map[string]*rate.Limiter),
}
for _, opt := range opts {
opt(c)
}
if c.rateLimit == 0 {
c.rateLimit = 1
}
if c.burst == 0 {
c.burst = 1
}
fsClient, err := httpclient.NewFlareSolverrClient()
if err == nil {
c.fsClient = fsClient
}
return c
}
func (c *Client) SetFlareSolverrClient(fs *httpclient.FlareSolverrClient) {
c.fsClient = fs
}
func (c *Client) doRequest(ctx context.Context, method string, rawURL string, body string) (*Response, error) {
if c.fsClient == nil {
return nil, fmt.Errorf("FlareSolverr client not configured")
}
c.mu.Lock()
limiter, ok := c.limiters[rawURL]
if !ok {
limiter = rate.NewLimiter(rate.Limit(c.rateLimit), c.burst)
c.limiters[rawURL] = limiter
}
c.mu.Unlock()
if err := limiter.Wait(ctx); err != nil {
return nil, err
}
var html string
var cookies []*http.Cookie
var err error
if method == http.MethodGet {
html, cookies, err = c.fsClient.Get(ctx, rawURL)
} else {
html, cookies, err = c.fsClient.Post(ctx, rawURL, body)
}
if err != nil {
return nil, err
}
jar, _ := cookiejar.New(nil)
if len(cookies) > 0 {
httpURL, _ := http.NewRequest(http.MethodGet, rawURL, nil)
jar.SetCookies(httpURL.URL, cookies)
}
fakeResp := &http.Response{
StatusCode: http.StatusOK,
Header: http.Header{},
Body: io.NopCloser(strings.NewReader(html)),
Request: &http.Request{URL: &url.URL{Path: rawURL}},
}
return &Response{Response: fakeResp, Body: fakeResp.Body}, nil
}
func (c *Client) Get(ctx context.Context, rawURL string) (*Response, error) {
return c.doRequest(ctx, http.MethodGet, rawURL, "")
}
func (c *Client) Post(ctx context.Context, rawURL string, bodyType string, body io.Reader) (*Response, error) {
bodyStr, _ := io.ReadAll(body)
return c.doRequest(ctx, http.MethodPost, rawURL, string(bodyStr))
}
func (c *Client) GetHTML(ctx context.Context, url string) (io.ReadCloser, error) {
resp, err := c.Get(ctx, url)
if err != nil {
return nil, err
}
if resp.StatusCode != http.StatusOK {
resp.Body.Close()
return nil, fmt.Errorf("HTTP %d for %s", resp.StatusCode, url)
}
return resp.Body, nil
}
func (c *Client) PostHTML(ctx context.Context, url string, body string) (io.ReadCloser, error) {
resp, err := c.Post(ctx, url, "application/x-www-form-urlencoded", strings.NewReader(body))
if err != nil {
return nil, err
}
if resp.StatusCode != http.StatusOK {
resp.Body.Close()
return nil, fmt.Errorf("HTTP %d for %s", resp.StatusCode, url)
}
return resp.Body, nil
}
func (c *Client) GetBytes(ctx context.Context, url string) ([]byte, error) {
resp, err := c.Get(ctx, url)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("HTTP %d for %s", resp.StatusCode, url)
}
return io.ReadAll(resp.Body)
}
func (c *Client) SetProxy(proxyURL string) error {
return nil
}
func (c *Client) Do(req *http.Request) (*http.Response, error) {
url := req.URL.String()
var body string
if req.Body != nil {
b, _ := io.ReadAll(req.Body)
body = string(b)
}
resp, err := c.doRequest(req.Context(), req.Method, url, body)
if err != nil {
return nil, err
}
return resp.Response, nil
}
sources/base/libgroup/libgroup.go
+9 -8
View File
@@ -10,7 +10,7 @@ import (
"net/http"
"strings"
"goyomi/internal/httpclient"
"goyomi/internal/httpclient/flare"
"goyomi/internal/source"
"goyomi/sources/base/util"
)
@@ -18,17 +18,17 @@ import (
const defaultAPIDomain = "https://api.lib.social"
type Config struct {
Name string
BaseURL string
Lang string
SiteID int
APIDomain string // defaults to "https://api.lib.social"
Name string
BaseURL string
Lang string
SiteID int
APIDomain string // defaults to "https://api.lib.social"
BearerToken string // optional; set after WebView acquisition
}
type Source struct {
cfg Config
client *httpclient.Client
client *flare.Client
id int64
}
@@ -36,7 +36,8 @@ func New(cfg Config) *Source {
if cfg.APIDomain == "" {
cfg.APIDomain = defaultAPIDomain
}
c := httpclient.NewClient(httpclient.WithRateLimit(1, 1))
opts := []flare.Option{flare.WithRateLimit(1, 1)}
c := flare.NewClient(opts...)
return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)}
}
sources/base/liliana/liliana.go
+5 -3
View File
@@ -11,7 +11,7 @@ import (
"github.com/PuerkitoBio/goquery"
"goyomi/internal/httpclient"
"goyomi/internal/httpclient/flare"
"goyomi/internal/source"
"goyomi/sources/base/util"
)
@@ -24,12 +24,14 @@ type Config struct {
type Source struct {
cfg Config
client *httpclient.Client
client *flare.Client
id int64
}
func New(cfg Config) *Source {
c := httpclient.NewClient(httpclient.WithRateLimit(1, 2))
opts := []flare.Option{flare.WithRateLimit(1, 2)}
c := flare.NewClient(opts...)
return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)}
return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)}
}
sources/base/madara/madara.go
+7 -6
View File
@@ -13,16 +13,16 @@ import (
"github.com/PuerkitoBio/goquery"
"goyomi/internal/httpclient"
"goyomi/internal/httpclient/flare"
"goyomi/internal/source"
"goyomi/sources/base/util"
)
// Config holds per-source configuration and overridable CSS selectors.
type Config struct {
Name string
BaseURL string
Lang string
Name string
BaseURL string
Lang string
// MangaSubString is the URL path segment for manga listings (default "manga").
MangaSubString string
@@ -98,13 +98,14 @@ func (c *Config) setDefaults() {
// Source implements source.CatalogueSource for Madara-based sites.
type Source struct {
cfg Config
client *httpclient.Client
client *flare.Client
id int64
}
func New(cfg Config) *Source {
cfg.setDefaults()
c := httpclient.NewClient(httpclient.WithRateLimit(1, 2))
opts := []flare.Option{flare.WithRateLimit(1, 2)}
c := flare.NewClient(opts...)
return &Source{
cfg: cfg,
client: c,
sources/base/mangareader/mangareader.go
+9 -8
View File
@@ -10,22 +10,22 @@ import (
"github.com/PuerkitoBio/goquery"
"goyomi/internal/httpclient"
"goyomi/internal/httpclient/flare"
"goyomi/internal/source"
"goyomi/sources/base/util"
)
type Config struct {
Name string
BaseURL string
Lang string
TypeParam string // "comic", "manga", "manhwa", etc.
ChapterListID string // ID of chapter list container, default "en-chapters"
Name string
BaseURL string
Lang string
TypeParam string // "comic", "manga", "manhwa", etc.
ChapterListID string // ID of chapter list container, default "en-chapters"
}
type Source struct {
cfg Config
client *httpclient.Client
client *flare.Client
id int64
}
@@ -33,7 +33,8 @@ func New(cfg Config) *Source {
if cfg.ChapterListID == "" {
cfg.ChapterListID = "en-chapters"
}
c := httpclient.NewClient(httpclient.WithRateLimit(1, 2))
opts := []flare.Option{flare.WithRateLimit(1, 2)}
c := flare.NewClient(opts...)
return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)}
}
sources/base/mangathemesia/mangathemesia.go
+4 -3
View File
@@ -13,7 +13,7 @@ import (
"github.com/PuerkitoBio/goquery"
"goyomi/internal/httpclient"
"goyomi/internal/httpclient/flare"
"goyomi/internal/source"
"goyomi/sources/base/util"
)
@@ -73,13 +73,14 @@ func (c *Config) setDefaults() {
// Source implements source.CatalogueSource for MangaThemesia sites.
type Source struct {
cfg Config
client *httpclient.Client
client *flare.Client
id int64
}
func New(cfg Config) *Source {
cfg.setDefaults()
c := httpclient.NewClient(httpclient.WithRateLimit(1, 2))
opts := []flare.Option{flare.WithRateLimit(1, 2)}
c := flare.NewClient(opts...)
return &Source{
cfg: cfg,
client: c,
sources/base/mangaworld/mangaworld.go
+4 -3
View File
@@ -10,7 +10,7 @@ import (
"github.com/PuerkitoBio/goquery"
"goyomi/internal/httpclient"
"goyomi/internal/httpclient/flare"
"goyomi/internal/source"
"goyomi/sources/base/util"
)
@@ -23,12 +23,13 @@ type Config struct {
type Source struct {
cfg Config
client *httpclient.Client
client *flare.Client
id int64
}
func New(cfg Config) *Source {
c := httpclient.NewClient(httpclient.WithRateLimit(1, 2))
opts := []flare.Option{flare.WithRateLimit(1, 2)}
c := flare.NewClient(opts...)
return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)}
}
sources/base/zmanga/zmanga.go
+4 -3
View File
@@ -10,7 +10,7 @@ import (
"github.com/PuerkitoBio/goquery"
"goyomi/internal/httpclient"
"goyomi/internal/httpclient/flare"
"goyomi/internal/source"
"goyomi/sources/base/util"
)
@@ -23,12 +23,13 @@ type Config struct {
type Source struct {
cfg Config
client *httpclient.Client
client *flare.Client
id int64
}
func New(cfg Config) *Source {
c := httpclient.NewClient(httpclient.WithRateLimit(1, 2))
opts := []flare.Option{flare.WithRateLimit(1, 2)}
c := flare.NewClient(opts...)
return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)}
}