diff --git a/.env.example b/.env.example index 8c586d1..10fb2a9 100755 --- a/.env.example +++ b/.env.example @@ -8,6 +8,7 @@ DATABASE_URL=postgres://goyomi:goyomi@postgres:5432/goyomi?sslmode=disable FLARESOLVERR_URL=http://flaresolverr:8191 FLARESOLVERR_LOG_LEVEL=info FLARESOLVERR_PROXY=0 +FLARESOLVERR_SESSION=goyomi ADDR=:3300 # Connection pool diff --git a/compose.yml b/compose.yml index a695210..cef7fee 100755 --- a/compose.yml +++ b/compose.yml @@ -13,6 +13,8 @@ services: - go_build_cache:/root/.cache/go-build ports: - "${ADDR:-8080}:8080" + environment: + FLARESOLVERR_SESSION: ${FLARESOLVERR_SESSION:-goyomi} depends_on: postgres: condition: service_healthy @@ -43,6 +45,8 @@ services: restart: unless-stopped environment: LOG_LEVEL: ${FLARESOLVERR_LOG_LEVEL} + ports: + - "8191:8191" networks: - goyomi_dev diff --git a/internal/httpclient/client.go b/internal/httpclient/client.go index 12ff4d0..2b119f0 100755 --- a/internal/httpclient/client.go +++ b/internal/httpclient/client.go @@ -1,11 +1,14 @@ package httpclient import ( + "bytes" "context" + "fmt" "io" "log" "net/http" "net/http/cookiejar" + "net/url" "strconv" "sync" "time" @@ -15,17 +18,58 @@ import ( const defaultUserAgent = "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Mobile Safari/537.36" -var verboseLog bool +var ( + verboseLog bool + defaultOnce sync.Once + defaultClient *Client +) -func SetVerboseLog(enabled bool) { - verboseLog = enabled +func SetVerboseLog(enabled bool) { verboseLog = enabled } + +// DefaultClient returns the shared singleton HTTP client. +// FlareSolverr is auto-configured if the FLARESOLVERR_URL env var is set. +// All sources share the same rate limiter (+ cookie jar) through this client. +func DefaultClient() *Client { + defaultOnce.Do(func() { + defaultClient = newClient() + }) + return defaultClient +} + +// NewClient creates a standalone client with optional per-source overrides. +// Only create a new client when the source needs different behaviour +// (e.g. a custom rate limit); otherwise use DefaultClient. +func NewClient(opts ...Option) *Client { + c := newClient() + for _, o := range opts { + o(c) + } + return c +} + +func newClient() *Client { + jar, _ := cookiejar.New(nil) + c := &Client{ + http: &http.Client{Timeout: 30 * time.Second, Jar: jar}, + rateLimit: 1, + burst: 1, + userAgent: defaultUserAgent, + limiters: map[string]*rate.Limiter{}, + verboseLog: verboseLog, + } + fsClient, err := NewFlareSolverrClient() + if err == nil { + c.fsClient = fsClient + } + return c } type Client struct { http *http.Client + fsClient *FlareSolverrClient rateLimit float64 burst int - referer string + userAgent string verboseLog bool mu sync.Mutex @@ -45,29 +89,14 @@ func WithTimeout(d time.Duration) Option { return func(c *Client) { c.http.Timeout = d } } -func WithReferer(referer string) Option { - return func(c *Client) { c.referer = referer } +func WithUserAgent(ua string) Option { + return func(c *Client) { c.userAgent = ua } } func WithVerboseLog(enabled bool) Option { return func(c *Client) { c.verboseLog = enabled } } -func NewClient(opts ...Option) *Client { - jar, _ := cookiejar.New(nil) - c := &Client{ - http: &http.Client{Timeout: 30 * time.Second, Jar: jar}, - rateLimit: 1, - burst: 1, - limiters: map[string]*rate.Limiter{}, - verboseLog: verboseLog, - } - for _, o := range opts { - o(c) - } - return c -} - func (c *Client) limiter(host string) *rate.Limiter { c.mu.Lock() defer c.mu.Unlock() @@ -79,23 +108,55 @@ func (c *Client) limiter(host string) *rate.Limiter { return l } +// Do tries a direct HTTP request first. If the server returns 403/503 (a +// Cloudflare or DDoS challenge) and FlareSolverr is available, it falls back +// to FlareSolverr raw mode to solve the challenge and return the actual body. +// +// When FlareSolverr is used, the Chrome HTML wrapper is stripped from the +// response so that both JSON and HTML callers receive the real server output. func (c *Client) Do(req *http.Request) (*http.Response, error) { + if err := c.limiter(req.URL.Host).Wait(req.Context()); err != nil { + return nil, err + } + if req.Header.Get("User-Agent") == "" { + req.Header.Set("User-Agent", c.userAgent) + } + + // Always route through FlareSolverr when configured. Go's TLS fingerprint + // doesn't match Chrome's, so Cloudflare clearance cookies from FS are + // rejected by Go's net/http — meaning every direct request gets challenged + // again. FS Chrome caches the clearance internally, so subsequent calls + // for the same domain are near-instant. + // + // When FS is not configured, fall back to direct HTTP. + if c.fsClient != nil { + return c.doFS(req, 0) + } + + // --- direct-first path (commented out — see TLS fingerprint issue above) --- + // resp, err := c.doDirect(req) + // var directStatus int + // if err == nil { + // directStatus = resp.StatusCode + // if resp.StatusCode != http.StatusForbidden && resp.StatusCode != http.StatusServiceUnavailable { + // return resp, nil + // } + // resp.Body.Close() + // } + // if c.fsClient == nil { + // if err != nil { + // return nil, err + // } + // return nil, fmt.Errorf("HTTP %d (challenge detected but FlareSolverr not configured)", resp.StatusCode) + // } + // return c.doFS(req, directStatus) + return c.doDirect(req) } func (c *Client) doDirect(req *http.Request) (*http.Response, error) { - if err := c.limiter(req.URL.Host).Wait(req.Context()); err != nil { - return nil, err - } - if c.referer != "" && req.Header.Get("Referer") == "" { - req.Header.Set("Referer", c.referer) - } - if req.Header.Get("User-Agent") == "" { - req.Header.Set("User-Agent", defaultUserAgent) - } - if c.verboseLog { - log.Printf("[httpclient] DIRECT GET %s", req.URL.String()) + log.Printf("[httpclient] DIRECT %s %s", req.Method, req.URL.String()) } const maxRetries = 3 @@ -124,6 +185,156 @@ func (c *Client) doDirect(req *http.Request) (*http.Response, error) { panic("unreachable") } +func (c *Client) doFS(req *http.Request, directStatus int) (*http.Response, error) { + if c.verboseLog { + log.Printf("[httpclient] FS FALLBACK %s %s", req.Method, req.URL.String()) + } + + rawURL := req.URL.String() + rawBody, statusCode, fsHeaders, cookies, fsRespURL, err := c.fsClient.GetRaw(req.Context(), rawURL) + if err != nil { + return nil, err + } + + // Use the actual response URL from FlareSolverr (follows redirects + // through Chrome) so cookies are associated with the right domain. + respURL := rawURL + if fsRespURL != "" { + respURL = fsRespURL + } + + // Feed FlareSolverr cookies into the shared jar so subsequent direct + // requests to the same host skip the challenge. + if len(cookies) > 0 { + if u, uErr := url.Parse(respURL); uErr == nil { + c.http.Jar.SetCookies(u, cookies) + } + } + + // When FlareSolverr returns status 200, Chrome rendered the page. + // Check if the body actually contains Cloudflare challenge indicators + // rather than relying on structural heuristics (
wrapper).
+ if statusCode == 200 {
+ if isCloudflareChallenge([]byte(rawBody)) {
+ statusCode = directStatus
+ }
+ }
+
+ // Build response headers from the actual FS response headers,
+ // falling back to the request headers for keys not present in the
+ // FS response (e.g. Content-Type on an empty GET body).
+ hdr := make(http.Header)
+ if len(fsHeaders) > 0 {
+ for k, v := range fsHeaders {
+ switch val := v.(type) {
+ case string:
+ hdr.Set(k, val)
+ case []any:
+ for _, sv := range val {
+ hdr.Add(k, fmt.Sprint(sv))
+ }
+ }
+ }
+ }
+ // Ensure Set-Cookie headers from FS cookies are present even if FS
+ // omitted them from the headers map.
+ if len(cookies) > 0 {
+ for _, ck := range cookies {
+ hdr.Add("Set-Cookie", ck.String())
+ }
+ }
+ // Copy any request headers not present in the FS response (e.g. Host).
+ for k, v := range req.Header {
+ if hdr.Get(k) == "" {
+ hdr[k] = v
+ }
+ }
+
+ body := stripFSWrapper([]byte(rawBody))
+
+ return &http.Response{
+ StatusCode: statusCode,
+ Header: hdr,
+ Body: io.NopCloser(bytes.NewReader(body)),
+ Request: req,
+ }, nil
+}
+
+// HTTPClient returns the underlying *http.Client (for passing to graphql etc.).
+func (c *Client) HTTPClient() *http.Client { return c.http }
+
+// Cookie returns the value of a named cookie stored in the jar for the given
+// host (e.g. "mangahub.io"). Returns empty string when the cookie is not found.
+func (c *Client) Cookie(name, host string) string {
+ u := &url.URL{Scheme: "https", Host: host}
+ for _, ck := range c.http.Jar.Cookies(u) {
+ if ck.Name == name {
+ return ck.Value
+ }
+ }
+ return ""
+}
+
+// Get is a convenience wrapper around Do. To add custom headers, build the
+// request manually and call Do.
+func (c *Client) Get(ctx context.Context, urlStr string) (*http.Response, error) {
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, urlStr, nil)
+ if err != nil {
+ return nil, err
+ }
+ return c.Do(req)
+}
+
+// Post is a convenience wrapper around Do.
+func (c *Client) Post(ctx context.Context, urlStr string, bodyType string, body io.Reader) (*http.Response, error) {
+ req, err := http.NewRequestWithContext(ctx, http.MethodPost, urlStr, body)
+ if err != nil {
+ return nil, err
+ }
+ req.Header.Set("Content-Type", bodyType)
+ return c.Do(req)
+}
+
+// isCloudflareChallenge detects whether the response body is a Cloudflare
+// challenge page (i.e. FS failed to solve it and Chrome rendered the challenge).
+func isCloudflareChallenge(body []byte) bool {
+ indicators := []string{
+ "Just a moment...",
+ "cf_chl_opt",
+ "challenges.cloudflare.com",
+ "/cdn-cgi/challenge-platform",
+ "Enable JavaScript and cookies",
+ }
+ for _, ind := range indicators {
+ if bytes.Contains(body, []byte(ind)) {
+ return true
+ }
+ }
+ return false
+}
+
+// stripFSWrapper removes FlareSolverr's Chrome HTML wrapper.
+// FlareSolverr wraps all responses in:
+//
+// ......actual_body
+//
+// If a tag is found inside the wrapper, its content is returned.
+// Otherwise the body is returned unchanged (HTML pages rendered by Chrome).
+func stripFSWrapper(body []byte) []byte {
+ if !bytes.HasPrefix(bytes.TrimSpace(body), []byte(""))
+ if preStart < 0 {
+ return body
+ }
+ preEnd := bytes.LastIndex(body, []byte(""))
+ if preEnd <= preStart {
+ return body
+ }
+ return body[preStart+5 : preEnd]
+}
+
func retryAfter(resp *http.Response) time.Duration {
ra := resp.Header.Get("Retry-After")
if ra == "" {
@@ -140,29 +351,3 @@ func retryAfter(resp *http.Response) time.Duration {
}
return 5 * time.Second
}
-
-func (c *Client) Get(ctx context.Context, url string, headers map[string]string) (*http.Response, error) {
- req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
- if err != nil {
- return nil, err
- }
- for k, v := range headers {
- req.Header.Set(k, v)
- }
- return c.Do(req)
-}
-
-func (c *Client) Post(ctx context.Context, url string, body io.Reader, contentType string, headers map[string]string) (*http.Response, error) {
- req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, body)
- if err != nil {
- return nil, err
- }
- req.Header.Set("Content-Type", contentType)
- for k, v := range headers {
- req.Header.Set(k, v)
- }
- return c.Do(req)
-}
-
-// HTTPClient returns the underlying *http.Client (for passing to graphql helper etc.)
-func (c *Client) HTTPClient() *http.Client { return c.http }
diff --git a/internal/httpclient/flare/flare.go b/internal/httpclient/flare/flare.go
index fc4196b..3f9b819 100644
--- a/internal/httpclient/flare/flare.go
+++ b/internal/httpclient/flare/flare.go
@@ -1,184 +1,31 @@
+// Package flare provides backward-compatible HTTP client helpers.
+//
+// All client logic now lives in the parent httpclient package.
+// This package re-exports httpclient.Client for sources that already import
+// "goyomi/internal/httpclient/flare" and is kept to avoid breaking existing code.
package flare
-import (
- "context"
- "fmt"
- "io"
- "net/http"
- "net/http/cookiejar"
- "net/url"
- "strings"
- "sync"
- "time"
+import "goyomi/internal/httpclient"
- "goyomi/internal/httpclient"
- "golang.org/x/time/rate"
-)
+// Client is an alias for httpclient.Client.
+//
+// Sources that need the shared singleton should call httpclient.DefaultClient().
+// Sources that need custom configuration (e.g., different rate limit) should
+// call httpclient.NewClient() directly.
+type Client = httpclient.Client
-const defaultUserAgent = "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Mobile Safari/537.36"
-
-type Client struct {
- fsClient *httpclient.FlareSolverrClient
- rateLimit float64
- burst int
- referer string
-
- mu sync.Mutex
- limiters map[string]*rate.Limiter
+// NewClient creates a new httpclient.Client.
+//
+// Deprecated: prefer httpclient.DefaultClient() for the shared singleton or
+// httpclient.NewClient(...) for sources with custom configuration.
+func NewClient(opts ...httpclient.Option) *Client {
+ return httpclient.NewClient(opts...)
}
-type Response struct {
- *http.Response
- Body io.ReadCloser
-}
-
-func (r *Response) Close() error {
- return r.Body.Close()
-}
-
-type Option func(*Client)
+// Option aliases httpclient.Option.
+type Option = httpclient.Option
+// WithRateLimit aliases httpclient.WithRateLimit.
func WithRateLimit(rps float64, burst int) Option {
- return func(c *Client) {
- c.rateLimit = rps
- c.burst = burst
- }
+ return httpclient.WithRateLimit(rps, burst)
}
-
-func WithReferer(referer string) Option {
- return func(c *Client) { c.referer = referer }
-}
-
-func WithTimeout(d time.Duration) Option {
- return func(c *Client) {}
-}
-
-func NewClient(opts ...Option) *Client {
- c := &Client{
- limiters: make(map[string]*rate.Limiter),
- }
- for _, opt := range opts {
- opt(c)
- }
- if c.rateLimit == 0 {
- c.rateLimit = 1
- }
- if c.burst == 0 {
- c.burst = 1
- }
- fsClient, err := httpclient.NewFlareSolverrClient()
- if err == nil {
- c.fsClient = fsClient
- }
- return c
-}
-
-func (c *Client) SetFlareSolverrClient(fs *httpclient.FlareSolverrClient) {
- c.fsClient = fs
-}
-
-func (c *Client) doRequest(ctx context.Context, method string, rawURL string, body string) (*Response, error) {
- if c.fsClient == nil {
- return nil, fmt.Errorf("FlareSolverr client not configured")
- }
- c.mu.Lock()
- limiter, ok := c.limiters[rawURL]
- if !ok {
- limiter = rate.NewLimiter(rate.Limit(c.rateLimit), c.burst)
- c.limiters[rawURL] = limiter
- }
- c.mu.Unlock()
- if err := limiter.Wait(ctx); err != nil {
- return nil, err
- }
-
- var html string
- var cookies []*http.Cookie
- var err error
-
- if method == http.MethodGet {
- html, cookies, err = c.fsClient.Get(ctx, rawURL)
- } else {
- html, cookies, err = c.fsClient.Post(ctx, rawURL, body)
- }
- if err != nil {
- return nil, err
- }
-
- jar, _ := cookiejar.New(nil)
- if len(cookies) > 0 {
- httpURL, _ := http.NewRequest(http.MethodGet, rawURL, nil)
- jar.SetCookies(httpURL.URL, cookies)
- }
-
- fakeResp := &http.Response{
- StatusCode: http.StatusOK,
- Header: http.Header{},
- Body: io.NopCloser(strings.NewReader(html)),
- Request: &http.Request{URL: &url.URL{Path: rawURL}},
- }
- return &Response{Response: fakeResp, Body: fakeResp.Body}, nil
-}
-
-func (c *Client) Get(ctx context.Context, rawURL string) (*Response, error) {
- return c.doRequest(ctx, http.MethodGet, rawURL, "")
-}
-
-func (c *Client) Post(ctx context.Context, rawURL string, bodyType string, body io.Reader) (*Response, error) {
- bodyStr, _ := io.ReadAll(body)
- return c.doRequest(ctx, http.MethodPost, rawURL, string(bodyStr))
-}
-
-func (c *Client) GetHTML(ctx context.Context, url string) (io.ReadCloser, error) {
- resp, err := c.Get(ctx, url)
- if err != nil {
- return nil, err
- }
- if resp.StatusCode != http.StatusOK {
- resp.Body.Close()
- return nil, fmt.Errorf("HTTP %d for %s", resp.StatusCode, url)
- }
- return resp.Body, nil
-}
-
-func (c *Client) PostHTML(ctx context.Context, url string, body string) (io.ReadCloser, error) {
- resp, err := c.Post(ctx, url, "application/x-www-form-urlencoded", strings.NewReader(body))
- if err != nil {
- return nil, err
- }
- if resp.StatusCode != http.StatusOK {
- resp.Body.Close()
- return nil, fmt.Errorf("HTTP %d for %s", resp.StatusCode, url)
- }
- return resp.Body, nil
-}
-
-func (c *Client) GetBytes(ctx context.Context, url string) ([]byte, error) {
- resp, err := c.Get(ctx, url)
- if err != nil {
- return nil, err
- }
- defer resp.Body.Close()
- if resp.StatusCode != http.StatusOK {
- return nil, fmt.Errorf("HTTP %d for %s", resp.StatusCode, url)
- }
- return io.ReadAll(resp.Body)
-}
-
-func (c *Client) SetProxy(proxyURL string) error {
- return nil
-}
-
-func (c *Client) Do(req *http.Request) (*http.Response, error) {
- url := req.URL.String()
- var body string
- if req.Body != nil {
- b, _ := io.ReadAll(req.Body)
- body = string(b)
- }
- resp, err := c.doRequest(req.Context(), req.Method, url, body)
- if err != nil {
- return nil, err
- }
- return resp.Response, nil
-}
\ No newline at end of file
diff --git a/internal/httpclient/flaresolverr.go b/internal/httpclient/flaresolverr.go
index bbc286b..29a222e 100755
--- a/internal/httpclient/flaresolverr.go
+++ b/internal/httpclient/flaresolverr.go
@@ -10,8 +10,9 @@ import (
)
type FlareSolverrClient struct {
- endpoint string
- client *http.Client
+ endpoint string
+ client *http.Client
+ sessionID string
}
func NewFlareSolverrClient() (*FlareSolverrClient, error) {
@@ -19,9 +20,14 @@ func NewFlareSolverrClient() (*FlareSolverrClient, error) {
if ep == "" {
return nil, fmt.Errorf("FLARESOLVERR_URL not set")
}
+ sessionID := os.Getenv("FLARESOLVERR_SESSION")
+ if sessionID == "" {
+ sessionID = "goyomi"
+ }
return &FlareSolverrClient{
- endpoint: ep,
- client: &http.Client{},
+ endpoint: ep,
+ client: &http.Client{},
+ sessionID: sessionID,
}, nil
}
@@ -58,6 +64,66 @@ func (f *FlareSolverrClient) Get(ctx context.Context, url string) (html string,
return f.request(ctx, "request.get", url, "", nil)
}
+// GetRaw fetches a Cloudflare-protected URL via FlareSolverr with raw mode.
+// Returns the raw response body, actual HTTP status code, response headers, cookies,
+// and the final response URL (after redirects). Unlike Get, this does NOT render the
+// page through Chrome — it returns the raw server response, making it suitable for
+// JSON API endpoints behind Cloudflare.
+func (f *FlareSolverrClient) GetRaw(ctx context.Context, url string) (body string, statusCode int, respHeaders map[string]any, cookies []*http.Cookie, respURL string, err error) {
+ type fsRequest struct {
+ Cmd string `json:"cmd"`
+ URL string `json:"url"`
+ PostData string `json:"postData,omitempty"`
+ Headers map[string]string `json:"headers,omitempty"`
+ MaxTimeout int `json:"maxTimeout"`
+ Raw bool `json:"raw,omitempty"`
+ Session string `json:"session,omitempty"`
+ }
+
+ reqData := fsRequest{
+ Cmd: "request.get",
+ URL: url,
+ MaxTimeout: 60000,
+ Raw: true,
+ Session: f.sessionID,
+ }
+
+ payload, _ := json.Marshal(reqData)
+
+ httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, f.endpoint+"/v1", bytes.NewReader(payload))
+ if err != nil {
+ return "", 0, nil, nil, "", err
+ }
+ httpReq.Header.Set("Content-Type", "application/json")
+
+ resp, err := f.client.Do(httpReq)
+ if err != nil {
+ return "", 0, nil, nil, "", err
+ }
+ defer resp.Body.Close()
+
+ var fsResp FlareSolverrResponse
+ if err := json.NewDecoder(resp.Body).Decode(&fsResp); err != nil {
+ return "", 0, nil, nil, "", err
+ }
+ if fsResp.Status != "ok" {
+ return "", 0, nil, nil, "", fmt.Errorf("flaresolverr: status %q", fsResp.Status)
+ }
+
+ for _, c := range fsResp.Solution.Cookies {
+ cookies = append(cookies, &http.Cookie{
+ Name: c.Name,
+ Value: c.Value,
+ Domain: c.Domain,
+ Path: c.Path,
+ HttpOnly: c.HTTPOnly,
+ Secure: c.Secure,
+ })
+ }
+
+ return fsResp.Solution.Response, fsResp.Solution.Status, fsResp.Solution.Headers, cookies, fsResp.Solution.URL, nil
+}
+
func (f *FlareSolverrClient) Post(ctx context.Context, url string, body string) (html string, cookies []*http.Cookie, err error) {
return f.request(ctx, "request.post", url, body, map[string]string{"Content-Type": "application/x-www-form-urlencoded"})
}
@@ -69,12 +135,14 @@ func (f *FlareSolverrClient) request(ctx context.Context, cmd, url, body string,
PostData string `json:"postData,omitempty"`
Headers map[string]string `json:"headers,omitempty"`
MaxTimeout int `json:"maxTimeout"`
+ Session string `json:"session,omitempty"`
}
req := fsRequest{
Cmd: cmd,
URL: url,
MaxTimeout: 60000,
+ Session: f.sessionID,
}
if body != "" {
req.PostData = body
diff --git a/internal/sourcetest/sourcetest.go b/internal/sourcetest/sourcetest.go
index 23e2887..93c577c 100644
--- a/internal/sourcetest/sourcetest.go
+++ b/internal/sourcetest/sourcetest.go
@@ -39,6 +39,12 @@ func Run(t *testing.T, s source.CatalogueSource, wantName, wantLang string) {
if len(page.Mangas) == 0 {
t.Fatal("GetPopularManga returned 0 results")
}
+ if testing.Verbose() {
+ t.Logf("--- GetPopularManga (%d results) ---", len(page.Mangas))
+ for i, m := range page.Mangas {
+ t.Logf(" [%d] %-60s %s", i, m.Title, m.URL)
+ }
+ }
for i, m := range page.Mangas {
if m.Title == "" {
t.Errorf("manga[%d].Title is empty", i)
@@ -61,6 +67,12 @@ func Run(t *testing.T, s source.CatalogueSource, wantName, wantLang string) {
if len(page.Mangas) == 0 {
t.Fatal("GetLatestUpdates returned 0 results")
}
+ if testing.Verbose() {
+ t.Logf("--- GetLatestUpdates (%d results) ---", len(page.Mangas))
+ for i, m := range page.Mangas {
+ t.Logf(" [%d] %-60s %s", i, m.Title, m.URL)
+ }
+ }
for i, m := range page.Mangas {
if m.Title == "" {
t.Errorf("manga[%d].Title is empty", i)
diff --git a/scripts/test-sources.sh b/scripts/test-sources.sh
index d158d64..d4deb85 100755
--- a/scripts/test-sources.sh
+++ b/scripts/test-sources.sh
@@ -2,9 +2,11 @@
# test-sources.sh — run source integration tests inside the compose network.
#
# Usage:
-# ./scripts/test-sources.sh # run all source packages
-# ./scripts/test-sources.sh ./sources/en/... # run specific packages
-# ./scripts/test-sources.sh -short # metadata-only, no network calls
+# ./scripts/test-sources.sh # run all source packages
+# ./scripts/test-sources.sh ./sources/en/... # run specific packages
+# ./scripts/test-sources.sh -short # metadata-only, no network calls
+# ./scripts/test-sources.sh -v # verbose (print manga lists)
+# ./scripts/test-sources.sh -v ./sources/en/aquamanga/... # verbose for one source
#
# Environment:
# PARALLELISM number of packages to test in parallel (default: 4)
@@ -229,12 +231,18 @@ ok "Network $NETWORK exists"
# --- resolve packages to test ------------------------------------------------
SHORT_FLAG=""
+VERBOSE_FLAG=""
+FLAGS=""
+while [ $# -gt 0 ]; do
+ case "$1" in
+ -short) SHORT_FLAG="-short"; FLAGS="$FLAGS -short"; shift ;;
+ -v) VERBOSE_FLAG="-v"; FLAGS="$FLAGS -v"; shift ;;
+ *) break ;;
+ esac
+done
+
if [ $# -eq 0 ]; then
PACKAGES="./sources/en/... ./sources/all/..."
-elif [ "$1" = "-short" ]; then
- PACKAGES="./sources/en/... ./sources/all/..."
- SHORT_FLAG="-short"
- shift
else
PACKAGES="$*"
fi
@@ -243,6 +251,7 @@ info "Packages : $PACKAGES"
info "Parallel : $PARALLELISM"
info "Timeout : $PKG_TIMEOUT"
[ -n "$SHORT_FLAG" ] && info "Mode : short (metadata only)" || info "Mode : full (live network)"
+[ -n "$VERBOSE_FLAG" ] && info "Verbose : on"
# --- run tests ---------------------------------------------------------------
@@ -258,7 +267,7 @@ docker run --rm \
-v goyomi_go_build_cache:/root/.cache/go-build \
-w /workspace \
"$GO_IMAGE" \
- go test -json -count=1 -p "$PARALLELISM" -timeout "$PKG_TIMEOUT" $SHORT_FLAG $PACKAGES \
+ go test -json -count=1 -p "$PARALLELISM" -timeout "$PKG_TIMEOUT" $FLAGS $PACKAGES \
| python3 -u "$PRINTER_PY" "$JSON_LOG"
EXIT_CODE=$?
set -e