diff --git a/.env.example b/.env.example index 10fb2a9..4e7434c 100755 --- a/.env.example +++ b/.env.example @@ -8,7 +8,7 @@ DATABASE_URL=postgres://goyomi:goyomi@postgres:5432/goyomi?sslmode=disable FLARESOLVERR_URL=http://flaresolverr:8191 FLARESOLVERR_LOG_LEVEL=info FLARESOLVERR_PROXY=0 -FLARESOLVERR_SESSION=goyomi +FLARESOLVERR_SESSION=goyomi # FS browser session ID; shared session = faster after 1st request. Set empty to disable (each request gets a fresh browser = parallel but resource-heavy) ADDR=:3300 # Connection pool diff --git a/internal/httpclient/client.go b/internal/httpclient/client.go index 2b119f0..fafc938 100755 --- a/internal/httpclient/client.go +++ b/internal/httpclient/client.go @@ -211,12 +211,13 @@ func (c *Client) doFS(req *http.Request, directStatus int) (*http.Response, erro } } - // When FlareSolverr returns status 200, Chrome rendered the page. - // Check if the body actually contains Cloudflare challenge indicators - // rather than relying on structural heuristics (
wrapper).
- if statusCode == 200 {
- if isCloudflareChallenge([]byte(rawBody)) {
+ // If FS returned the challenge page instead of the real content,
+ // reject it (HTTP 0 case when directStatus=0).
+ if statusCode == 200 && isCloudflareChallenge([]byte(rawBody)) {
+ if directStatus >= 400 {
statusCode = directStatus
+ } else {
+ return nil, fmt.Errorf("FlareSolverr returned challenge page for %s", rawURL)
}
}
diff --git a/internal/httpclient/flaresolverr.go b/internal/httpclient/flaresolverr.go
index 29a222e..ed1fcdb 100755
--- a/internal/httpclient/flaresolverr.go
+++ b/internal/httpclient/flaresolverr.go
@@ -39,6 +39,7 @@ type flareSolverrRequest struct {
type FlareSolverrResponse struct {
Status string `json:"status"`
+ Message string `json:"message,omitempty"`
Solution struct {
Response string `json:"response"`
Cookies []fsCookie `json:"cookies"`
@@ -107,7 +108,11 @@ func (f *FlareSolverrClient) GetRaw(ctx context.Context, url string) (body strin
return "", 0, nil, nil, "", err
}
if fsResp.Status != "ok" {
- return "", 0, nil, nil, "", fmt.Errorf("flaresolverr: status %q", fsResp.Status)
+ msg := fsResp.Message
+ if msg == "" {
+ msg = fmt.Sprintf("status %q", fsResp.Status)
+ }
+ return "", 0, nil, nil, "", fmt.Errorf("flaresolverr: %s", msg)
}
for _, c := range fsResp.Solution.Cookies {
@@ -170,7 +175,11 @@ func (f *FlareSolverrClient) request(ctx context.Context, cmd, url, body string,
return "", nil, err
}
if fsResp.Status != "ok" {
- return "", nil, fmt.Errorf("flaresolverr: status %q", fsResp.Status)
+ msg := fsResp.Message
+ if msg == "" {
+ msg = fmt.Sprintf("status %q", fsResp.Status)
+ }
+ return "", nil, fmt.Errorf("flaresolverr: %s", msg)
}
for _, c := range fsResp.Solution.Cookies {