fix(httpclient): fix HTTP 0 status and restore FS session default
- Guard isCloudflareChallenge with directStatus >= 400 to prevent overriding status to 0 when no direct request was made - When FS returns challenge page without a prior direct status, return an error instead of silently passing HTTP 0 - Restore default FS session ID to 'goyomi' — without a session, each request spawns a new Chrome, causing timeouts under load - Add Message field to FlareSolverrResponse for better error reporting - Document FLARESOLVERR_SESSION env var: shared session = fast after 1st request, but serializes. Set empty for parallel (resource-heavy).
This commit is contained in:
+1
-1
@@ -8,7 +8,7 @@ DATABASE_URL=postgres://goyomi:goyomi@postgres:5432/goyomi?sslmode=disable
|
|||||||
FLARESOLVERR_URL=http://flaresolverr:8191
|
FLARESOLVERR_URL=http://flaresolverr:8191
|
||||||
FLARESOLVERR_LOG_LEVEL=info
|
FLARESOLVERR_LOG_LEVEL=info
|
||||||
FLARESOLVERR_PROXY=0
|
FLARESOLVERR_PROXY=0
|
||||||
FLARESOLVERR_SESSION=goyomi
|
FLARESOLVERR_SESSION=goyomi # FS browser session ID; shared session = faster after 1st request. Set empty to disable (each request gets a fresh browser = parallel but resource-heavy)
|
||||||
ADDR=:3300
|
ADDR=:3300
|
||||||
|
|
||||||
# Connection pool
|
# Connection pool
|
||||||
|
|||||||
@@ -211,12 +211,13 @@ func (c *Client) doFS(req *http.Request, directStatus int) (*http.Response, erro
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// When FlareSolverr returns status 200, Chrome rendered the page.
|
// If FS returned the challenge page instead of the real content,
|
||||||
// Check if the body actually contains Cloudflare challenge indicators
|
// reject it (HTTP 0 case when directStatus=0).
|
||||||
// rather than relying on structural heuristics (<pre> wrapper).
|
if statusCode == 200 && isCloudflareChallenge([]byte(rawBody)) {
|
||||||
if statusCode == 200 {
|
if directStatus >= 400 {
|
||||||
if isCloudflareChallenge([]byte(rawBody)) {
|
|
||||||
statusCode = directStatus
|
statusCode = directStatus
|
||||||
|
} else {
|
||||||
|
return nil, fmt.Errorf("FlareSolverr returned challenge page for %s", rawURL)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -39,6 +39,7 @@ type flareSolverrRequest struct {
|
|||||||
|
|
||||||
type FlareSolverrResponse struct {
|
type FlareSolverrResponse struct {
|
||||||
Status string `json:"status"`
|
Status string `json:"status"`
|
||||||
|
Message string `json:"message,omitempty"`
|
||||||
Solution struct {
|
Solution struct {
|
||||||
Response string `json:"response"`
|
Response string `json:"response"`
|
||||||
Cookies []fsCookie `json:"cookies"`
|
Cookies []fsCookie `json:"cookies"`
|
||||||
@@ -107,7 +108,11 @@ func (f *FlareSolverrClient) GetRaw(ctx context.Context, url string) (body strin
|
|||||||
return "", 0, nil, nil, "", err
|
return "", 0, nil, nil, "", err
|
||||||
}
|
}
|
||||||
if fsResp.Status != "ok" {
|
if fsResp.Status != "ok" {
|
||||||
return "", 0, nil, nil, "", fmt.Errorf("flaresolverr: status %q", fsResp.Status)
|
msg := fsResp.Message
|
||||||
|
if msg == "" {
|
||||||
|
msg = fmt.Sprintf("status %q", fsResp.Status)
|
||||||
|
}
|
||||||
|
return "", 0, nil, nil, "", fmt.Errorf("flaresolverr: %s", msg)
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, c := range fsResp.Solution.Cookies {
|
for _, c := range fsResp.Solution.Cookies {
|
||||||
@@ -170,7 +175,11 @@ func (f *FlareSolverrClient) request(ctx context.Context, cmd, url, body string,
|
|||||||
return "", nil, err
|
return "", nil, err
|
||||||
}
|
}
|
||||||
if fsResp.Status != "ok" {
|
if fsResp.Status != "ok" {
|
||||||
return "", nil, fmt.Errorf("flaresolverr: status %q", fsResp.Status)
|
msg := fsResp.Message
|
||||||
|
if msg == "" {
|
||||||
|
msg = fmt.Sprintf("status %q", fsResp.Status)
|
||||||
|
}
|
||||||
|
return "", nil, fmt.Errorf("flaresolverr: %s", msg)
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, c := range fsResp.Solution.Cookies {
|
for _, c := range fsResp.Solution.Cookies {
|
||||||
|
|||||||
Reference in New Issue
Block a user