fix: read full response body instead of truncating at 32KB
The challenge detection only needs a sample (32KB), but when the body is not a challenge, the full body must be returned. The previous code discarded everything after 32KB, causing Madara parsers to find 0 manga items in truncated HTML.
This commit is contained in:
@@ -123,23 +123,31 @@ func (c *Client) Do(req *http.Request) (*http.Response, error) {
|
|||||||
// Some Cloudflare challenge pages return HTTP 200 but contain challenge
|
// Some Cloudflare challenge pages return HTTP 200 but contain challenge
|
||||||
// JavaScript. Check for challenge content on any response.
|
// JavaScript. Check for challenge content on any response.
|
||||||
if resp.StatusCode == http.StatusOK {
|
if resp.StatusCode == http.StatusOK {
|
||||||
body, readErr := io.ReadAll(io.LimitReader(resp.Body, 32*1024))
|
// Read a sample to check for challenge content
|
||||||
resp.Body.Close()
|
sample, readErr := io.ReadAll(io.LimitReader(resp.Body, 32*1024))
|
||||||
if readErr == nil && len(body) > 0 {
|
if readErr == nil && len(sample) > 0 && isCloudflareChallenge(sample) {
|
||||||
if isCloudflareChallenge(body) {
|
resp.Body.Close()
|
||||||
directStatus = http.StatusForbidden
|
directStatus = http.StatusForbidden
|
||||||
} else {
|
} else {
|
||||||
// Not a challenge — wrap body and return
|
// Read the rest of the body
|
||||||
|
rest, restErr := io.ReadAll(resp.Body)
|
||||||
|
resp.Body.Close()
|
||||||
|
if restErr == nil {
|
||||||
|
fullBody := append(sample, rest...)
|
||||||
return &http.Response{
|
return &http.Response{
|
||||||
StatusCode: resp.StatusCode,
|
StatusCode: resp.StatusCode,
|
||||||
Header: resp.Header,
|
Header: resp.Header,
|
||||||
Body: io.NopCloser(bytes.NewReader(body)),
|
Body: io.NopCloser(bytes.NewReader(fullBody)),
|
||||||
ContentLength: int64(len(body)),
|
ContentLength: int64(len(fullBody)),
|
||||||
Request: req,
|
Request: req,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
} else {
|
return &http.Response{
|
||||||
return resp, nil
|
StatusCode: resp.StatusCode,
|
||||||
|
Header: resp.Header,
|
||||||
|
Body: io.NopCloser(bytes.NewReader(sample)),
|
||||||
|
Request: req,
|
||||||
|
}, nil
|
||||||
}
|
}
|
||||||
} else if resp.StatusCode != http.StatusForbidden && resp.StatusCode != http.StatusServiceUnavailable {
|
} else if resp.StatusCode != http.StatusForbidden && resp.StatusCode != http.StatusServiceUnavailable {
|
||||||
return resp, nil
|
return resp, nil
|
||||||
|
|||||||
Reference in New Issue
Block a user