fix: read full response body instead of truncating at 32KB
The challenge detection only needs a sample (32KB), but when the body is not a challenge, the full body must be returned. The previous code discarded everything after 32KB, causing Madara parsers to find 0 manga items in truncated HTML.
This commit is contained in:
@@ -123,23 +123,31 @@ func (c *Client) Do(req *http.Request) (*http.Response, error) {
|
||||
// Some Cloudflare challenge pages return HTTP 200 but contain challenge
|
||||
// JavaScript. Check for challenge content on any response.
|
||||
if resp.StatusCode == http.StatusOK {
|
||||
body, readErr := io.ReadAll(io.LimitReader(resp.Body, 32*1024))
|
||||
resp.Body.Close()
|
||||
if readErr == nil && len(body) > 0 {
|
||||
if isCloudflareChallenge(body) {
|
||||
directStatus = http.StatusForbidden
|
||||
} else {
|
||||
// Not a challenge — wrap body and return
|
||||
// Read a sample to check for challenge content
|
||||
sample, readErr := io.ReadAll(io.LimitReader(resp.Body, 32*1024))
|
||||
if readErr == nil && len(sample) > 0 && isCloudflareChallenge(sample) {
|
||||
resp.Body.Close()
|
||||
directStatus = http.StatusForbidden
|
||||
} else {
|
||||
// Read the rest of the body
|
||||
rest, restErr := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
if restErr == nil {
|
||||
fullBody := append(sample, rest...)
|
||||
return &http.Response{
|
||||
StatusCode: resp.StatusCode,
|
||||
Header: resp.Header,
|
||||
Body: io.NopCloser(bytes.NewReader(body)),
|
||||
ContentLength: int64(len(body)),
|
||||
Body: io.NopCloser(bytes.NewReader(fullBody)),
|
||||
ContentLength: int64(len(fullBody)),
|
||||
Request: req,
|
||||
}, nil
|
||||
}
|
||||
} else {
|
||||
return resp, nil
|
||||
return &http.Response{
|
||||
StatusCode: resp.StatusCode,
|
||||
Header: resp.Header,
|
||||
Body: io.NopCloser(bytes.NewReader(sample)),
|
||||
Request: req,
|
||||
}, nil
|
||||
}
|
||||
} else if resp.StatusCode != http.StatusForbidden && resp.StatusCode != http.StatusServiceUnavailable {
|
||||
return resp, nil
|
||||
|
||||
Reference in New Issue
Block a user