goyomi/internal/httpclient/flaresolverr.go

package httpclient

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"os"
)

type FlareSolverrClient struct {
	endpoint  string
	client    *http.Client
	sessionID string
}

func NewFlareSolverrClient() (*FlareSolverrClient, error) {
	ep := os.Getenv("FLARESOLVERR_URL")
	if ep == "" {
		return nil, fmt.Errorf("FLARESOLVERR_URL not set")
	}
	sessionID := os.Getenv("FLARESOLVERR_SESSION")
	if sessionID == "" {
		sessionID = "goyomi"
	}
	return &FlareSolverrClient{
		endpoint:  ep,
		client:    &http.Client{},
		sessionID: sessionID,
	}, nil
}

type flareSolverrRequest struct {
	Cmd        string `json:"cmd"`
	URL        string `json:"url"`
	MaxTimeout int    `json:"maxTimeout"`
}

type FlareSolverrResponse struct {
	Status   string `json:"status"`
	Message  string `json:"message,omitempty"`
	Solution struct {
		Response string          `json:"response"`
		Cookies  []fsCookie      `json:"cookies"`
		Headers  map[string]any  `json:"headers"`
		URL      string          `json:"url"`
		Status   int             `json:"status"`
	} `json:"solution"`
}

type fsCookie struct {
	Name     string  `json:"name"`
	Value    string  `json:"value"`
	Domain   string  `json:"domain"`
	Path     string  `json:"path"`
	Expires  float64 `json:"expires"`
	HTTPOnly bool    `json:"httpOnly"`
	Secure   bool    `json:"secure"`
}

// Get fetches a Cloudflare-protected URL via FlareSolverr.
// Returns rendered HTML and extracted cookies.
func (f *FlareSolverrClient) Get(ctx context.Context, url string) (html string, cookies []*http.Cookie, err error) {
	return f.request(ctx, "request.get", url, "", nil)
}

// GetRaw fetches a Cloudflare-protected URL via FlareSolverr with raw mode.
// Returns the raw response body, actual HTTP status code, response headers, cookies,
// and the final response URL (after redirects). Unlike Get, this does NOT render the
// page through Chrome — it returns the raw server response, making it suitable for
// JSON API endpoints behind Cloudflare.
func (f *FlareSolverrClient) GetRaw(ctx context.Context, url string) (body string, statusCode int, respHeaders map[string]any, cookies []*http.Cookie, respURL string, err error) {
	type fsRequest struct {
		Cmd        string            `json:"cmd"`
		URL        string            `json:"url"`
		PostData   string            `json:"postData,omitempty"`
		Headers    map[string]string `json:"headers,omitempty"`
		MaxTimeout int               `json:"maxTimeout"`
		Raw        bool              `json:"raw,omitempty"`
		Session    string            `json:"session,omitempty"`
	}

	reqData := fsRequest{
		Cmd:        "request.get",
		URL:        url,
		MaxTimeout: 120000,
		Raw:        true,
		Session:    f.sessionID,
	}

	payload, _ := json.Marshal(reqData)

	httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, f.endpoint+"/v1", bytes.NewReader(payload))
	if err != nil {
		return "", 0, nil, nil, "", err
	}
	httpReq.Header.Set("Content-Type", "application/json")

	resp, err := f.client.Do(httpReq)
	if err != nil {
		return "", 0, nil, nil, "", err
	}
	defer resp.Body.Close()

	var fsResp FlareSolverrResponse
	if err := json.NewDecoder(resp.Body).Decode(&fsResp); err != nil {
		return "", 0, nil, nil, "", err
	}
	if fsResp.Status != "ok" {
		msg := fsResp.Message
		if msg == "" {
			msg = fmt.Sprintf("status %q", fsResp.Status)
		}
		return "", 0, nil, nil, "", fmt.Errorf("flaresolverr: %s", msg)
	}

	for _, c := range fsResp.Solution.Cookies {
		cookies = append(cookies, &http.Cookie{
			Name:     c.Name,
			Value:    c.Value,
			Domain:   c.Domain,
			Path:     c.Path,
			HttpOnly: c.HTTPOnly,
			Secure:   c.Secure,
		})
	}

	return fsResp.Solution.Response, fsResp.Solution.Status, fsResp.Solution.Headers, cookies, fsResp.Solution.URL, nil
}

func (f *FlareSolverrClient) Post(ctx context.Context, url string, body string) (html string, cookies []*http.Cookie, err error) {
	return f.request(ctx, "request.post", url, body, map[string]string{"Content-Type": "application/x-www-form-urlencoded"})
}

func (f *FlareSolverrClient) request(ctx context.Context, cmd, url, body string, headers map[string]string) (html string, cookies []*http.Cookie, err error) {
	type fsRequest struct {
		Cmd        string            `json:"cmd"`
		URL        string            `json:"url"`
		PostData   string            `json:"postData,omitempty"`
		Headers    map[string]string `json:"headers,omitempty"`
		MaxTimeout int               `json:"maxTimeout"`
		Session    string            `json:"session,omitempty"`
	}

	req := fsRequest{
		Cmd:        cmd,
		URL:        url,
		MaxTimeout: 120000,
		Session:    f.sessionID,
	}
	if body != "" {
		req.PostData = body
	}
	if headers != nil {
		req.Headers = headers
	}

	payload, _ := json.Marshal(req)

	httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, f.endpoint+"/v1", bytes.NewReader(payload))
	if err != nil {
		return "", nil, err
	}
	httpReq.Header.Set("Content-Type", "application/json")

	resp, err := f.client.Do(httpReq)
	if err != nil {
		return "", nil, err
	}
	defer resp.Body.Close()

	var fsResp FlareSolverrResponse
	if err := json.NewDecoder(resp.Body).Decode(&fsResp); err != nil {
		return "", nil, err
	}
	if fsResp.Status != "ok" {
		msg := fsResp.Message
		if msg == "" {
			msg = fmt.Sprintf("status %q", fsResp.Status)
		}
		return "", nil, fmt.Errorf("flaresolverr: %s", msg)
	}

	for _, c := range fsResp.Solution.Cookies {
		cookies = append(cookies, &http.Cookie{
			Name:     c.Name,
			Value:    c.Value,
			Domain:   c.Domain,
			Path:     c.Path,
			HttpOnly: c.HTTPOnly,
			Secure:   c.Secure,
		})
	}
	return fsResp.Solution.Response, cookies, nil
}