feat: initial Phase 1 implementation — core framework + Docker

- Data types (SManga, SChapter, Page, MangasPage, all Filter variants) - Source interfaces (Source, CatalogueSource) with MD5-based ID generation matching Tachiyomi/Suwayomi - HTTP client with per-host rate limiting, cookie jar, and 429 retry - FlareSolverr v1 client (FLARESOLVERR_URL env) - Generic GraphQL POST helper - goquery HTML parser wrappers - Source registry (panics on duplicate ID) - Multi-stage Dockerfile (golang:1.26-alpine + distroless) and compose.yml (postgres, flaresolverr, app)
2026-05-10 21:23:24 +07:00
commit 85d2ea6143
23 changed files with 2864 additions and 0 deletions
@@ -0,0 +1,147 @@
+package httpclient
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"net/http/cookiejar"
+	"strconv"
+	"sync"
+	"time"
+
+	"golang.org/x/time/rate"
+)
+
+const defaultUserAgent = "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Mobile Safari/537.36"
+
+type Client struct {
+	http      *http.Client
+	rateLimit float64
+	burst     int
+	referer   string
+
+	mu       sync.Mutex
+	limiters map[string]*rate.Limiter
+}
+
+type Option func(*Client)
+
+func WithRateLimit(rps float64, burst int) Option {
+	return func(c *Client) {
+		c.rateLimit = rps
+		c.burst = burst
+	}
+}
+
+func WithTimeout(d time.Duration) Option {
+	return func(c *Client) { c.http.Timeout = d }
+}
+
+func WithReferer(referer string) Option {
+	return func(c *Client) { c.referer = referer }
+}
+
+func NewClient(opts ...Option) *Client {
+	jar, _ := cookiejar.New(nil)
+	c := &Client{
+		http: &http.Client{
+			Timeout: 30 * time.Second,
+			Jar:     jar,
+		},
+		rateLimit: 1,
+		burst:     1,
+		limiters:  map[string]*rate.Limiter{},
+	}
+	for _, o := range opts {
+		o(c)
+	}
+	return c
+}
+
+func (c *Client) limiter(host string) *rate.Limiter {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	l, ok := c.limiters[host]
+	if !ok {
+		l = rate.NewLimiter(rate.Limit(c.rateLimit), c.burst)
+		c.limiters[host] = l
+	}
+	return l
+}
+
+func (c *Client) Do(req *http.Request) (*http.Response, error) {
+	if err := c.limiter(req.URL.Host).Wait(req.Context()); err != nil {
+		return nil, err
+	}
+	if c.referer != "" && req.Header.Get("Referer") == "" {
+		req.Header.Set("Referer", c.referer)
+	}
+	if req.Header.Get("User-Agent") == "" {
+		req.Header.Set("User-Agent", defaultUserAgent)
+	}
+
+	const maxRetries = 3
+	for attempt := 0; attempt <= maxRetries; attempt++ {
+		resp, err := c.http.Do(req)
+		if err != nil {
+			return nil, err
+		}
+		if resp.StatusCode != http.StatusTooManyRequests {
+			return resp, nil
+		}
+		resp.Body.Close()
+		if attempt == maxRetries {
+			return resp, nil
+		}
+		sleep := retryAfter(resp)
+		select {
+		case <-req.Context().Done():
+			return nil, req.Context().Err()
+		case <-time.After(sleep):
+		}
+	}
+	panic("unreachable")
+}
+
+func retryAfter(resp *http.Response) time.Duration {
+	ra := resp.Header.Get("Retry-After")
+	if ra == "" {
+		return 5 * time.Second
+	}
+	if secs, err := strconv.ParseFloat(ra, 64); err == nil {
+		return time.Duration(secs * float64(time.Second))
+	}
+	if t, err := http.ParseTime(ra); err == nil {
+		d := time.Until(t)
+		if d > 0 {
+			return d
+		}
+	}
+	return 5 * time.Second
+}
+
+func (c *Client) Get(ctx context.Context, url string, headers map[string]string) (*http.Response, error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return nil, err
+	}
+	for k, v := range headers {
+		req.Header.Set(k, v)
+	}
+	return c.Do(req)
+}
+
+func (c *Client) Post(ctx context.Context, url string, body io.Reader, contentType string, headers map[string]string) (*http.Response, error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, body)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("Content-Type", contentType)
+	for k, v := range headers {
+		req.Header.Set(k, v)
+	}
+	return c.Do(req)
+}
+
+// HTTPClient returns the underlying *http.Client (for passing to graphql helper etc.)
+func (c *Client) HTTPClient() *http.Client { return c.http }
@@ -0,0 +1,95 @@
+package httpclient
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"os"
+)
+
+type FlareSolverrClient struct {
+	endpoint string
+	client   *http.Client
+}
+
+func NewFlareSolverrClient() (*FlareSolverrClient, error) {
+	ep := os.Getenv("FLARESOLVERR_URL")
+	if ep == "" {
+		return nil, fmt.Errorf("FLARESOLVERR_URL not set")
+	}
+	return &FlareSolverrClient{
+		endpoint: ep,
+		client:   &http.Client{},
+	}, nil
+}
+
+type flareSolverrRequest struct {
+	Cmd        string `json:"cmd"`
+	URL        string `json:"url"`
+	MaxTimeout int    `json:"maxTimeout"`
+}
+
+type FlareSolverrResponse struct {
+	Status   string `json:"status"`
+	Solution struct {
+		Response string          `json:"response"`
+		Cookies  []fsCookie      `json:"cookies"`
+		Headers  map[string]any  `json:"headers"`
+		URL      string          `json:"url"`
+		Status   int             `json:"status"`
+	} `json:"solution"`
+}
+
+type fsCookie struct {
+	Name     string  `json:"name"`
+	Value    string  `json:"value"`
+	Domain   string  `json:"domain"`
+	Path     string  `json:"path"`
+	Expires  float64 `json:"expires"`
+	HTTPOnly bool    `json:"httpOnly"`
+	Secure   bool    `json:"secure"`
+}
+
+// Get fetches a Cloudflare-protected URL via FlareSolverr.
+// Returns rendered HTML and extracted cookies.
+func (f *FlareSolverrClient) Get(ctx context.Context, url string) (html string, cookies []*http.Cookie, err error) {
+	payload, _ := json.Marshal(flareSolverrRequest{
+		Cmd:        "request.get",
+		URL:        url,
+		MaxTimeout: 60000,
+	})
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, f.endpoint+"/v1", bytes.NewReader(payload))
+	if err != nil {
+		return "", nil, err
+	}
+	req.Header.Set("Content-Type", "application/json")
+
+	resp, err := f.client.Do(req)
+	if err != nil {
+		return "", nil, err
+	}
+	defer resp.Body.Close()
+
+	var fsResp FlareSolverrResponse
+	if err := json.NewDecoder(resp.Body).Decode(&fsResp); err != nil {
+		return "", nil, err
+	}
+	if fsResp.Status != "ok" {
+		return "", nil, fmt.Errorf("flaresolverr: status %q", fsResp.Status)
+	}
+
+	for _, c := range fsResp.Solution.Cookies {
+		cookies = append(cookies, &http.Cookie{
+			Name:     c.Name,
+			Value:    c.Value,
+			Domain:   c.Domain,
+			Path:     c.Path,
+			HttpOnly: c.HTTPOnly,
+			Secure:   c.Secure,
+		})
+	}
+	return fsResp.Solution.Response, cookies, nil
+}
@@ -0,0 +1,55 @@
+package httpclient
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+)
+
+type GraphQLRequest struct {
+	Query     string `json:"query"`
+	Variables any    `json:"variables,omitempty"`
+}
+
+type graphQLResponse[T any] struct {
+	Data   T `json:"data"`
+	Errors []struct {
+		Message string `json:"message"`
+	} `json:"errors"`
+}
+
+// Post sends a GraphQL request and unmarshals the `data` field into T.
+func Post[T any](ctx context.Context, client *http.Client, url string, req GraphQLRequest, headers map[string]string) (T, error) {
+	var zero T
+	body, err := json.Marshal(req)
+	if err != nil {
+		return zero, err
+	}
+
+	httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body))
+	if err != nil {
+		return zero, err
+	}
+	httpReq.Header.Set("Content-Type", "application/json")
+	httpReq.Header.Set("Accept", "application/json")
+	for k, v := range headers {
+		httpReq.Header.Set(k, v)
+	}
+
+	resp, err := client.Do(httpReq)
+	if err != nil {
+		return zero, err
+	}
+	defer resp.Body.Close()
+
+	var gqlResp graphQLResponse[T]
+	if err := json.NewDecoder(resp.Body).Decode(&gqlResp); err != nil {
+		return zero, err
+	}
+	if len(gqlResp.Errors) > 0 {
+		return zero, fmt.Errorf("graphql: %s", gqlResp.Errors[0].Message)
+	}
+	return gqlResp.Data, nil
+}
@@ -0,0 +1,42 @@
+package httpclient
+
+const (
+	androidUA = "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Mobile Safari/537.36"
+	desktopUA  = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
+)
+
+func AndroidUA() string { return androidUA }
+func DesktopUA() string { return desktopUA }
+
+func JSONHeaders() map[string]string {
+	return map[string]string{
+		"Content-Type": "application/json",
+		"Accept":       "application/json",
+	}
+}
+
+func FormHeaders() map[string]string {
+	return map[string]string{
+		"Content-Type": "application/x-www-form-urlencoded",
+	}
+}
+
+func WithRefererHeader(headers map[string]string, referer string) map[string]string {
+	out := clone(headers)
+	out["Referer"] = referer
+	return out
+}
+
+func WithOrigin(headers map[string]string, origin string) map[string]string {
+	out := clone(headers)
+	out["Origin"] = origin
+	return out
+}
+
+func clone(m map[string]string) map[string]string {
+	out := make(map[string]string, len(m)+1)
+	for k, v := range m {
+		out[k] = v
+	}
+	return out
+}