diff --git a/.air.toml b/.air.toml index 504c64c..c5e37f5 100755 --- a/.air.toml +++ b/.air.toml @@ -2,7 +2,7 @@ root = "." tmp_dir = "tmp" [build] -cmd = "go build -o ./tmp/goyomi ./cmd/server" +cmd = "go build -buildvcs=false -o ./tmp/goyomi ./cmd/server" bin = "./tmp/goyomi" include_ext = ["go", "tpl", "tmpl", "html", "yaml", "yml", "json"] exclude_dir = ["tmp", "vendor", ".git"] diff --git a/.gitignore b/.gitignore index dcb6f1d..d9433b6 100755 --- a/.gitignore +++ b/.gitignore @@ -8,7 +8,9 @@ # Go tooling *.test *.out +*.log /vendor/ +tmp/ # IDE .idea/ diff --git a/Makefile b/Makefile index 3324cf0..c776db4 100755 --- a/Makefile +++ b/Makefile @@ -1,4 +1,5 @@ -DEV_COMPOSE = docker compose -f compose-dev.yml --env-file dev.env +DEV_COMPOSE = docker compose -f compose-dev.yml +DEV_GO = /usr/local/go/bin/go .PHONY: dev-up dev-shell dev-down dev-logs dev-build dev-run smoke-keyoapp-plain smoke-keyoapp-plain-dev @@ -25,4 +26,4 @@ smoke-keyoapp-plain: smoke-keyoapp-plain-dev: $(DEV_COMPOSE) up -d postgres flaresolverr dev - $(DEV_COMPOSE) exec dev sh -lc "GOYOMI_SMOKE=1 go test -count=1 -v ./sources -run TestStandaloneKeyoappSmoke" + $(DEV_COMPOSE) exec dev sh -lc "GOYOMI_SMOKE=1 $(DEV_GO) test -count=1 -v ./sources -run TestStandaloneKeyoappSmoke" \ No newline at end of file diff --git a/cmd/server/main.go b/cmd/server/main.go index 13e3ce4..9f0dc30 100755 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -2,12 +2,15 @@ package main import ( "context" + "encoding/json" "fmt" "log" "net/http" + "os" "goyomi/internal/config" "goyomi/internal/db" + "goyomi/internal/httpclient" _ "goyomi/internal/registry" _ "goyomi/sources/all/hentaihand" _ "goyomi/sources/all/kemono" @@ -40,13 +43,56 @@ func main() { } defer database.Close() + // Initialize config manager + config.InitConfigManager(database.Queries) + + if os.Getenv("FLARESOLVERR_URL") != "" { + fsClient, err := httpclient.NewFlareSolverrClient() + if err != nil { + log.Printf("flaresolverr: client creation failed: %v", err) + } else { + httpclient.SetGlobalFlareSolverr(fsClient) + log.Printf("flaresolverr: client initialized") + } + } + mux := http.NewServeMux() mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) { fmt.Fprintln(w, "ok") }) + // Config endpoints + mux.HandleFunc("/api/config/flaresolverr", handleFlareSolverrConfig) + log.Printf("listening on %s", cfg.Addr) if err := http.ListenAndServe(cfg.Addr, mux); err != nil { log.Fatal(err) } } + +func handleFlareSolverrConfig(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + switch r.Method { + case "GET": + enabled := config.IsFlareSolverrProxyEnabled() + json.NewEncoder(w).Encode(map[string]bool{"flaresolverr_proxy": enabled}) + + case "POST": + var req struct { + Enabled bool `json:"enabled"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + http.Error(w, "invalid request", http.StatusBadRequest) + return + } + if err := config.SetFlareSolverrProxyEnabled(ctx, req.Enabled); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + json.NewEncoder(w).Encode(map[string]bool{"flaresolverr_proxy": req.Enabled}) + + default: + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + } +} \ No newline at end of file diff --git a/dev.env b/dev.env index c9f121a..db14641 100755 --- a/dev.env +++ b/dev.env @@ -19,3 +19,4 @@ CHAPTER_LIST_TTL_SECONDS=600 # FlareSolverr FLARESOLVERR_LOG_LEVEL=info +FLARESOLVERR_PROXY=0 diff --git a/internal/config/config.go b/internal/config/config.go index bdf96d4..78bd4b5 100755 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -7,26 +7,28 @@ import ( ) type Config struct { - DatabaseURL string - FlareSolverrURL string - Addr string - MangaListTTL time.Duration - MangaDetailTTL time.Duration + DatabaseURL string + FlareSolverrURL string + FlareSolverrProxy bool + Addr string + MangaListTTL time.Duration + MangaDetailTTL time.Duration ChapterListTTL time.Duration - DBMaxConns int - DBMinConns int + DBMaxConns int + DBMinConns int } func Load() Config { return Config{ - DatabaseURL: os.Getenv("DATABASE_URL"), - FlareSolverrURL: os.Getenv("FLARESOLVERR_URL"), - Addr: envStr("ADDR", ":8080"), - MangaListTTL: time.Duration(envInt("MANGA_LIST_TTL_SECONDS", 600)) * time.Second, - MangaDetailTTL: time.Duration(envInt("MANGA_DETAIL_TTL_SECONDS", 3600)) * time.Second, - ChapterListTTL: time.Duration(envInt("CHAPTER_LIST_TTL_SECONDS", 600)) * time.Second, - DBMaxConns: envInt("DB_MAX_CONNS", 10), - DBMinConns: envInt("DB_MIN_CONNS", 2), + DatabaseURL: os.Getenv("DATABASE_URL"), + FlareSolverrURL: os.Getenv("FLARESOLVERR_URL"), + FlareSolverrProxy: envBool("FLARESOLVERR_PROXY", false), + Addr: envStr("ADDR", ":8080"), + MangaListTTL: time.Duration(envInt("MANGA_LIST_TTL_SECONDS", 600)) * time.Second, + MangaDetailTTL: time.Duration(envInt("MANGA_DETAIL_TTL_SECONDS", 3600)) * time.Second, + ChapterListTTL: time.Duration(envInt("CHAPTER_LIST_TTL_SECONDS", 600)) * time.Second, + DBMaxConns: envInt("DB_MAX_CONNS", 10), + DBMinConns: envInt("DB_MIN_CONNS", 2), } } @@ -45,3 +47,10 @@ func envInt(key string, def int) int { } return def } + +func envBool(key string, def bool) bool { + if v := os.Getenv(key); v != "" { + return v == "1" || v == "true" || v == "yes" + } + return def +} diff --git a/internal/config/config_manager.go b/internal/config/config_manager.go new file mode 100644 index 0000000..675ae49 --- /dev/null +++ b/internal/config/config_manager.go @@ -0,0 +1,66 @@ +package config + +import ( + "context" + "sync" + + "goyomi/internal/db/queries" +) + +type ConfigManager struct { + mu sync.RWMutex + flaresolverrProxy bool + queries *queries.Queries +} + +var ( + manager *ConfigManager + managerOnce sync.Once +) + +func InitConfigManager(q *queries.Queries) { + managerOnce.Do(func() { + manager = &ConfigManager{queries: q} + manager.loadFromDB(context.Background()) + }) +} + +func (m *ConfigManager) loadFromDB(ctx context.Context) { + if m.queries == nil { + return + } + val, err := m.queries.GetConfigValue(ctx, "flaresolverr_proxy") + if err == nil && val != "" { + m.flaresolverrProxy = val == "true" + } +} + +func IsFlareSolverrProxyEnabled() bool { + if manager == nil { + return false + } + manager.mu.RLock() + defer manager.mu.RUnlock() + return manager.flaresolverrProxy +} + +func SetFlareSolverrProxyEnabled(ctx context.Context, enabled bool) error { + if manager == nil { + return nil + } + value := "false" + if enabled { + value = "true" + } + err := manager.queries.SetConfigValue(ctx, queries.SetConfigValueParams{ + Key: "flaresolverr_proxy", + Value: value, + }) + if err != nil { + return err + } + manager.mu.Lock() + manager.flaresolverrProxy = enabled + manager.mu.Unlock() + return nil +} \ No newline at end of file diff --git a/internal/db/migrations/000002_config.down.sql b/internal/db/migrations/000002_config.down.sql new file mode 100644 index 0000000..a242012 --- /dev/null +++ b/internal/db/migrations/000002_config.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS config; \ No newline at end of file diff --git a/internal/db/migrations/000002_config.up.sql b/internal/db/migrations/000002_config.up.sql new file mode 100644 index 0000000..ead3806 --- /dev/null +++ b/internal/db/migrations/000002_config.up.sql @@ -0,0 +1,6 @@ +CREATE TABLE IF NOT EXISTS config ( + key VARCHAR(256) PRIMARY KEY, + value TEXT NOT NULL +); + +INSERT INTO config (key, value) VALUES ('flaresolverr_proxy', 'false') ON CONFLICT DO NOTHING; \ No newline at end of file diff --git a/internal/db/queries/config.sql b/internal/db/queries/config.sql new file mode 100644 index 0000000..0b4f2b6 --- /dev/null +++ b/internal/db/queries/config.sql @@ -0,0 +1,6 @@ +-- name: GetConfigValue :one +SELECT value FROM config WHERE key = $1; + +-- name: SetConfigValue :exec +INSERT INTO config (key, value) VALUES ($1, $2) +ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value; \ No newline at end of file diff --git a/internal/db/queries/config.sql.go b/internal/db/queries/config.sql.go new file mode 100644 index 0000000..d4bcf3a --- /dev/null +++ b/internal/db/queries/config.sql.go @@ -0,0 +1,36 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: config.sql + +package queries + +import ( + "context" +) + +const getConfigValue = `-- name: GetConfigValue :one +SELECT value FROM config WHERE key = $1 +` + +func (q *Queries) GetConfigValue(ctx context.Context, key string) (string, error) { + row := q.db.QueryRow(ctx, getConfigValue, key) + var value string + err := row.Scan(&value) + return value, err +} + +const setConfigValue = `-- name: SetConfigValue :exec +INSERT INTO config (key, value) VALUES ($1, $2) +ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value +` + +type SetConfigValueParams struct { + Key string `db:"key" json:"key"` + Value string `db:"value" json:"value"` +} + +func (q *Queries) SetConfigValue(ctx context.Context, arg SetConfigValueParams) error { + _, err := q.db.Exec(ctx, setConfigValue, arg.Key, arg.Value) + return err +} diff --git a/internal/db/queries/models.go b/internal/db/queries/models.go index 2f9482f..4879c2e 100755 --- a/internal/db/queries/models.go +++ b/internal/db/queries/models.go @@ -23,6 +23,11 @@ type Chapter struct { PageCount int32 `db:"page_count" json:"page_count"` } +type Config struct { + Key string `db:"key" json:"key"` + Value string `db:"value" json:"value"` +} + type Manga struct { ID int32 `db:"id" json:"id"` SourceID int64 `db:"source_id" json:"source_id"` diff --git a/internal/httpclient/client.go b/internal/httpclient/client.go index ba1791b..c24a7a5 100755 --- a/internal/httpclient/client.go +++ b/internal/httpclient/client.go @@ -2,23 +2,58 @@ package httpclient import ( "context" + "fmt" "io" + "log" "net/http" "net/http/cookiejar" "strconv" + "strings" "sync" "time" + "goyomi/internal/config" "golang.org/x/time/rate" ) const defaultUserAgent = "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Mobile Safari/537.36" +var ( + globalFSClient *FlareSolverrClient + globalProxy bool + verboseLog bool +) + +func SetGlobalFlareSolverr(client *FlareSolverrClient) { + globalFSClient = client +} + +func SetGlobalProxy(enabled bool) { + globalProxy = enabled +} + +func SetVerboseLog(enabled bool) { + verboseLog = enabled +} + +func ProxyEnabled() bool { + if globalProxy { + return true + } + if config.IsFlareSolverrProxyEnabled() { + return true + } + return false +} + type Client struct { - http *http.Client - rateLimit float64 - burst int - referer string + http *http.Client + fsClient *FlareSolverrClient + proxy bool + rateLimit float64 + burst int + referer string + verboseLog bool mu sync.Mutex limiters map[string]*rate.Limiter @@ -41,16 +76,28 @@ func WithReferer(referer string) Option { return func(c *Client) { c.referer = referer } } +func WithFlareSolverr(fs *FlareSolverrClient) Option { + return func(c *Client) { c.fsClient = fs } +} + +func WithProxy(enabled bool) Option { + return func(c *Client) { c.proxy = enabled } +} + +func WithVerboseLog(enabled bool) Option { + return func(c *Client) { c.verboseLog = enabled } +} + func NewClient(opts ...Option) *Client { jar, _ := cookiejar.New(nil) c := &Client{ - http: &http.Client{ - Timeout: 30 * time.Second, - Jar: jar, - }, - rateLimit: 1, - burst: 1, - limiters: map[string]*rate.Limiter{}, + http: &http.Client{Timeout: 30 * time.Second, Jar: jar}, + fsClient: globalFSClient, + proxy: globalProxy, + rateLimit: 1, + burst: 1, + limiters: map[string]*rate.Limiter{}, + verboseLog: verboseLog, } for _, o := range opts { o(c) @@ -70,6 +117,13 @@ func (c *Client) limiter(host string) *rate.Limiter { } func (c *Client) Do(req *http.Request) (*http.Response, error) { + if (c.proxy || ProxyEnabled()) && c.fsClient != nil { + return c.doViaFlareSolverr(req) + } + return c.doDirect(req) +} + +func (c *Client) doDirect(req *http.Request) (*http.Response, error) { if err := c.limiter(req.URL.Host).Wait(req.Context()); err != nil { return nil, err } @@ -80,12 +134,19 @@ func (c *Client) Do(req *http.Request) (*http.Response, error) { req.Header.Set("User-Agent", defaultUserAgent) } + if c.verboseLog { + log.Printf("[httpclient] DIRECT GET %s", req.URL.String()) + } + const maxRetries = 3 for attempt := 0; attempt <= maxRetries; attempt++ { resp, err := c.http.Do(req) if err != nil { return nil, err } + if c.verboseLog { + log.Printf("[httpclient] DIRECT RESPONSE %s status=%d", req.URL.String(), resp.StatusCode) + } if resp.StatusCode != http.StatusTooManyRequests { return resp, nil } @@ -103,6 +164,68 @@ func (c *Client) Do(req *http.Request) (*http.Response, error) { panic("unreachable") } +func (c *Client) doViaFlareSolverr(req *http.Request) (*http.Response, error) { + if err := c.limiter(req.URL.Host).Wait(req.Context()); err != nil { + return nil, err + } + + url := req.URL.String() + + if c.verboseLog { + log.Printf("[httpclient] FLARESOLVERR %s %s", req.Method, url) + } + + var html string + var cookies []*http.Cookie + var err error + + if req.Method == http.MethodGet { + html, cookies, err = c.fsClient.Get(req.Context(), url) + } else if req.Method == http.MethodPost { + var bodyStr string + if req.Body != nil { + bodyBytes, _ := io.ReadAll(req.Body) + bodyStr = string(bodyBytes) + if c.verboseLog { + log.Printf("[httpclient] FLARESOLVERR POST BODY: %s", bodyStr) + } + } + html, cookies, err = c.fsClient.Post(req.Context(), url, bodyStr) + } else { + return c.doDirect(req) + } + + if err != nil { + return nil, fmt.Errorf("flaresolverr: %w", err) + } + + if c.verboseLog { + log.Printf("[httpclient] FLARESOLVERR RESPONSE length=%d cookies=%d", len(html), len(cookies)) + if len(html) > 0 { + truncate := html + if len(html) > 500 { + truncate = html[:500] + } + log.Printf("[httpclient] FLARESOLVERR RESPONSE HTML (first 500 chars): %s", truncate) + } + } + + resp := &http.Response{ + StatusCode: http.StatusOK, + Header: http.Header{}, + Body: io.NopCloser(strings.NewReader(html)), + } + resp.Header.Set("Content-Type", "text/html; charset=utf-8") + + if c.http.Jar != nil { + for _, cookie := range cookies { + c.http.Jar.SetCookies(req.URL, []*http.Cookie{cookie}) + } + } + + return resp, nil +} + func retryAfter(resp *http.Response) time.Duration { ra := resp.Header.Get("Retry-After") if ra == "" { diff --git a/internal/httpclient/flaresolverr.go b/internal/httpclient/flaresolverr.go index 69a531e..bbc286b 100755 --- a/internal/httpclient/flaresolverr.go +++ b/internal/httpclient/flaresolverr.go @@ -55,19 +55,43 @@ type fsCookie struct { // Get fetches a Cloudflare-protected URL via FlareSolverr. // Returns rendered HTML and extracted cookies. func (f *FlareSolverrClient) Get(ctx context.Context, url string) (html string, cookies []*http.Cookie, err error) { - payload, _ := json.Marshal(flareSolverrRequest{ - Cmd: "request.get", + return f.request(ctx, "request.get", url, "", nil) +} + +func (f *FlareSolverrClient) Post(ctx context.Context, url string, body string) (html string, cookies []*http.Cookie, err error) { + return f.request(ctx, "request.post", url, body, map[string]string{"Content-Type": "application/x-www-form-urlencoded"}) +} + +func (f *FlareSolverrClient) request(ctx context.Context, cmd, url, body string, headers map[string]string) (html string, cookies []*http.Cookie, err error) { + type fsRequest struct { + Cmd string `json:"cmd"` + URL string `json:"url"` + PostData string `json:"postData,omitempty"` + Headers map[string]string `json:"headers,omitempty"` + MaxTimeout int `json:"maxTimeout"` + } + + req := fsRequest{ + Cmd: cmd, URL: url, MaxTimeout: 60000, - }) + } + if body != "" { + req.PostData = body + } + if headers != nil { + req.Headers = headers + } - req, err := http.NewRequestWithContext(ctx, http.MethodPost, f.endpoint+"/v1", bytes.NewReader(payload)) + payload, _ := json.Marshal(req) + + httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, f.endpoint+"/v1", bytes.NewReader(payload)) if err != nil { return "", nil, err } - req.Header.Set("Content-Type", "application/json") + httpReq.Header.Set("Content-Type", "application/json") - resp, err := f.client.Do(req) + resp, err := f.client.Do(httpReq) if err != nil { return "", nil, err } diff --git a/tmp/build-errors.log b/tmp/build-errors.log deleted file mode 100755 index 05e5985..0000000 --- a/tmp/build-errors.log +++ /dev/null @@ -1 +0,0 @@ -exit status 1 \ No newline at end of file