admin/goyomi
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add FlareSolverr proxy support with DB-backed config

Browse Source 
- Add config table for storing FlareSolverr proxy setting
- Add HTTP endpoints to get/set proxy status (GET/POST /api/config/flaresolverr)
- Refactor httpclient to support proxy mode (requests go through FlareSolverr)
- Add verbose logging for debugging
- Add POST support to FlareSolverr client

Usage:
  GET /api/config/flaresolverr - returns {flaresolverr_proxy: bool}
  POST /api/config/flaresolverr - body: {enabled: true/false}
This commit is contained in:
Achmad
2026-05-11 09:25:48 +00:00
parent 3741f4f696
commit b992080c95
15 changed files with 361 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.air.toml
+1 -1
View File
@@ -2,7 +2,7 @@ root = "."
tmp_dir = "tmp"
[build]
cmd = "go build -o ./tmp/goyomi ./cmd/server"
cmd = "go build -buildvcs=false -o ./tmp/goyomi ./cmd/server"
bin = "./tmp/goyomi"
include_ext = ["go", "tpl", "tmpl", "html", "yaml", "yml", "json"]
exclude_dir = ["tmp", "vendor", ".git"]
.gitignore
+2
View File
@@ -8,7 +8,9 @@
# Go tooling
*.test
*.out
*.log
/vendor/
tmp/
# IDE
.idea/
Makefile
+3 -2
View File
@@ -1,4 +1,5 @@
DEV_COMPOSE = docker compose -f compose-dev.yml --env-file dev.env
DEV_COMPOSE = docker compose -f compose-dev.yml
DEV_GO = /usr/local/go/bin/go
.PHONY: dev-up dev-shell dev-down dev-logs dev-build dev-run smoke-keyoapp-plain smoke-keyoapp-plain-dev
@@ -25,4 +26,4 @@ smoke-keyoapp-plain:
smoke-keyoapp-plain-dev:
$(DEV_COMPOSE) up -d postgres flaresolverr dev
$(DEV_COMPOSE) exec dev sh -lc "GOYOMI_SMOKE=1 go test -count=1 -v ./sources -run TestStandaloneKeyoappSmoke"
$(DEV_COMPOSE) exec dev sh -lc "GOYOMI_SMOKE=1 $(DEV_GO) test -count=1 -v ./sources -run TestStandaloneKeyoappSmoke"
cmd/server/main.go
+46
View File
@@ -2,12 +2,15 @@ package main
import (
"context"
"encoding/json"
"fmt"
"log"
"net/http"
"os"
"goyomi/internal/config"
"goyomi/internal/db"
"goyomi/internal/httpclient"
_ "goyomi/internal/registry"
_ "goyomi/sources/all/hentaihand"
_ "goyomi/sources/all/kemono"
@@ -40,13 +43,56 @@ func main() {
}
defer database.Close()
// Initialize config manager
config.InitConfigManager(database.Queries)
if os.Getenv("FLARESOLVERR_URL") != "" {
fsClient, err := httpclient.NewFlareSolverrClient()
if err != nil {
log.Printf("flaresolverr: client creation failed: %v", err)
} else {
httpclient.SetGlobalFlareSolverr(fsClient)
log.Printf("flaresolverr: client initialized")
}
}
mux := http.NewServeMux()
mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
fmt.Fprintln(w, "ok")
})
// Config endpoints
mux.HandleFunc("/api/config/flaresolverr", handleFlareSolverrConfig)
log.Printf("listening on %s", cfg.Addr)
if err := http.ListenAndServe(cfg.Addr, mux); err != nil {
log.Fatal(err)
}
}
func handleFlareSolverrConfig(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
switch r.Method {
case "GET":
enabled := config.IsFlareSolverrProxyEnabled()
json.NewEncoder(w).Encode(map[string]bool{"flaresolverr_proxy": enabled})
case "POST":
var req struct {
Enabled bool `json:"enabled"`
}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
http.Error(w, "invalid request", http.StatusBadRequest)
return
}
if err := config.SetFlareSolverrProxyEnabled(ctx, req.Enabled); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
json.NewEncoder(w).Encode(map[string]bool{"flaresolverr_proxy": req.Enabled})
default:
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
}
}
dev.env
+1
View File
@@ -19,3 +19,4 @@ CHAPTER_LIST_TTL_SECONDS=600
# FlareSolverr
FLARESOLVERR_LOG_LEVEL=info
FLARESOLVERR_PROXY=0
internal/config/config.go
+24 -15
View File
@@ -7,26 +7,28 @@ import (
)
type Config struct {
DatabaseURL string
FlareSolverrURL string
Addr string
MangaListTTL time.Duration
MangaDetailTTL time.Duration
DatabaseURL string
FlareSolverrURL string
FlareSolverrProxy bool
Addr string
MangaListTTL time.Duration
MangaDetailTTL time.Duration
ChapterListTTL time.Duration
DBMaxConns int
DBMinConns int
DBMaxConns int
DBMinConns int
}
func Load() Config {
return Config{
DatabaseURL: os.Getenv("DATABASE_URL"),
FlareSolverrURL: os.Getenv("FLARESOLVERR_URL"),
Addr: envStr("ADDR", ":8080"),
MangaListTTL: time.Duration(envInt("MANGA_LIST_TTL_SECONDS", 600)) * time.Second,
MangaDetailTTL: time.Duration(envInt("MANGA_DETAIL_TTL_SECONDS", 3600)) * time.Second,
ChapterListTTL: time.Duration(envInt("CHAPTER_LIST_TTL_SECONDS", 600)) * time.Second,
DBMaxConns: envInt("DB_MAX_CONNS", 10),
DBMinConns: envInt("DB_MIN_CONNS", 2),
DatabaseURL: os.Getenv("DATABASE_URL"),
FlareSolverrURL: os.Getenv("FLARESOLVERR_URL"),
FlareSolverrProxy: envBool("FLARESOLVERR_PROXY", false),
Addr: envStr("ADDR", ":8080"),
MangaListTTL: time.Duration(envInt("MANGA_LIST_TTL_SECONDS", 600)) * time.Second,
MangaDetailTTL: time.Duration(envInt("MANGA_DETAIL_TTL_SECONDS", 3600)) * time.Second,
ChapterListTTL: time.Duration(envInt("CHAPTER_LIST_TTL_SECONDS", 600)) * time.Second,
DBMaxConns: envInt("DB_MAX_CONNS", 10),
DBMinConns: envInt("DB_MIN_CONNS", 2),
}
}
@@ -45,3 +47,10 @@ func envInt(key string, def int) int {
}
return def
}
func envBool(key string, def bool) bool {
if v := os.Getenv(key); v != "" {
return v == "1" || v == "true" || v == "yes"
}
return def
}
internal/config/config_manager.go
+66
View File
@@ -0,0 +1,66 @@
package config
import (
"context"
"sync"
"goyomi/internal/db/queries"
)
type ConfigManager struct {
mu sync.RWMutex
flaresolverrProxy bool
queries *queries.Queries
}
var (
manager *ConfigManager
managerOnce sync.Once
)
func InitConfigManager(q *queries.Queries) {
managerOnce.Do(func() {
manager = &ConfigManager{queries: q}
manager.loadFromDB(context.Background())
})
}
func (m *ConfigManager) loadFromDB(ctx context.Context) {
if m.queries == nil {
return
}
val, err := m.queries.GetConfigValue(ctx, "flaresolverr_proxy")
if err == nil && val != "" {
m.flaresolverrProxy = val == "true"
}
}
func IsFlareSolverrProxyEnabled() bool {
if manager == nil {
return false
}
manager.mu.RLock()
defer manager.mu.RUnlock()
return manager.flaresolverrProxy
}
func SetFlareSolverrProxyEnabled(ctx context.Context, enabled bool) error {
if manager == nil {
return nil
}
value := "false"
if enabled {
value = "true"
}
err := manager.queries.SetConfigValue(ctx, queries.SetConfigValueParams{
Key: "flaresolverr_proxy",
Value: value,
})
if err != nil {
return err
}
manager.mu.Lock()
manager.flaresolverrProxy = enabled
manager.mu.Unlock()
return nil
}
internal/db/migrations/000002_config.down.sql
+1
View File
@@ -0,0 +1 @@
DROP TABLE IF EXISTS config;
internal/db/migrations/000002_config.up.sql
+6
View File
@@ -0,0 +1,6 @@
CREATE TABLE IF NOT EXISTS config (
key VARCHAR(256) PRIMARY KEY,
value TEXT NOT NULL
);
INSERT INTO config (key, value) VALUES ('flaresolverr_proxy', 'false') ON CONFLICT DO NOTHING;
internal/db/queries/config.sql
+6
View File
@@ -0,0 +1,6 @@
-- name: GetConfigValue :one
SELECT value FROM config WHERE key = $1;
-- name: SetConfigValue :exec
INSERT INTO config (key, value) VALUES ($1, $2)
ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value;
internal/db/queries/config.sql.go
+36
View File
@@ -0,0 +1,36 @@
// Code generated by sqlc. DO NOT EDIT.
// versions:
// sqlc v1.31.1
// source: config.sql
package queries
import (
"context"
)
const getConfigValue = `-- name: GetConfigValue :one
SELECT value FROM config WHERE key = $1
`
func (q *Queries) GetConfigValue(ctx context.Context, key string) (string, error) {
row := q.db.QueryRow(ctx, getConfigValue, key)
var value string
err := row.Scan(&value)
return value, err
}
const setConfigValue = `-- name: SetConfigValue :exec
INSERT INTO config (key, value) VALUES ($1, $2)
ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value
`
type SetConfigValueParams struct {
Key string `db:"key" json:"key"`
Value string `db:"value" json:"value"`
}
func (q *Queries) SetConfigValue(ctx context.Context, arg SetConfigValueParams) error {
_, err := q.db.Exec(ctx, setConfigValue, arg.Key, arg.Value)
return err
}
internal/db/queries/models.go
+5
View File
@@ -23,6 +23,11 @@ type Chapter struct {
PageCount int32 `db:"page_count" json:"page_count"`
}
type Config struct {
Key string `db:"key" json:"key"`
Value string `db:"value" json:"value"`
}
type Manga struct {
ID int32 `db:"id" json:"id"`
SourceID int64 `db:"source_id" json:"source_id"`
internal/httpclient/client.go
+134 -11
View File
@@ -2,23 +2,58 @@ package httpclient
import (
"context"
"fmt"
"io"
"log"
"net/http"
"net/http/cookiejar"
"strconv"
"strings"
"sync"
"time"
"goyomi/internal/config"
"golang.org/x/time/rate"
)
const defaultUserAgent = "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Mobile Safari/537.36"
var (
globalFSClient *FlareSolverrClient
globalProxy bool
verboseLog bool
)
func SetGlobalFlareSolverr(client *FlareSolverrClient) {
globalFSClient = client
}
func SetGlobalProxy(enabled bool) {
globalProxy = enabled
}
func SetVerboseLog(enabled bool) {
verboseLog = enabled
}
func ProxyEnabled() bool {
if globalProxy {
return true
}
if config.IsFlareSolverrProxyEnabled() {
return true
}
return false
}
type Client struct {
http *http.Client
rateLimit float64
burst int
referer string
http *http.Client
fsClient *FlareSolverrClient
proxy bool
rateLimit float64
burst int
referer string
verboseLog bool
mu sync.Mutex
limiters map[string]*rate.Limiter
@@ -41,16 +76,28 @@ func WithReferer(referer string) Option {
return func(c *Client) { c.referer = referer }
}
func WithFlareSolverr(fs *FlareSolverrClient) Option {
return func(c *Client) { c.fsClient = fs }
}
func WithProxy(enabled bool) Option {
return func(c *Client) { c.proxy = enabled }
}
func WithVerboseLog(enabled bool) Option {
return func(c *Client) { c.verboseLog = enabled }
}
func NewClient(opts ...Option) *Client {
jar, _ := cookiejar.New(nil)
c := &Client{
http: &http.Client{
Timeout: 30 * time.Second,
Jar: jar,
},
rateLimit: 1,
burst: 1,
limiters: map[string]*rate.Limiter{},
http: &http.Client{Timeout: 30 * time.Second, Jar: jar},
fsClient: globalFSClient,
proxy: globalProxy,
rateLimit: 1,
burst: 1,
limiters: map[string]*rate.Limiter{},
verboseLog: verboseLog,
}
for _, o := range opts {
o(c)
@@ -70,6 +117,13 @@ func (c *Client) limiter(host string) *rate.Limiter {
}
func (c *Client) Do(req *http.Request) (*http.Response, error) {
if (c.proxy || ProxyEnabled()) && c.fsClient != nil {
return c.doViaFlareSolverr(req)
}
return c.doDirect(req)
}
func (c *Client) doDirect(req *http.Request) (*http.Response, error) {
if err := c.limiter(req.URL.Host).Wait(req.Context()); err != nil {
return nil, err
}
@@ -80,12 +134,19 @@ func (c *Client) Do(req *http.Request) (*http.Response, error) {
req.Header.Set("User-Agent", defaultUserAgent)
}
if c.verboseLog {
log.Printf("[httpclient] DIRECT GET %s", req.URL.String())
}
const maxRetries = 3
for attempt := 0; attempt <= maxRetries; attempt++ {
resp, err := c.http.Do(req)
if err != nil {
return nil, err
}
if c.verboseLog {
log.Printf("[httpclient] DIRECT RESPONSE %s status=%d", req.URL.String(), resp.StatusCode)
}
if resp.StatusCode != http.StatusTooManyRequests {
return resp, nil
}
@@ -103,6 +164,68 @@ func (c *Client) Do(req *http.Request) (*http.Response, error) {
panic("unreachable")
}
func (c *Client) doViaFlareSolverr(req *http.Request) (*http.Response, error) {
if err := c.limiter(req.URL.Host).Wait(req.Context()); err != nil {
return nil, err
}
url := req.URL.String()
if c.verboseLog {
log.Printf("[httpclient] FLARESOLVERR %s %s", req.Method, url)
}
var html string
var cookies []*http.Cookie
var err error
if req.Method == http.MethodGet {
html, cookies, err = c.fsClient.Get(req.Context(), url)
} else if req.Method == http.MethodPost {
var bodyStr string
if req.Body != nil {
bodyBytes, _ := io.ReadAll(req.Body)
bodyStr = string(bodyBytes)
if c.verboseLog {
log.Printf("[httpclient] FLARESOLVERR POST BODY: %s", bodyStr)
}
}
html, cookies, err = c.fsClient.Post(req.Context(), url, bodyStr)
} else {
return c.doDirect(req)
}
if err != nil {
return nil, fmt.Errorf("flaresolverr: %w", err)
}
if c.verboseLog {
log.Printf("[httpclient] FLARESOLVERR RESPONSE length=%d cookies=%d", len(html), len(cookies))
if len(html) > 0 {
truncate := html
if len(html) > 500 {
truncate = html[:500]
}
log.Printf("[httpclient] FLARESOLVERR RESPONSE HTML (first 500 chars): %s", truncate)
}
}
resp := &http.Response{
StatusCode: http.StatusOK,
Header: http.Header{},
Body: io.NopCloser(strings.NewReader(html)),
}
resp.Header.Set("Content-Type", "text/html; charset=utf-8")
if c.http.Jar != nil {
for _, cookie := range cookies {
c.http.Jar.SetCookies(req.URL, []*http.Cookie{cookie})
}
}
return resp, nil
}
func retryAfter(resp *http.Response) time.Duration {
ra := resp.Header.Get("Retry-After")
if ra == "" {
internal/httpclient/flaresolverr.go
+30 -6
View File
@@ -55,19 +55,43 @@ type fsCookie struct {
// Get fetches a Cloudflare-protected URL via FlareSolverr.
// Returns rendered HTML and extracted cookies.
func (f *FlareSolverrClient) Get(ctx context.Context, url string) (html string, cookies []*http.Cookie, err error) {
payload, _ := json.Marshal(flareSolverrRequest{
Cmd: "request.get",
return f.request(ctx, "request.get", url, "", nil)
}
func (f *FlareSolverrClient) Post(ctx context.Context, url string, body string) (html string, cookies []*http.Cookie, err error) {
return f.request(ctx, "request.post", url, body, map[string]string{"Content-Type": "application/x-www-form-urlencoded"})
}
func (f *FlareSolverrClient) request(ctx context.Context, cmd, url, body string, headers map[string]string) (html string, cookies []*http.Cookie, err error) {
type fsRequest struct {
Cmd string `json:"cmd"`
URL string `json:"url"`
PostData string `json:"postData,omitempty"`
Headers map[string]string `json:"headers,omitempty"`
MaxTimeout int `json:"maxTimeout"`
}
req := fsRequest{
Cmd: cmd,
URL: url,
MaxTimeout: 60000,
})
}
if body != "" {
req.PostData = body
}
if headers != nil {
req.Headers = headers
}
req, err := http.NewRequestWithContext(ctx, http.MethodPost, f.endpoint+"/v1", bytes.NewReader(payload))
payload, _ := json.Marshal(req)
httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, f.endpoint+"/v1", bytes.NewReader(payload))
if err != nil {
return "", nil, err
}
req.Header.Set("Content-Type", "application/json")
httpReq.Header.Set("Content-Type", "application/json")
resp, err := f.client.Do(req)
resp, err := f.client.Do(httpReq)
if err != nil {
return "", nil, err
}
tmp/build-errors.log
-1
View File
@@ -1 +0,0 @@
exit status 1