admin/goyomi
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add FlareSolverr proxy support with DB-backed config

Browse Source 
- Add config table for storing FlareSolverr proxy setting
- Add HTTP endpoints to get/set proxy status (GET/POST /api/config/flaresolverr)
- Refactor httpclient to support proxy mode (requests go through FlareSolverr)
- Add verbose logging for debugging
- Add POST support to FlareSolverr client

Usage:
  GET /api/config/flaresolverr - returns {flaresolverr_proxy: bool}
  POST /api/config/flaresolverr - body: {enabled: true/false}
This commit is contained in:
Achmad
2026-05-11 09:25:48 +00:00
parent 3741f4f696
commit b992080c95
15 changed files with 361 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.air.toml
+1 -1
View File
@@ -2,7 +2,7 @@ root = "."
tmp_dir = "tmp"
[build]
cmd = "go build -o ./tmp/goyomi ./cmd/server"
cmd = "go build -buildvcs=false -o ./tmp/goyomi ./cmd/server"
bin = "./tmp/goyomi"
include_ext = ["go", "tpl", "tmpl", "html", "yaml", "yml", "json"]
exclude_dir = ["tmp", "vendor", ".git"]
.gitignore
+2
View File
@@ -8,7 +8,9 @@
# Go tooling
*.test
*.out
*.log
/vendor/
tmp/
# IDE
.idea/
Makefile
+3 -2
View File
@@ -1,4 +1,5 @@
DEV_COMPOSE = docker compose -f compose-dev.yml --env-file dev.env
DEV_COMPOSE = docker compose -f compose-dev.yml
DEV_GO = /usr/local/go/bin/go
.PHONY: dev-up dev-shell dev-down dev-logs dev-build dev-run smoke-keyoapp-plain smoke-keyoapp-plain-dev
@@ -25,4 +26,4 @@ smoke-keyoapp-plain:
smoke-keyoapp-plain-dev:
$(DEV_COMPOSE) up -d postgres flaresolverr dev
$(DEV_COMPOSE) exec dev sh -lc "GOYOMI_SMOKE=1 go test -count=1 -v ./sources -run TestStandaloneKeyoappSmoke"
$(DEV_COMPOSE) exec dev sh -lc "GOYOMI_SMOKE=1 $(DEV_GO) test -count=1 -v ./sources -run TestStandaloneKeyoappSmoke"
cmd/server/main.go
+46
View File
@@ -2,12 +2,15 @@ package main
import (
"context"
"encoding/json"
"fmt"
"log"
"net/http"
"os"
"goyomi/internal/config"
"goyomi/internal/db"
"goyomi/internal/httpclient"
_ "goyomi/internal/registry"
_ "goyomi/sources/all/hentaihand"
_ "goyomi/sources/all/kemono"
@@ -40,13 +43,56 @@ func main() {
}
defer database.Close()
// Initialize config manager
config.InitConfigManager(database.Queries)
if os.Getenv("FLARESOLVERR_URL") != "" {
fsClient, err := httpclient.NewFlareSolverrClient()
if err != nil {
log.Printf("flaresolverr: client creation failed: %v", err)
} else {
httpclient.SetGlobalFlareSolverr(fsClient)
log.Printf("flaresolverr: client initialized")
}
}
mux := http.NewServeMux()
mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
fmt.Fprintln(w, "ok")
})
// Config endpoints
mux.HandleFunc("/api/config/flaresolverr", handleFlareSolverrConfig)
log.Printf("listening on %s", cfg.Addr)
if err := http.ListenAndServe(cfg.Addr, mux); err != nil {
log.Fatal(err)
}
}
func handleFlareSolverrConfig(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
switch r.Method {
case "GET":
enabled := config.IsFlareSolverrProxyEnabled()
json.NewEncoder(w).Encode(map[string]bool{"flaresolverr_proxy": enabled})
case "POST":
var req struct {
Enabled bool `json:"enabled"`
}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
http.Error(w, "invalid request", http.StatusBadRequest)
return
}
if err := config.SetFlareSolverrProxyEnabled(ctx, req.Enabled); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
json.NewEncoder(w).Encode(map[string]bool{"flaresolverr_proxy": req.Enabled})
default:
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
}
}
dev.env
+1
View File
@@ -19,3 +19,4 @@ CHAPTER_LIST_TTL_SECONDS=600
# FlareSolverr
FLARESOLVERR_LOG_LEVEL=info
FLARESOLVERR_PROXY=0
internal/config/config.go
+9
View File
@@ -9,6 +9,7 @@ import (
type Config struct {
DatabaseURL string
FlareSolverrURL string
FlareSolverrProxy bool
Addr string
MangaListTTL time.Duration
MangaDetailTTL time.Duration
@@ -21,6 +22,7 @@ func Load() Config {
return Config{
DatabaseURL: os.Getenv("DATABASE_URL"),
FlareSolverrURL: os.Getenv("FLARESOLVERR_URL"),
FlareSolverrProxy: envBool("FLARESOLVERR_PROXY", false),
Addr: envStr("ADDR", ":8080"),
MangaListTTL: time.Duration(envInt("MANGA_LIST_TTL_SECONDS", 600)) * time.Second,
MangaDetailTTL: time.Duration(envInt("MANGA_DETAIL_TTL_SECONDS", 3600)) * time.Second,
@@ -45,3 +47,10 @@ func envInt(key string, def int) int {
}
return def
}
func envBool(key string, def bool) bool {
if v := os.Getenv(key); v != "" {
return v == "1" || v == "true" || v == "yes"
}
return def
}
internal/config/config_manager.go
+66
View File
@@ -0,0 +1,66 @@
package config
import (
"context"
"sync"
"goyomi/internal/db/queries"
)
type ConfigManager struct {
mu sync.RWMutex
flaresolverrProxy bool
queries *queries.Queries
}
var (
manager *ConfigManager
managerOnce sync.Once
)
func InitConfigManager(q *queries.Queries) {
managerOnce.Do(func() {
manager = &ConfigManager{queries: q}
manager.loadFromDB(context.Background())
})
}
func (m *ConfigManager) loadFromDB(ctx context.Context) {
if m.queries == nil {
return
}
val, err := m.queries.GetConfigValue(ctx, "flaresolverr_proxy")
if err == nil && val != "" {
m.flaresolverrProxy = val == "true"
}
}
func IsFlareSolverrProxyEnabled() bool {
if manager == nil {
return false
}
manager.mu.RLock()
defer manager.mu.RUnlock()
return manager.flaresolverrProxy
}
func SetFlareSolverrProxyEnabled(ctx context.Context, enabled bool) error {
if manager == nil {
return nil
}
value := "false"
if enabled {
value = "true"
}
err := manager.queries.SetConfigValue(ctx, queries.SetConfigValueParams{
Key: "flaresolverr_proxy",
Value: value,
})
if err != nil {
return err
}
manager.mu.Lock()
manager.flaresolverrProxy = enabled
manager.mu.Unlock()
return nil
}
internal/db/migrations/000002_config.down.sql
+1
View File
@@ -0,0 +1 @@
DROP TABLE IF EXISTS config;
internal/db/migrations/000002_config.up.sql
+6
View File
@@ -0,0 +1,6 @@
CREATE TABLE IF NOT EXISTS config (
key VARCHAR(256) PRIMARY KEY,
value TEXT NOT NULL
);
INSERT INTO config (key, value) VALUES ('flaresolverr_proxy', 'false') ON CONFLICT DO NOTHING;
internal/db/queries/config.sql
+6
View File
@@ -0,0 +1,6 @@
-- name: GetConfigValue :one
SELECT value FROM config WHERE key = $1;
-- name: SetConfigValue :exec
INSERT INTO config (key, value) VALUES ($1, $2)
ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value;
internal/db/queries/config.sql.go
+36
View File
@@ -0,0 +1,36 @@
// Code generated by sqlc. DO NOT EDIT.
// versions:
// sqlc v1.31.1
// source: config.sql
package queries
import (
"context"
)
const getConfigValue = `-- name: GetConfigValue :one
SELECT value FROM config WHERE key = $1
`
func (q *Queries) GetConfigValue(ctx context.Context, key string) (string, error) {
row := q.db.QueryRow(ctx, getConfigValue, key)
var value string
err := row.Scan(&value)
return value, err
}
const setConfigValue = `-- name: SetConfigValue :exec
INSERT INTO config (key, value) VALUES ($1, $2)
ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value
`
type SetConfigValueParams struct {
Key string `db:"key" json:"key"`
Value string `db:"value" json:"value"`
}
func (q *Queries) SetConfigValue(ctx context.Context, arg SetConfigValueParams) error {
_, err := q.db.Exec(ctx, setConfigValue, arg.Key, arg.Value)
return err
}
internal/db/queries/models.go
+5
View File
@@ -23,6 +23,11 @@ type Chapter struct {
PageCount int32 `db:"page_count" json:"page_count"`
}
type Config struct {
Key string `db:"key" json:"key"`
Value string `db:"value" json:"value"`
}
type Manga struct {
ID int32 `db:"id" json:"id"`
SourceID int64 `db:"source_id" json:"source_id"`
internal/httpclient/client.go
+127 -4
View File
@@ -2,23 +2,58 @@ package httpclient
import (
"context"
"fmt"
"io"
"log"
"net/http"
"net/http/cookiejar"
"strconv"
"strings"
"sync"
"time"
"goyomi/internal/config"
"golang.org/x/time/rate"
)
const defaultUserAgent = "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Mobile Safari/537.36"
var (
globalFSClient *FlareSolverrClient
globalProxy bool
verboseLog bool
)
func SetGlobalFlareSolverr(client *FlareSolverrClient) {
globalFSClient = client
}
func SetGlobalProxy(enabled bool) {
globalProxy = enabled
}
func SetVerboseLog(enabled bool) {
verboseLog = enabled
}
func ProxyEnabled() bool {
if globalProxy {
return true
}
if config.IsFlareSolverrProxyEnabled() {
return true
}
return false
}
type Client struct {
http *http.Client
fsClient *FlareSolverrClient
proxy bool
rateLimit float64
burst int
referer string
verboseLog bool
mu sync.Mutex
limiters map[string]*rate.Limiter
@@ -41,16 +76,28 @@ func WithReferer(referer string) Option {
return func(c *Client) { c.referer = referer }
}
func WithFlareSolverr(fs *FlareSolverrClient) Option {
return func(c *Client) { c.fsClient = fs }
}
func WithProxy(enabled bool) Option {
return func(c *Client) { c.proxy = enabled }
}
func WithVerboseLog(enabled bool) Option {
return func(c *Client) { c.verboseLog = enabled }
}
func NewClient(opts ...Option) *Client {
jar, _ := cookiejar.New(nil)
c := &Client{
http: &http.Client{
Timeout: 30 * time.Second,
Jar: jar,
},
http: &http.Client{Timeout: 30 * time.Second, Jar: jar},
fsClient: globalFSClient,
proxy: globalProxy,
rateLimit: 1,
burst: 1,
limiters: map[string]*rate.Limiter{},
verboseLog: verboseLog,
}
for _, o := range opts {
o(c)
@@ -70,6 +117,13 @@ func (c *Client) limiter(host string) *rate.Limiter {
}
func (c *Client) Do(req *http.Request) (*http.Response, error) {
if (c.proxy || ProxyEnabled()) && c.fsClient != nil {
return c.doViaFlareSolverr(req)
}
return c.doDirect(req)
}
func (c *Client) doDirect(req *http.Request) (*http.Response, error) {
if err := c.limiter(req.URL.Host).Wait(req.Context()); err != nil {
return nil, err
}
@@ -80,12 +134,19 @@ func (c *Client) Do(req *http.Request) (*http.Response, error) {
req.Header.Set("User-Agent", defaultUserAgent)
}
if c.verboseLog {
log.Printf("[httpclient] DIRECT GET %s", req.URL.String())
}
const maxRetries = 3
for attempt := 0; attempt <= maxRetries; attempt++ {
resp, err := c.http.Do(req)
if err != nil {
return nil, err
}
if c.verboseLog {
log.Printf("[httpclient] DIRECT RESPONSE %s status=%d", req.URL.String(), resp.StatusCode)
}
if resp.StatusCode != http.StatusTooManyRequests {
return resp, nil
}
@@ -103,6 +164,68 @@ func (c *Client) Do(req *http.Request) (*http.Response, error) {
panic("unreachable")
}
func (c *Client) doViaFlareSolverr(req *http.Request) (*http.Response, error) {
if err := c.limiter(req.URL.Host).Wait(req.Context()); err != nil {
return nil, err
}
url := req.URL.String()
if c.verboseLog {
log.Printf("[httpclient] FLARESOLVERR %s %s", req.Method, url)
}
var html string
var cookies []*http.Cookie
var err error
if req.Method == http.MethodGet {
html, cookies, err = c.fsClient.Get(req.Context(), url)
} else if req.Method == http.MethodPost {
var bodyStr string
if req.Body != nil {
bodyBytes, _ := io.ReadAll(req.Body)
bodyStr = string(bodyBytes)
if c.verboseLog {
log.Printf("[httpclient] FLARESOLVERR POST BODY: %s", bodyStr)
}
}
html, cookies, err = c.fsClient.Post(req.Context(), url, bodyStr)
} else {
return c.doDirect(req)
}
if err != nil {
return nil, fmt.Errorf("flaresolverr: %w", err)
}
if c.verboseLog {
log.Printf("[httpclient] FLARESOLVERR RESPONSE length=%d cookies=%d", len(html), len(cookies))
if len(html) > 0 {
truncate := html
if len(html) > 500 {
truncate = html[:500]
}
log.Printf("[httpclient] FLARESOLVERR RESPONSE HTML (first 500 chars): %s", truncate)
}
}
resp := &http.Response{
StatusCode: http.StatusOK,
Header: http.Header{},
Body: io.NopCloser(strings.NewReader(html)),
}
resp.Header.Set("Content-Type", "text/html; charset=utf-8")
if c.http.Jar != nil {
for _, cookie := range cookies {
c.http.Jar.SetCookies(req.URL, []*http.Cookie{cookie})
}
}
return resp, nil
}
func retryAfter(resp *http.Response) time.Duration {
ra := resp.Header.Get("Retry-After")
if ra == "" {
internal/httpclient/flaresolverr.go
+30 -6
View File
@@ -55,19 +55,43 @@ type fsCookie struct {
// Get fetches a Cloudflare-protected URL via FlareSolverr.
// Returns rendered HTML and extracted cookies.
func (f *FlareSolverrClient) Get(ctx context.Context, url string) (html string, cookies []*http.Cookie, err error) {
payload, _ := json.Marshal(flareSolverrRequest{
Cmd: "request.get",
return f.request(ctx, "request.get", url, "", nil)
}
func (f *FlareSolverrClient) Post(ctx context.Context, url string, body string) (html string, cookies []*http.Cookie, err error) {
return f.request(ctx, "request.post", url, body, map[string]string{"Content-Type": "application/x-www-form-urlencoded"})
}
func (f *FlareSolverrClient) request(ctx context.Context, cmd, url, body string, headers map[string]string) (html string, cookies []*http.Cookie, err error) {
type fsRequest struct {
Cmd string `json:"cmd"`
URL string `json:"url"`
PostData string `json:"postData,omitempty"`
Headers map[string]string `json:"headers,omitempty"`
MaxTimeout int `json:"maxTimeout"`
}
req := fsRequest{
Cmd: cmd,
URL: url,
MaxTimeout: 60000,
})
}
if body != "" {
req.PostData = body
}
if headers != nil {
req.Headers = headers
}
req, err := http.NewRequestWithContext(ctx, http.MethodPost, f.endpoint+"/v1", bytes.NewReader(payload))
payload, _ := json.Marshal(req)
httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, f.endpoint+"/v1", bytes.NewReader(payload))
if err != nil {
return "", nil, err
}
req.Header.Set("Content-Type", "application/json")
httpReq.Header.Set("Content-Type", "application/json")
resp, err := f.client.Do(req)
resp, err := f.client.Do(httpReq)
if err != nil {
return "", nil, err
}
tmp/build-errors.log
-1
View File
@@ -1 +0,0 @@
exit status 1