feat: initial Phase 1 implementation — core framework + Docker
- Data types (SManga, SChapter, Page, MangasPage, all Filter variants) - Source interfaces (Source, CatalogueSource) with MD5-based ID generation matching Tachiyomi/Suwayomi - HTTP client with per-host rate limiting, cookie jar, and 429 retry - FlareSolverr v1 client (FLARESOLVERR_URL env) - Generic GraphQL POST helper - goquery HTML parser wrappers - Source registry (panics on duplicate ID) - Multi-stage Dockerfile (golang:1.26-alpine + distroless) and compose.yml (postgres, flaresolverr, app)
This commit is contained in:
achmad
+17
@@ -0,0 +1,17 @@
|
||||
# Binaries
|
||||
/goyomi
|
||||
/cmd/server/server
|
||||
|
||||
# Environment
|
||||
.env
|
||||
.env.*
|
||||
|
||||
# Go tooling
|
||||
*.test
|
||||
*.out
|
||||
/vendor/
|
||||
|
||||
# IDE
|
||||
.idea/
|
||||
.vscode/
|
||||
*.swp
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
FROM golang:1.26-alpine AS builder
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
|
||||
COPY . .
|
||||
RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /goyomi ./cmd/server
|
||||
|
||||
FROM gcr.io/distroless/static-debian12
|
||||
|
||||
COPY --from=builder /goyomi /goyomi
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
ENTRYPOINT ["/goyomi"]
|
||||
@@ -0,0 +1,521 @@
|
||||
# Plan: Port Tachiyomi Extensions to Go
|
||||
|
||||
## Reference Projects
|
||||
|
||||
- **extensions-source**: `/Users/achmad/Documents/Belajar/Android/extensions-source`
|
||||
- `lib-multisrc/` — base source implementations (Phase 3)
|
||||
- `src/all/`, `src/en/` — standalone sources (Phase 4)
|
||||
- `core/src/` — CatalogueSource/HttpSource interface contracts (Phase 1)
|
||||
- **Suwayomi-Server**: `/Users/achmad/Documents/Belajar/Web/Suwayomi-Server`
|
||||
- `server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/` — DB schema reference (Phase 2)
|
||||
- `server/src/main/kotlin/suwayomi/tachidesk/server/database/migration/` — migration patterns (Phase 2)
|
||||
- `server/src/main/kotlin/suwayomi/tachidesk/manga/api/` — API route patterns (Phase 5)
|
||||
|
||||
---
|
||||
|
||||
## Context
|
||||
|
||||
Suwayomi-Server currently loads manga source extensions by downloading APKs, converting DEX bytecode to JARs, and instantiating Kotlin classes at runtime via reflection. The goal is a standalone Go service that reimplements all source logic, persists fetched data into PostgreSQL, and exposes a REST API for any consumer to query. The Go service is self-contained — no dependency on the JVM project.
|
||||
|
||||
Two rendering modes:
|
||||
- **Direct HTTP**: REST/JSON API sources — plain `net/http` + JSON unmarshal
|
||||
- **FlareSolverr**: Cloudflare-protected or JS-rendered sites — POST to FlareSolverr, get rendered HTML, parse with goquery
|
||||
|
||||
---
|
||||
|
||||
## New Project: `tachiyomi-go`
|
||||
|
||||
Separate Go module. Uses PostgreSQL (same DB the Docker Compose in this repo already sets up).
|
||||
|
||||
### Directory Structure
|
||||
|
||||
```
|
||||
tachiyomi-go/
|
||||
├── cmd/server/main.go
|
||||
├── internal/
|
||||
│ ├── source/
|
||||
│ │ ├── types.go # SManga, SChapter, Page, MangasPage, Filter
|
||||
│ │ └── interfaces.go # Source, CatalogueSource interfaces
|
||||
│ ├── httpclient/
|
||||
│ │ ├── client.go # Base HTTP client, cookie jar, per-host rate limiter
|
||||
│ │ ├── flaresolverr.go # FlareSolverr integration
|
||||
│ │ ├── graphql.go # GraphQL POST helper
|
||||
│ │ └── headers.go # Common header builders
|
||||
│ ├── parser/
|
||||
│ │ └── html.go # goquery helper wrappers
|
||||
│ ├── db/
|
||||
│ │ ├── db.go # pgx pool init, migration runner
|
||||
│ │ ├── queries/ # sqlc-generated query files
|
||||
│ │ │ ├── manga.sql.go
|
||||
│ │ │ ├── chapter.sql.go
|
||||
│ │ │ ├── page.sql.go
|
||||
│ │ │ └── source.sql.go
|
||||
│ │ └── migrations/
|
||||
│ │ ├── 001_init.sql
|
||||
│ │ └── ...
|
||||
│ └── registry/
|
||||
│ └── registry.go # Global source map, init-time registration
|
||||
├── sources/
|
||||
│ ├── base/ # ~67 base source implementations
|
||||
│ └── all/ + en/ # ~562 standalone source implementations
|
||||
└── api/
|
||||
└── handler.go
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Phase 1: Core Framework
|
||||
|
||||
### 1.1 Data Types (`internal/source/types.go`)
|
||||
|
||||
```go
|
||||
type SManga struct {
|
||||
URL string
|
||||
Title string
|
||||
Artist string
|
||||
Author string
|
||||
Description string
|
||||
Genre string // comma-separated
|
||||
Status int // 0=unknown,1=ongoing,2=completed,3=licensed,5=hiatus,6=cancelled
|
||||
ThumbnailURL string
|
||||
Initialized bool
|
||||
}
|
||||
|
||||
type SChapter struct {
|
||||
URL string
|
||||
Name string
|
||||
DateUpload int64 // unix milliseconds
|
||||
ChapterNumber float32
|
||||
Scanlator string
|
||||
}
|
||||
|
||||
type Page struct {
|
||||
Index int
|
||||
URL string
|
||||
ImageURL string
|
||||
}
|
||||
|
||||
type MangasPage struct {
|
||||
Mangas []SManga
|
||||
HasNextPage bool
|
||||
}
|
||||
```
|
||||
|
||||
Filter types: `SelectFilter`, `TextFilter`, `CheckboxFilter`, `TriStateFilter`, `GroupFilter`, `SortFilter`.
|
||||
|
||||
### 1.2 Source Interfaces (`internal/source/interfaces.go`)
|
||||
|
||||
```go
|
||||
type CatalogueSource interface {
|
||||
ID() int64
|
||||
Name() string
|
||||
Lang() string
|
||||
SupportsLatest() bool
|
||||
GetPopularManga(page int) (MangasPage, error)
|
||||
GetLatestUpdates(page int) (MangasPage, error)
|
||||
GetSearchManga(page int, query string, filters []Filter) (MangasPage, error)
|
||||
GetMangaDetails(manga SManga) (SManga, error)
|
||||
GetChapterList(manga SManga) ([]SChapter, error)
|
||||
GetPageList(chapter SChapter) ([]Page, error)
|
||||
GetImageURL(page Page) (string, error)
|
||||
GetFilterList() []Filter
|
||||
}
|
||||
```
|
||||
|
||||
#### ID Generation
|
||||
|
||||
Source IDs use the same formula as Tachiyomi/Suwayomi `HttpSource.generateId`:
|
||||
|
||||
```
|
||||
key = strings.ToLower(name) + "/" + lang + "/" + strconv.Itoa(versionId)
|
||||
hash = MD5(key) // 16 bytes
|
||||
id = first 8 bytes as big-endian int64
|
||||
id &= math.MaxInt64 // clear sign bit (Long.MAX_VALUE mask)
|
||||
```
|
||||
|
||||
Default `versionId` is 1. The earlier note about Java's `String.hashCode()` in the original plan was incorrect — the authoritative source is `HttpSource.kt` in Suwayomi-Server.
|
||||
|
||||
### 1.3 HTTP Client (`internal/httpclient/client.go`)
|
||||
|
||||
- Per-host rate limiting with `golang.org/x/time/rate`
|
||||
- Persistent cookie jar per source instance
|
||||
- Configurable timeout, user-agent, referer
|
||||
- Transparent retry on 429 (honor Retry-After header)
|
||||
|
||||
### 1.4 FlareSolverr (`internal/httpclient/flaresolverr.go`)
|
||||
|
||||
POST `{"cmd":"request.get","url":"...","maxTimeout":60000}` to `/v1`. Extract `solution.response` (HTML) and `solution.cookies`. After first clearance, reuse cookies via normal HTTP client — only re-invoke FlareSolverr on 403.
|
||||
|
||||
### 1.5 GraphQL Helper (`internal/httpclient/graphql.go`)
|
||||
|
||||
Only used internally to call upstream sources that expose GraphQL APIs (mangahub, senkuro, allanime, luscious, stashapp). Our own API is REST.
|
||||
|
||||
```go
|
||||
type GraphQLRequest struct {
|
||||
Query string `json:"query"`
|
||||
Variables any `json:"variables"`
|
||||
}
|
||||
|
||||
func Post(ctx context.Context, client *http.Client, url string, req GraphQLRequest, headers map[string]string) (*http.Response, error)
|
||||
```
|
||||
|
||||
### 1.6 HTML Parser (`internal/parser/html.go`)
|
||||
|
||||
Thin wrappers over `github.com/PuerkitoBio/goquery` (Go equivalent of JSoup):
|
||||
|
||||
```go
|
||||
func Parse(html string) (*goquery.Document, error)
|
||||
func Select(doc *goquery.Document, css string) *goquery.Selection
|
||||
func SelectFrom(sel *goquery.Selection, css string) *goquery.Selection
|
||||
func Attr(sel *goquery.Selection, name string) string
|
||||
func AbsURL(sel *goquery.Selection, attr string, baseURL string) string
|
||||
func OwnText(sel *goquery.Selection) string
|
||||
func TextTrim(sel *goquery.Selection) string
|
||||
func First(sel *goquery.Selection) *goquery.Selection
|
||||
```
|
||||
|
||||
### 1.7 Registry (`internal/registry/registry.go`)
|
||||
|
||||
```go
|
||||
var mu sync.RWMutex
|
||||
var sources = map[int64]source.CatalogueSource{}
|
||||
|
||||
func Register(s source.CatalogueSource)
|
||||
func Get(id int64) (source.CatalogueSource, bool)
|
||||
func All() []source.CatalogueSource
|
||||
```
|
||||
|
||||
Each source package calls `registry.Register(NewMySource())` in its `init()` function. All source packages are blank-imported in `cmd/server/main.go` so their `init()` runs at startup.
|
||||
|
||||
---
|
||||
|
||||
## Phase 2: Database Layer
|
||||
|
||||
### 2.1 Decision: New Schema Compatible with Suwayomi-Server
|
||||
|
||||
Suwayomi-Server uses either H2 (embedded) or PostgreSQL via Exposed ORM. The existing table structure (MangaTable, ChapterTable, PageTable, SourceTable) is a good reference. We adapt it for Go/PostgreSQL with a **compatible schema** so data can be shared if both services point to the same DB.
|
||||
|
||||
Key differences from Suwayomi-Server schema:
|
||||
- No `ExtensionTable` (sources are compiled in, not loaded from APKs)
|
||||
- `SourceTable` has no `extension` FK; sources are identified by their built-in ID
|
||||
- Add `fetched_at` timestamps on manga list results for cache invalidation
|
||||
- Use `BIGSERIAL` primary keys where Suwayomi uses `IntIdTable`
|
||||
|
||||
### 2.2 Schema (`internal/db/migrations/001_init.sql`)
|
||||
|
||||
```sql
|
||||
CREATE TABLE sources (
|
||||
id BIGINT PRIMARY KEY, -- generated from name+lang same as Tachiyomi: abs(("$name/" + lang + "/1").hashCode())
|
||||
name VARCHAR(128) NOT NULL,
|
||||
lang VARCHAR(32) NOT NULL,
|
||||
is_nsfw BOOLEAN NOT NULL DEFAULT FALSE
|
||||
);
|
||||
|
||||
CREATE TABLE manga (
|
||||
id SERIAL PRIMARY KEY,
|
||||
source_id BIGINT NOT NULL REFERENCES sources(id),
|
||||
url VARCHAR(2048) NOT NULL,
|
||||
title VARCHAR(512) NOT NULL,
|
||||
initialized BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
artist TEXT,
|
||||
author TEXT,
|
||||
description TEXT,
|
||||
genre TEXT, -- comma-separated
|
||||
status INTEGER NOT NULL DEFAULT 0,
|
||||
thumbnail_url VARCHAR(2048),
|
||||
thumbnail_last_fetched BIGINT NOT NULL DEFAULT 0,
|
||||
in_library BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
in_library_at BIGINT NOT NULL DEFAULT 0,
|
||||
real_url VARCHAR(2048),
|
||||
last_fetched_at BIGINT NOT NULL DEFAULT 0,
|
||||
chapters_last_fetched_at BIGINT NOT NULL DEFAULT 0,
|
||||
update_strategy VARCHAR(64) NOT NULL DEFAULT 'ALWAYS_UPDATE',
|
||||
UNIQUE (source_id, url)
|
||||
);
|
||||
|
||||
CREATE TABLE chapters (
|
||||
id SERIAL PRIMARY KEY,
|
||||
manga_id INTEGER NOT NULL REFERENCES manga(id) ON DELETE CASCADE,
|
||||
url VARCHAR(2048) NOT NULL,
|
||||
name VARCHAR(512) NOT NULL,
|
||||
date_upload BIGINT NOT NULL DEFAULT 0,
|
||||
chapter_number REAL NOT NULL DEFAULT -1,
|
||||
scanlator VARCHAR(256),
|
||||
source_order INTEGER NOT NULL,
|
||||
is_read BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
is_bookmarked BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
last_page_read INTEGER NOT NULL DEFAULT 0,
|
||||
last_read_at BIGINT NOT NULL DEFAULT 0,
|
||||
fetched_at BIGINT NOT NULL DEFAULT 0,
|
||||
real_url VARCHAR(2048),
|
||||
is_downloaded BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
page_count INTEGER NOT NULL DEFAULT -1,
|
||||
UNIQUE (manga_id, url)
|
||||
);
|
||||
|
||||
CREATE TABLE pages (
|
||||
id SERIAL PRIMARY KEY,
|
||||
chapter_id INTEGER NOT NULL REFERENCES chapters(id) ON DELETE CASCADE,
|
||||
"index" INTEGER NOT NULL,
|
||||
url VARCHAR(2048) NOT NULL,
|
||||
image_url TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE source_meta (
|
||||
source_id BIGINT NOT NULL REFERENCES sources(id),
|
||||
key VARCHAR(256) NOT NULL,
|
||||
value TEXT NOT NULL,
|
||||
PRIMARY KEY (source_id, key)
|
||||
);
|
||||
```
|
||||
|
||||
### 2.3 Tooling
|
||||
|
||||
- **Driver**: `github.com/jackc/pgx/v5` (pgx native, no database/sql overhead)
|
||||
- **Query gen**: `sqlc` — write SQL queries in `.sql` files, generate type-safe Go functions
|
||||
- **Migrations**: `golang-migrate/migrate` with `pgx` driver, runs on startup
|
||||
- **Connection pool**: `pgxpool.Pool` with configurable max conns
|
||||
|
||||
### 2.4 Data Flow
|
||||
|
||||
API call → source fetches data → **upsert into DB** → return from API.
|
||||
|
||||
**Manga list** (`GetPopularManga`): Upsert each SManga into `manga` (on conflict `source_id, url` update title/thumbnail/status). Update `last_fetched_at`. Return from DB.
|
||||
|
||||
**Manga detail** (`GetMangaDetails`): Fetch full detail from source, upsert all fields into `manga`, set `initialized=true`.
|
||||
|
||||
**Chapter list** (`GetChapterList`): Upsert each SChapter into `chapters` (on conflict `manga_id, url` update name/date/chapter_number). Update `chapters_last_fetched_at` on manga row.
|
||||
|
||||
**Page list** (`GetPageList`): Insert pages into `pages` (skip if already present). If source requires a second call to resolve image URLs (`GetImageURL`), store resolved `image_url`.
|
||||
|
||||
**Cache**: If `last_fetched_at` is within TTL (configurable, default 10 min for lists, 1h for details), serve from DB without hitting the source. TTL bypassed by `?refresh=true`.
|
||||
|
||||
---
|
||||
|
||||
## Phase 3: Base Source Implementations
|
||||
|
||||
### Group A — WordPress/CMS HTML Scrapers
|
||||
|
||||
| Base | Endpoint pattern | CF? |
|
||||
|------|-----------------|-----|
|
||||
| **madara** | POST `{base}/wp-admin/admin-ajax.php` (list), GET `{url}` (detail/chapters) | Yes |
|
||||
| **mangathemesia** | GET `{base}/{dir}/?page={n}`, GET `{url}` | Yes |
|
||||
| **madtheme** | GET `{base}/search?page={n}` (all list types) | Yes |
|
||||
| **wpcomics** | GET `{base}/{popularPath}?page={n}` | Yes |
|
||||
| **fmreader** | GET `{base}/{requestPath}?page={n}&sort=...` | Yes |
|
||||
| **mmrcms** | GET `{base}/filterList?page={n}&sortBy=views`, POST `{base}/advSearchFilter` | No |
|
||||
| **mangareader** | GET `{base}/?page={n}&type={t}` | Yes |
|
||||
| **zmanga** | GET `{base}/advanced-search/page/{n}/?order=popular` | Yes |
|
||||
| **mangaworld** | GET `{base}/archive?sort=most_read&page={n}` | Yes |
|
||||
| **grouple** | GET `{base}/list?sortType=rate&offset={50*(n-1)}` | No |
|
||||
| **foolslide** | GET `{base}/directory/{n}/` + JSON chapter API | No |
|
||||
| **liliana** | GET `{base}/ranking/week/{n}` | Yes |
|
||||
| **scanreader** | GET `{base}/bibliotheque/page/{n}/?sort=views` | No |
|
||||
| **gigaviewer** | GET `{base}/series` (all at once, no pagination) | Yes |
|
||||
| Others (mangawork, manga18, manhwaz, masonry, multichan, sinmh, etc.) | Various HTML GET | Most Yes |
|
||||
|
||||
All Group A bases use goquery selectors. Each has a config struct of overridable CSS selectors. FlareSolverr used when CF=Yes.
|
||||
|
||||
### Group B — JSON REST API Sources
|
||||
|
||||
| Base | Key endpoints | CF? | Auth |
|
||||
|------|--------------|-----|------|
|
||||
| **heancms** | `GET {api}/series?page={n}`, `GET {api}/chapter/query?series_slug={s}` | No | None |
|
||||
| **iken** | `GET {api}/comic?order=view&page={n}` | Yes | CF cookies |
|
||||
| **hentaihand** | `GET {base}/api/comics?page={n}&order_by=...` | No | None |
|
||||
| **pizzareader** | `GET {api}/comics`, `GET {api}/comics/{slug}` | Yes | None |
|
||||
| **gmanga** | `GET {base}/api/releases?page={n}`, `POST {base}/api/mangas/search` | No | None |
|
||||
| **spicytheme** | `GET {base}/api/...` | Yes | None |
|
||||
| **zeistmanga** | Blogger Feeds JSON API | Yes | None |
|
||||
| **mccms** | REST JSON | Yes | None |
|
||||
| **kemono** | `GET {base}/api/v1/creators`, `GET {base}/api/v1/{service}/{creator}/posts` | Yes | None |
|
||||
| **lectormoe** | REST JSON | Yes | Token |
|
||||
| **libgroup** | `GET {api}/api/latest-updates`, `GET {api}/api/auth/me` | Yes | WebView token → use FlareSolverr to obtain |
|
||||
| **mangabox** | REST JSON | No | None |
|
||||
| **mangadventure** | REST JSON | No | None |
|
||||
| **ezmanhwa** | REST JSON | No | None |
|
||||
| **monochrome** | REST JSON | No | None |
|
||||
|
||||
### Group C — GraphQL Sources
|
||||
|
||||
| Base | Endpoint | Notes |
|
||||
|------|----------|-------|
|
||||
| **mangahub** | POST `{api}/graphql` | Cookie `mhub_access` acquired via intermediate GET to a random chapter URL |
|
||||
| **senkuro** | POST `{api}/graphql` | API domain configurable via preferences |
|
||||
|
||||
### Group D — Special/Unique Sources
|
||||
|
||||
| Base | Pattern | Gotcha |
|
||||
|------|---------|--------|
|
||||
| **mangotheme** | JSON list + XOR/AES-encrypted page URLs | Implement page URL decryption in Go; key embedded in JS |
|
||||
| **mmlook** | JSON + encrypted pages + CF | Page decryption + FlareSolverr |
|
||||
| **guya** | `GET {base}/api/get_all_series/` (all manga at once) | No pagination; scanlation group filter in response |
|
||||
| **bakkin** | Single JSON URL, no list/search | Enumerate from object keys |
|
||||
| **gigaviewer** | All series in one page HTML | Client-side filter only; latest = same request |
|
||||
|
||||
---
|
||||
|
||||
## Phase 4: Standalone Sources — Notable Gotchas
|
||||
|
||||
### `all/mangadex`
|
||||
- Complex filter system: tags (AND/OR modes), demographics, content rating, publication status, sort
|
||||
- Cover art comes from a separate `covers` relationship — make a second API call or include via `includes[]=cover_art`
|
||||
- Chapter language filtering: only fetch `translatedLanguage[]=en` (or user-configured)
|
||||
- Rate limit: 5 req/s global, stricter for search
|
||||
- `at-home` server URL for pages: `GET /at-home/server/{chapterId}` returns CDN base URL; pages = `{baseUrl}/{quality}/{hash}/{filename}`
|
||||
|
||||
### `all/nhentaicom`
|
||||
- Cloudflare-protected
|
||||
- Tag/artist/character search with prefix syntax: `tag:isekai artist:...`
|
||||
- Pages come from a JSON blob embedded in the HTML (`JSON.parse(document.getElementById('...'))`)
|
||||
|
||||
### `all/komga`
|
||||
- Self-hosted; user must configure base URL + credentials (Basic Auth)
|
||||
- Series = manga, Books = chapters, Pages from book API
|
||||
- Supports CBZ/PDF libraries — page URL is a direct book page endpoint
|
||||
|
||||
### `all/e621`
|
||||
- Pools = manga (collections of posts), Posts = pages
|
||||
- Basic Auth required for higher rate limits and adult content
|
||||
- Nested tag exclusion (e.g. `rating:s`) needs proper encoding
|
||||
|
||||
### `all/kemono`
|
||||
- Cloudflare-protected
|
||||
- Service + creator = manga (e.g. Patreon/PixivFANBOX creator)
|
||||
- Posts = chapters; attachments/files within a post = pages
|
||||
- File URLs may be relative to `{base}/data`
|
||||
|
||||
### `all/danbooru`
|
||||
- Tag-based search (up to 2 tags for free accounts)
|
||||
- Gold/Platinum tier content only accessible with credentials
|
||||
- Pools as manga, pool posts as pages
|
||||
|
||||
### `all/pixiv`
|
||||
- Session cookie (`PHPSESSID`) auth — no public API key
|
||||
- Illust series = manga; user illustrations = chapters
|
||||
- Multi-tier image URLs: `thumb_mini`, `small`, `regular`, `original` — must use correct Referer header or get 403
|
||||
- R18 content requires age-verified account
|
||||
|
||||
### `all/luscious` (GraphQL)
|
||||
- GraphQL POST to `/graphql/playground`
|
||||
- Albums = manga, Pictures = pages
|
||||
- Adult content; account may be required for some content
|
||||
|
||||
### `all/mangaplus`
|
||||
- Official Shueisha app API
|
||||
- Uses protobuf OR a JSON endpoint (`/api/title_detail?title_id={id}`)
|
||||
- Page URLs are encrypted/obfuscated: each image URL requires a key from the chapter response to XOR-decrypt the actual URL
|
||||
- Viewer is web-only for some titles
|
||||
|
||||
### `all/stashapp`
|
||||
- Self-hosted; configure base URL
|
||||
- GraphQL API
|
||||
|
||||
### `en/allanime` (GraphQL)
|
||||
- Complex query variables: `translationType`, `countryOrigin`, search payload
|
||||
- Episode-based (chapters are episodes); `episodeString` used as chapter number
|
||||
- CDN for pages uses multiple quality tiers
|
||||
|
||||
### `en/asurascans`
|
||||
- Cloudflare-protected
|
||||
- Discord-gated content warnings on some chapters (just HTML, parseable)
|
||||
- Chapter pages embedded as `<img>` in a protected div — selector varies by site layout version
|
||||
|
||||
### `en/mangafire`
|
||||
- React/NextJS SSR; some data in JSON embedded in `<script id="__NEXT_DATA__">` tag
|
||||
- Extract and parse the JSON blob rather than scraping DOM
|
||||
|
||||
### `en/webtoons`
|
||||
- Official API with `Sec-Webtoon-Client-Data` HMAC header — compute HMAC-SHA256 over `{url}_{timestamp}` with a fixed key baked into the app
|
||||
- Mobile API differs from web API; use web API for broader access
|
||||
|
||||
### `en/mangadraft`
|
||||
- NextJS SSR; data in `__NEXT_DATA__` JSON blob
|
||||
- Authentication state affects available chapters
|
||||
|
||||
### `en/globalcomix`
|
||||
- Paginated REST API
|
||||
- Issue-based chapters; page images behind signed CDN URLs with short expiry — don't cache image URLs
|
||||
|
||||
### `en/bookwalker`
|
||||
- WebView/JS required for page rendering (DRM-protected)
|
||||
- Only metadata (title, cover, description) is accessible without purchase — mark all pages as unavailable or skip `GetPageList`
|
||||
|
||||
---
|
||||
|
||||
## Phase 5: HTTP API (REST)
|
||||
|
||||
The service exposes a plain REST API over HTTP. No GraphQL. (Note: some upstream manga sources internally use GraphQL — those are handled by the `internal/httpclient/graphql.go` helper when calling *their* APIs, but our API is always REST JSON.)
|
||||
|
||||
```
|
||||
GET /api/sources → [{id, name, lang, supportsLatest, isNsfw}]
|
||||
GET /api/sources/{id}/popular?page=1 → {mangas:[...], hasNextPage:bool} — from DB if cached
|
||||
GET /api/sources/{id}/latest?page=1 → {mangas:[...], hasNextPage:bool}
|
||||
GET /api/sources/{id}/search?q=&page=1 → {mangas:[...], hasNextPage:bool}
|
||||
GET /api/sources/{id}/filters → [{type, name, values:[...]}]
|
||||
|
||||
GET /api/sources/{id}/manga?url={url} → {id, title, author, ...} — fetches detail + upserts
|
||||
GET /api/sources/{id}/manga?url={url}/chapters → [{url, name, chapterNumber, ...}]
|
||||
GET /api/sources/{id}/manga?url={url}/chapters/{chapterUrl}/pages → [{index, url, imageUrl}]
|
||||
|
||||
GET /api/manga/{id} → full manga row from DB
|
||||
GET /api/manga/{id}/chapters → chapter list from DB
|
||||
GET /api/chapters/{id} → chapter row from DB
|
||||
GET /api/chapters/{id}/pages → page list from DB
|
||||
|
||||
GET /api/image?url={encoded}&source_id={id} → proxied image bytes, correct Content-Type header
|
||||
```
|
||||
|
||||
Query param `?refresh=true` bypasses TTL and forces a re-fetch from the source.
|
||||
|
||||
All errors: `{"error": "message"}` with appropriate HTTP status code.
|
||||
|
||||
---
|
||||
|
||||
## Dependencies (Go)
|
||||
|
||||
```
|
||||
github.com/PuerkitoBio/goquery # HTML parsing (JSoup equivalent)
|
||||
golang.org/x/time/rate # Per-host rate limiting
|
||||
github.com/go-chi/chi/v5 # HTTP router
|
||||
github.com/jackc/pgx/v5 # PostgreSQL driver + pool
|
||||
github.com/golang-migrate/migrate # DB migrations
|
||||
github.com/sqlc-dev/sqlc # SQL → Go code generation (dev dependency)
|
||||
encoding/json # stdlib
|
||||
net/http # stdlib
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Implementation Order
|
||||
|
||||
1. `internal/source/` — types + interfaces
|
||||
2. `internal/httpclient/` — client, FlareSolverr, GraphQL helper
|
||||
3. `internal/parser/html.go`
|
||||
4. `internal/db/` — schema, migrations, pgx pool, sqlc queries
|
||||
5. `internal/registry/registry.go`
|
||||
6. `api/handler.go` + `cmd/server/main.go`
|
||||
7. **Bases (simple → complex)**:
|
||||
- heancms → iken → hentaihand → pizzareader → gmanga (JSON)
|
||||
- keyoapp → wpcomics → fmreader → madtheme (HTML)
|
||||
- madara → mangathemesia (complex AJAX)
|
||||
- mangahub + senkuro (GraphQL)
|
||||
- mangotheme + mmlook (encryption)
|
||||
- libgroup (WebView auth via FlareSolverr)
|
||||
- remaining bases alphabetically
|
||||
8. **Standalone `all/`** — JSON-API first (mangadex, kemono, e621, komga), then CF/HTML
|
||||
9. **Standalone `en/`** — JSON-API first, then CF/HTML
|
||||
10. Wire DB upserts into all source call paths
|
||||
|
||||
---
|
||||
|
||||
## Verification
|
||||
|
||||
- `GET /api/sources` lists all registered sources
|
||||
- `GET /api/sources/{heancms_id}/popular?page=1` returns ≥1 manga, data persisted in `manga` table
|
||||
- `GET /api/sources/{heancms_id}/popular?page=1` second call served from DB (no HTTP call to source, verify via logs)
|
||||
- `POST /api/sources/{id}/chapters` returns chapters for a known URL, persisted in `chapters` table
|
||||
- `POST /api/sources/{madara_id}/pages` resolves image URLs via FlareSolverr path
|
||||
- `GET /api/image?url=...` proxies correctly with right Content-Type
|
||||
- `?refresh=true` forces re-fetch and updates DB records
|
||||
- Run `psql` against the DB and confirm rows in `manga`, `chapters`, `pages` tables after API calls
|
||||
@@ -0,0 +1,12 @@
|
||||
# tachiyomi-go — Master TODO
|
||||
|
||||
Progress tracker across all implementation phases.
|
||||
Detailed checklists are in each phase doc under `docs/`.
|
||||
|
||||
---
|
||||
|
||||
- [x] **Phase 1 — Core Framework** → `docs/phase1-core-framework.md`
|
||||
- [ ] **Phase 2 — Database Layer** → `docs/phase2-database.md`
|
||||
- [ ] **Phase 3 — Base Source Implementations (68 bases)** → `docs/phase3-bases.md`
|
||||
- [ ] **Phase 4 — Standalone Sources (555 sources)** → `docs/phase4-standalone.md`
|
||||
- [ ] **Phase 5 — HTTP API** → `docs/phase5-api.md`
|
||||
@@ -0,0 +1,22 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
|
||||
_ "goyomi/internal/registry" // ensures registry package is initialized
|
||||
)
|
||||
|
||||
func main() {
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
|
||||
fmt.Fprintln(w, "ok")
|
||||
})
|
||||
|
||||
addr := ":8080"
|
||||
log.Printf("listening on %s", addr)
|
||||
if err := http.ListenAndServe(addr, mux); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
+40
@@ -0,0 +1,40 @@
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16-alpine
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
POSTGRES_DB: goyomi
|
||||
POSTGRES_USER: goyomi
|
||||
POSTGRES_PASSWORD: goyomi
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U goyomi -d goyomi"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 10
|
||||
|
||||
flaresolverr:
|
||||
image: ghcr.io/flaresolverr/flaresolverr:latest
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
LOG_LEVEL: info
|
||||
ports:
|
||||
- "8191:8191"
|
||||
|
||||
app:
|
||||
build: .
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "8080:8080"
|
||||
environment:
|
||||
DATABASE_URL: postgres://goyomi:goyomi@postgres:5432/goyomi?sslmode=disable
|
||||
FLARESOLVERR_URL: http://flaresolverr:8191
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
flaresolverr:
|
||||
condition: service_started
|
||||
|
||||
volumes:
|
||||
postgres_data:
|
||||
@@ -0,0 +1,170 @@
|
||||
# Phase 1 — Core Framework
|
||||
|
||||
All foundational types, interfaces, and helpers that every source implementation depends on.
|
||||
Nothing in `sources/` can be written until this phase is complete.
|
||||
|
||||
Reference:
|
||||
- Tachiyomi source contract: `/Users/achmad/Documents/Belajar/Android/extensions-source/core/src/` (HttpSource, CatalogueSource)
|
||||
- Kotlin originals: `/Users/achmad/Documents/Belajar/Android/extensions-source/lib-multisrc/madara/src/eu/kanade/tachiyomi/multisrc/madara/Madara.kt`
|
||||
- Suwayomi source interface: `/Users/achmad/Documents/Belajar/Web/Suwayomi-Server/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/`
|
||||
|
||||
---
|
||||
|
||||
## 1.1 Data Types — `internal/source/types.go`
|
||||
|
||||
- [x] `SManga` struct
|
||||
- [x] `URL`, `Title`, `Artist`, `Author`, `Description` string fields
|
||||
- [x] `Genre` string (comma-separated, matches Tachiyomi convention)
|
||||
- [x] `Status` int — 0=unknown, 1=ongoing, 2=completed, 3=licensed, 5=hiatus, 6=cancelled
|
||||
- [x] `ThumbnailURL` string
|
||||
- [x] `Initialized` bool (false until `GetMangaDetails` has been called)
|
||||
- [x] `SChapter` struct
|
||||
- [x] `URL`, `Name`, `Scanlator` string fields
|
||||
- [x] `DateUpload` int64 — unix milliseconds (matches Tachiyomi)
|
||||
- [x] `ChapterNumber` float32
|
||||
- [x] `Page` struct
|
||||
- [x] `Index` int
|
||||
- [x] `URL` string — page HTML URL or chapter URL
|
||||
- [x] `ImageURL` string — direct image URL (may be empty until `GetImageURL` resolves it)
|
||||
- [x] `MangasPage` struct
|
||||
- [x] `Mangas []SManga`
|
||||
- [x] `HasNextPage bool`
|
||||
- [x] Filter types
|
||||
- [x] `Filter` interface with `Name()` and `Value()` methods
|
||||
- [x] `TextFilter` — free-text input
|
||||
- [x] `CheckboxFilter` — boolean
|
||||
- [x] `TriStateFilter` — ignore / include / exclude (0/1/2)
|
||||
- [x] `SelectFilter` — dropdown with named options (name + values []string)
|
||||
- [x] `SortFilter` — sortable list with ascending/descending state
|
||||
- [x] `GroupFilter` — container of sub-filters
|
||||
- [x] Status constants (`StatusUnknown`, `StatusOngoing`, etc.)
|
||||
|
||||
---
|
||||
|
||||
## 1.2 Source Interfaces — `internal/source/interfaces.go`
|
||||
|
||||
- [x] `Source` interface (base)
|
||||
- [x] `ID() int64`
|
||||
- [x] `Name() string`
|
||||
- [x] `Lang() string`
|
||||
- [x] `CatalogueSource` interface (embeds Source)
|
||||
- [x] `SupportsLatest() bool`
|
||||
- [x] `GetPopularManga(page int) (MangasPage, error)`
|
||||
- [x] `GetLatestUpdates(page int) (MangasPage, error)`
|
||||
- [x] `GetSearchManga(page int, query string, filters []Filter) (MangasPage, error)`
|
||||
- [x] `GetMangaDetails(manga SManga) (SManga, error)`
|
||||
- [x] `GetChapterList(manga SManga) ([]SChapter, error)`
|
||||
- [x] `GetPageList(chapter SChapter) ([]Page, error)`
|
||||
- [x] `GetImageURL(page Page) (string, error)` — no-op default for sources that embed image URLs directly in pages
|
||||
- [x] `GetFilterList() []Filter`
|
||||
- [x] ID generation helper
|
||||
- [x] `GenerateSourceID(name, lang string) int64` — MD5 of `"${name.lowercase()}/$lang/1"`, first 8 bytes big-endian, sign bit cleared. Matches Suwayomi `HttpSource.generateId`. (The original plan incorrectly described Java `String.hashCode()`.)
|
||||
- [x] Unit test against computed IDs (MangaDex/en, MangaDex/all, HeanCms/en)
|
||||
|
||||
---
|
||||
|
||||
## 1.3 HTTP Client — `internal/httpclient/client.go`
|
||||
|
||||
- [x] `Client` struct wrapping `*http.Client`
|
||||
- [x] Persistent `cookiejar.Jar` (one per source instance)
|
||||
- [x] Per-host `rate.Limiter` map (keyed by `Host`) with `sync.Mutex`
|
||||
- [x] Configurable `RateLimit` (requests/sec) and `Burst` per source
|
||||
- [x] Configurable `Timeout` (default 30s)
|
||||
- [x] Default `User-Agent` header (Chrome/Android UA, matches Tachiyomi)
|
||||
- [x] `NewClient(opts ...Option) *Client` constructor with functional options
|
||||
- [x] `Do(req *http.Request) (*http.Response, error)` — applies rate limit before executing
|
||||
- [x] `Get(ctx context.Context, url string, headers map[string]string) (*http.Response, error)` helper
|
||||
- [x] `Post(ctx context.Context, url string, body io.Reader, contentType string, headers map[string]string) (*http.Response, error)` helper
|
||||
- [x] 429 retry logic
|
||||
- [x] Read `Retry-After` header (seconds or HTTP-date)
|
||||
- [x] Sleep and retry up to N times (configurable, default 3)
|
||||
- [x] Optional `Referer` injection (configurable per client)
|
||||
|
||||
---
|
||||
|
||||
## 1.4 FlareSolverr — `internal/httpclient/flaresolverr.go`
|
||||
|
||||
- [x] `FlareSolverrClient` struct
|
||||
- [x] `endpoint` string (FlareSolverr v1 base URL, e.g. `http://localhost:8191`)
|
||||
- [x] Underlying `*http.Client` for talking to FlareSolverr itself
|
||||
- [x] `Get(ctx context.Context, url string, cookies []Cookie) (html string, cookies []Cookie, err error)`
|
||||
- [x] POST `{"cmd":"request.get","url":"...","maxTimeout":60000}` to `/v1`
|
||||
- [x] Extract `solution.response` (rendered HTML body)
|
||||
- [x] Extract `solution.cookies` → convert to `[]*http.Cookie`
|
||||
- [x] Cookie reuse strategy — cookies returned to caller; caller injects into source's cookie jar
|
||||
- [x] `FlareSolverrResponse` JSON struct (full response shape)
|
||||
- [x] Config: `FLARESOLVERR_URL` env var (disable if empty — `NewFlareSolverrClient` returns error)
|
||||
|
||||
---
|
||||
|
||||
## 1.5 GraphQL Helper — `internal/httpclient/graphql.go`
|
||||
|
||||
Used only to talk to upstream sources that expose GraphQL (mangahub, senkuro, allanime, luscious, stashapp).
|
||||
Our own API is plain REST.
|
||||
|
||||
- [x] `GraphQLRequest` struct — `Query string`, `Variables any`
|
||||
- [x] `Post[T any](ctx context.Context, client *http.Client, url string, req GraphQLRequest, headers map[string]string) (T, error)`
|
||||
- [x] Marshal request to JSON
|
||||
- [x] POST with `Content-Type: application/json`
|
||||
- [x] Unmarshal `data` field of response into T
|
||||
- [x] Surface `errors[]` as a Go error if present
|
||||
- [x] `graphQLResponse[T any]` envelope struct
|
||||
|
||||
---
|
||||
|
||||
## 1.6 Header Builders — `internal/httpclient/headers.go`
|
||||
|
||||
- [x] `AndroidUA() string` — Chrome/Android user-agent string (matches Tachiyomi default)
|
||||
- [x] `DesktopUA() string` — desktop Chrome user-agent (for sources requiring desktop)
|
||||
- [x] `JSONHeaders() map[string]string` — `Content-Type: application/json`, `Accept: application/json`
|
||||
- [x] `FormHeaders() map[string]string` — `Content-Type: application/x-www-form-urlencoded`
|
||||
- [x] `WithRefererHeader(headers map[string]string, referer string) map[string]string`
|
||||
- [x] `WithOrigin(headers map[string]string, origin string) map[string]string`
|
||||
|
||||
---
|
||||
|
||||
## 1.7 HTML Parser — `internal/parser/html.go`
|
||||
|
||||
Thin wrappers over `github.com/PuerkitoBio/goquery` (Go equivalent of JSoup used in Tachiyomi extensions).
|
||||
|
||||
- [x] `Parse(html string) (*goquery.Document, error)` — parse raw HTML string
|
||||
- [x] `ParseResponse(resp *http.Response) (*goquery.Document, error)` — parse from HTTP response body
|
||||
- [x] `Select(doc *goquery.Document, css string) *goquery.Selection`
|
||||
- [x] `SelectFrom(sel *goquery.Selection, css string) *goquery.Selection`
|
||||
- [x] `Attr(sel *goquery.Selection, name string) string` — returns empty string if not found
|
||||
- [x] `AbsURL(sel *goquery.Selection, attr string, baseURL string) string` — resolves relative URLs
|
||||
- [x] `OwnText(sel *goquery.Selection) string` — text of element excluding child elements
|
||||
- [x] `TextTrim(sel *goquery.Selection) string` — `.Text()` with strings.TrimSpace
|
||||
- [x] `First(sel *goquery.Selection) *goquery.Selection`
|
||||
- [x] `Each(sel *goquery.Selection, fn func(i int, s *goquery.Selection))` — convenience wrapper
|
||||
|
||||
---
|
||||
|
||||
## 1.8 Registry — `internal/registry/registry.go`
|
||||
|
||||
- [x] Package-level `map[int64]source.CatalogueSource` protected by `sync.RWMutex`
|
||||
- [x] `Register(s source.CatalogueSource)` — panics on duplicate ID (caught at startup)
|
||||
- [x] `Get(id int64) (source.CatalogueSource, bool)`
|
||||
- [x] `All() []source.CatalogueSource` — returns sorted slice (by ID) for deterministic listing
|
||||
- [x] Each source package must call `registry.Register(NewXxx())` in its `init()` function
|
||||
- [ ] All source packages blank-imported in `cmd/server/main.go` (done when sources exist)
|
||||
|
||||
---
|
||||
|
||||
## 1.9 Docker — `Dockerfile` + `compose.yml`
|
||||
|
||||
- [x] `Dockerfile` — multi-stage build (`golang:1.26-alpine` builder + `distroless/static-debian12` runtime)
|
||||
- [x] `compose.yml` — services: `app`, `postgres`, `flaresolverr`
|
||||
- [x] `postgres` — official image, healthcheck, named volume
|
||||
- [x] `flaresolverr` — ghcr.io/flaresolverr/flaresolverr, depends on nothing
|
||||
- [x] `app` — builds from `Dockerfile`, depends on postgres healthcheck, env vars wired
|
||||
|
||||
---
|
||||
|
||||
## Checklist: Phase 1 Done When
|
||||
|
||||
- [x] `go build ./internal/...` succeeds with no errors
|
||||
- [x] `GenerateSourceID` matches Tachiyomi IDs for at least 3 known sources
|
||||
- [ ] `FlareSolverrClient.Get` returns rendered HTML for a Cloudflare-protected URL (manual test — requires running FlareSolverr)
|
||||
- [ ] `GraphQLPost` works against a public GraphQL endpoint (manual test)
|
||||
- [x] Registry panics on duplicate source ID (unit test)
|
||||
@@ -0,0 +1,183 @@
|
||||
# Phase 2 — Database Layer
|
||||
|
||||
Persistent storage using PostgreSQL via pgx. Migrations run on startup. sqlc generates type-safe query code.
|
||||
|
||||
Reference schema modeled after Suwayomi-Server:
|
||||
- `/Users/achmad/Documents/Belajar/Web/Suwayomi-Server/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/`
|
||||
- Migrations: `/Users/achmad/Documents/Belajar/Web/Suwayomi-Server/server/src/main/kotlin/suwayomi/tachidesk/server/database/migration/`
|
||||
|
||||
---
|
||||
|
||||
## 2.1 Schema Migration — `internal/db/migrations/001_init.sql`
|
||||
|
||||
- [ ] `sources` table
|
||||
- [ ] `id BIGINT PRIMARY KEY` — generated via `GenerateSourceID(name, lang)` same as Tachiyomi
|
||||
- [ ] `name VARCHAR(128) NOT NULL`
|
||||
- [ ] `lang VARCHAR(32) NOT NULL`
|
||||
- [ ] `is_nsfw BOOLEAN NOT NULL DEFAULT FALSE`
|
||||
- [ ] `manga` table
|
||||
- [ ] `id SERIAL PRIMARY KEY`
|
||||
- [ ] `source_id BIGINT NOT NULL REFERENCES sources(id)`
|
||||
- [ ] `url VARCHAR(2048) NOT NULL`
|
||||
- [ ] `title VARCHAR(512) NOT NULL`
|
||||
- [ ] `initialized BOOLEAN NOT NULL DEFAULT FALSE`
|
||||
- [ ] `artist TEXT`, `author TEXT`, `description TEXT`, `genre TEXT`
|
||||
- [ ] `status INTEGER NOT NULL DEFAULT 0`
|
||||
- [ ] `thumbnail_url VARCHAR(2048)`
|
||||
- [ ] `thumbnail_last_fetched BIGINT NOT NULL DEFAULT 0`
|
||||
- [ ] `in_library BOOLEAN NOT NULL DEFAULT FALSE`
|
||||
- [ ] `in_library_at BIGINT NOT NULL DEFAULT 0`
|
||||
- [ ] `real_url VARCHAR(2048)`
|
||||
- [ ] `last_fetched_at BIGINT NOT NULL DEFAULT 0`
|
||||
- [ ] `chapters_last_fetched_at BIGINT NOT NULL DEFAULT 0`
|
||||
- [ ] `update_strategy VARCHAR(64) NOT NULL DEFAULT 'ALWAYS_UPDATE'`
|
||||
- [ ] `UNIQUE (source_id, url)`
|
||||
- [ ] `chapters` table
|
||||
- [ ] `id SERIAL PRIMARY KEY`
|
||||
- [ ] `manga_id INTEGER NOT NULL REFERENCES manga(id) ON DELETE CASCADE`
|
||||
- [ ] `url VARCHAR(2048) NOT NULL`
|
||||
- [ ] `name VARCHAR(512) NOT NULL`
|
||||
- [ ] `date_upload BIGINT NOT NULL DEFAULT 0`
|
||||
- [ ] `chapter_number REAL NOT NULL DEFAULT -1`
|
||||
- [ ] `scanlator VARCHAR(256)`
|
||||
- [ ] `source_order INTEGER NOT NULL`
|
||||
- [ ] `is_read BOOLEAN NOT NULL DEFAULT FALSE`
|
||||
- [ ] `is_bookmarked BOOLEAN NOT NULL DEFAULT FALSE`
|
||||
- [ ] `last_page_read INTEGER NOT NULL DEFAULT 0`
|
||||
- [ ] `last_read_at BIGINT NOT NULL DEFAULT 0`
|
||||
- [ ] `fetched_at BIGINT NOT NULL DEFAULT 0`
|
||||
- [ ] `real_url VARCHAR(2048)`
|
||||
- [ ] `is_downloaded BOOLEAN NOT NULL DEFAULT FALSE`
|
||||
- [ ] `page_count INTEGER NOT NULL DEFAULT -1`
|
||||
- [ ] `UNIQUE (manga_id, url)`
|
||||
- [ ] `pages` table
|
||||
- [ ] `id SERIAL PRIMARY KEY`
|
||||
- [ ] `chapter_id INTEGER NOT NULL REFERENCES chapters(id) ON DELETE CASCADE`
|
||||
- [ ] `"index" INTEGER NOT NULL`
|
||||
- [ ] `url VARCHAR(2048) NOT NULL`
|
||||
- [ ] `image_url TEXT`
|
||||
- [ ] `source_meta` table
|
||||
- [ ] `source_id BIGINT NOT NULL REFERENCES sources(id)`
|
||||
- [ ] `key VARCHAR(256) NOT NULL`
|
||||
- [ ] `value TEXT NOT NULL`
|
||||
- [ ] `PRIMARY KEY (source_id, key)`
|
||||
- [ ] Indexes
|
||||
- [ ] `CREATE INDEX ON manga (source_id)`
|
||||
- [ ] `CREATE INDEX ON manga (last_fetched_at)`
|
||||
- [ ] `CREATE INDEX ON chapters (manga_id)`
|
||||
- [ ] `CREATE INDEX ON pages (chapter_id)`
|
||||
|
||||
---
|
||||
|
||||
## 2.2 DB Initialization — `internal/db/db.go`
|
||||
|
||||
- [ ] `pgxpool.Pool` init from `DATABASE_URL` env var
|
||||
- [ ] Configurable `MaxConns` (default 10)
|
||||
- [ ] Configurable `MinConns` (default 2)
|
||||
- [ ] Connection health check on startup
|
||||
- [ ] Migration runner using `golang-migrate/migrate`
|
||||
- [ ] Source: `iofs` (embed migration SQL files with `//go:embed`)
|
||||
- [ ] Driver: `pgx5`
|
||||
- [ ] Run `migrate.Up()` on startup; log version before/after
|
||||
- [ ] Non-fatal on "no change" (already at latest)
|
||||
- [ ] `Queries` struct wrapping sqlc-generated query clients
|
||||
- [ ] `Close()` method to drain pool on shutdown
|
||||
|
||||
---
|
||||
|
||||
## 2.3 SQL Queries — `internal/db/queries/`
|
||||
|
||||
Write `.sql` files, then run `sqlc generate` to produce type-safe Go.
|
||||
|
||||
### `manga.sql`
|
||||
|
||||
- [ ] `UpsertManga` — INSERT ... ON CONFLICT (source_id, url) DO UPDATE all fields; returns row
|
||||
- [ ] `GetMangaBySourceURL` — SELECT by source_id + url
|
||||
- [ ] `GetMangaByID` — SELECT by primary key
|
||||
- [ ] `ListMangaBySource` — SELECT by source_id ORDER BY last_fetched_at DESC
|
||||
- [ ] `UpdateMangaDetails` — UPDATE artist/author/description/genre/status/thumbnail/initialized for a manga id
|
||||
- [ ] `UpdateMangaFetchedAt` — UPDATE last_fetched_at = $now WHERE id = $id
|
||||
- [ ] `UpdateChaptersFetchedAt` — UPDATE chapters_last_fetched_at WHERE id = $id
|
||||
|
||||
### `chapter.sql`
|
||||
|
||||
- [ ] `UpsertChapter` — INSERT ... ON CONFLICT (manga_id, url) DO UPDATE name/date/chapter_number/scanlator/source_order
|
||||
- [ ] `GetChapterByID`
|
||||
- [ ] `ListChaptersByManga` — ORDER BY source_order
|
||||
- [ ] `UpdateChapterFetchedAt`
|
||||
|
||||
### `page.sql`
|
||||
|
||||
- [ ] `UpsertPage` — INSERT ... ON CONFLICT (chapter_id, index) DO UPDATE url/image_url
|
||||
- [ ] `ListPagesByChapter` — ORDER BY index
|
||||
- [ ] `UpdatePageImageURL` — UPDATE image_url WHERE id = $id
|
||||
|
||||
### `source.sql`
|
||||
|
||||
- [ ] `UpsertSource` — INSERT ... ON CONFLICT (id) DO UPDATE name/lang/is_nsfw
|
||||
- [ ] `ListSources`
|
||||
- [ ] `GetSourceByID`
|
||||
- [ ] `GetSourceMeta` — SELECT value WHERE source_id = $id AND key = $key
|
||||
- [ ] `SetSourceMeta` — INSERT ... ON CONFLICT DO UPDATE value
|
||||
|
||||
---
|
||||
|
||||
## 2.4 sqlc Configuration — `sqlc.yaml`
|
||||
|
||||
- [ ] `version: "2"`
|
||||
- [ ] engine: `postgresql`
|
||||
- [ ] schema path pointing to `internal/db/migrations/`
|
||||
- [ ] queries path pointing to `internal/db/queries/*.sql`
|
||||
- [ ] output package: `internal/db/queries`
|
||||
- [ ] `emit_json_tags: true`
|
||||
- [ ] `emit_db_tags: true`
|
||||
|
||||
---
|
||||
|
||||
## 2.5 Data Flow & Cache Logic
|
||||
|
||||
Four upsert flows — each called from the API handler before returning data.
|
||||
|
||||
### Manga List (`GetPopularManga` / `GetLatestUpdates` / `GetSearchManga`)
|
||||
|
||||
- [ ] Check `manga.last_fetched_at`: if within TTL (default 10 min) and `?refresh=false`, return DB rows
|
||||
- [ ] Otherwise call source, then `UpsertManga` for each returned SManga
|
||||
- [ ] Update `last_fetched_at` on upserted rows
|
||||
- [ ] `UpsertSource` for the source record (idempotent on every list call)
|
||||
|
||||
### Manga Detail (`GetMangaDetails`)
|
||||
|
||||
- [ ] Check `manga.initialized`: if true and within TTL (default 1h), return DB row
|
||||
- [ ] Otherwise call source, then `UpdateMangaDetails` + set `initialized=true`
|
||||
|
||||
### Chapter List (`GetChapterList`)
|
||||
|
||||
- [ ] Check `manga.chapters_last_fetched_at`: if within TTL (default 10 min), return DB rows
|
||||
- [ ] Otherwise call source, then `UpsertChapter` for each SChapter (preserving `source_order` = slice index)
|
||||
- [ ] Update `chapters_last_fetched_at` on the manga row
|
||||
|
||||
### Page List (`GetPageList`)
|
||||
|
||||
- [ ] Check if pages exist for chapter in DB
|
||||
- [ ] Otherwise call source, then `UpsertPage` for each Page
|
||||
- [ ] If source needs a second call to resolve image URLs (via `GetImageURL`), call it and `UpdatePageImageURL`
|
||||
- [ ] Store resolved `image_url` — do NOT cache for sources with signed/expiring URLs (globalcomix, etc.)
|
||||
|
||||
### TTL Configuration
|
||||
|
||||
- [ ] `MANGA_LIST_TTL_SECONDS` env var (default 600)
|
||||
- [ ] `MANGA_DETAIL_TTL_SECONDS` env var (default 3600)
|
||||
- [ ] `CHAPTER_LIST_TTL_SECONDS` env var (default 600)
|
||||
- [ ] `?refresh=true` query param bypasses TTL on any route
|
||||
|
||||
---
|
||||
|
||||
## Checklist: Phase 2 Done When
|
||||
|
||||
- [ ] `golang-migrate` runs `001_init.sql` on a fresh DB without error
|
||||
- [ ] `sqlc generate` completes without errors; generated files compile
|
||||
- [ ] `UpsertManga` + `GetMangaBySourceURL` round-trip test passes
|
||||
- [ ] `UpsertChapter` correctly sets `source_order` from slice position
|
||||
- [ ] TTL cache logic returns DB rows on second call (verify no source HTTP call via log)
|
||||
- [ ] `?refresh=true` bypasses TTL and re-fetches from source
|
||||
- [ ] All tables visible in `psql` after API calls
|
||||
@@ -0,0 +1,261 @@
|
||||
# Phase 3 — Base Source Implementations
|
||||
|
||||
Complete port checklist for all 68 bases in `/Users/achmad/Documents/Belajar/Android/extensions-source/lib-multisrc/`.
|
||||
Each base goes into `sources/base/{name}/`. Check a box when the base compiles and
|
||||
at least one derived source passes a smoke test.
|
||||
|
||||
Detailed implementation notes for complex bases are in the **Notes** section at the bottom.
|
||||
|
||||
---
|
||||
|
||||
## All Bases — 68 total
|
||||
|
||||
- [ ] `base/bakkin` ⚠️ see notes
|
||||
- [ ] `base/colorlibanime`
|
||||
- [ ] `base/comicaso`
|
||||
- [ ] `base/comiciviewer`
|
||||
- [ ] `base/eromuse`
|
||||
- [ ] `base/ezmanhwa`
|
||||
- [ ] `base/fansubscat`
|
||||
- [ ] `base/fmreader` ⚠️ see notes
|
||||
- [ ] `base/foolslide` ⚠️ see notes
|
||||
- [ ] `base/fuzzydoodle`
|
||||
- [ ] `base/galleryadults`
|
||||
- [ ] `base/gattsu`
|
||||
- [ ] `base/gigaviewer` ⚠️ see notes
|
||||
- [ ] `base/gmanga` ⚠️ see notes
|
||||
- [ ] `base/goda`
|
||||
- [ ] `base/gravureblogger`
|
||||
- [ ] `base/greenshit`
|
||||
- [ ] `base/grouple` ⚠️ see notes
|
||||
- [ ] `base/guya` ⚠️ see notes
|
||||
- [ ] `base/heancms` ⚠️ see notes
|
||||
- [ ] `base/hentaihand` ⚠️ see notes
|
||||
- [ ] `base/hotcomics`
|
||||
- [ ] `base/iken` ⚠️ see notes
|
||||
- [ ] `base/initmanga`
|
||||
- [ ] `base/kemono` ⚠️ see notes
|
||||
- [ ] `base/keyoapp`
|
||||
- [ ] `base/lectormoe` ⚠️ see notes
|
||||
- [ ] `base/libgroup` ⚠️ see notes
|
||||
- [ ] `base/liliana` ⚠️ see notes
|
||||
- [ ] `base/madara` ⚠️ see notes
|
||||
- [ ] `base/madtheme` ⚠️ see notes
|
||||
- [ ] `base/manga18`
|
||||
- [ ] `base/mangabox`
|
||||
- [ ] `base/mangacatalog`
|
||||
- [ ] `base/mangadventure` ⚠️ see notes
|
||||
- [ ] `base/mangahub` ⚠️ see notes
|
||||
- [ ] `base/mangareader` ⚠️ see notes
|
||||
- [ ] `base/mangataro`
|
||||
- [ ] `base/mangathemesia` ⚠️ see notes
|
||||
- [ ] `base/mangawork`
|
||||
- [ ] `base/mangaworld` ⚠️ see notes
|
||||
- [ ] `base/mangotheme` ⚠️ see notes
|
||||
- [ ] `base/manhwaz`
|
||||
- [ ] `base/masonry`
|
||||
- [ ] `base/mccms`
|
||||
- [ ] `base/mmlook` ⚠️ see notes
|
||||
- [ ] `base/mmrcms` ⚠️ see notes
|
||||
- [ ] `base/monochrome`
|
||||
- [ ] `base/multichan`
|
||||
- [ ] `base/natsuid`
|
||||
- [ ] `base/oceanwp`
|
||||
- [ ] `base/paprika`
|
||||
- [ ] `base/peachscan`
|
||||
- [ ] `base/pizzareader` ⚠️ see notes
|
||||
- [ ] `base/raijinscans`
|
||||
- [ ] `base/scanr`
|
||||
- [ ] `base/scanreader` ⚠️ see notes
|
||||
- [ ] `base/senkuro` ⚠️ see notes
|
||||
- [ ] `base/sinmh`
|
||||
- [ ] `base/spicytheme`
|
||||
- [ ] `base/stalkercms`
|
||||
- [ ] `base/uzaymanga`
|
||||
- [ ] `base/vercomics`
|
||||
- [ ] `base/wpcomics` ⚠️ see notes
|
||||
- [ ] `base/yuyu`
|
||||
- [ ] `base/zeistmanga`
|
||||
- [ ] `base/zmanga` ⚠️ see notes
|
||||
|
||||
---
|
||||
|
||||
## Notes — Complex Bases
|
||||
|
||||
### `base/heancms` ⚠️
|
||||
- Config struct: `BaseURL`, `APIURL`, `Lang`, `DateFormat`
|
||||
- `GetPopularManga` — `GET {api}/series?page={n}&order=views`
|
||||
- `GetLatestUpdates` — `GET {api}/series?page={n}&order=latest`
|
||||
- `GetSearchManga` — `GET {api}/series?page={n}&query={q}`
|
||||
- `GetMangaDetails` — `GET {api}/series/{slug}`
|
||||
- `GetChapterList` — `GET {api}/chapter/query?series_slug={s}&limit=9999`
|
||||
- `GetPageList` — `GET {api}/chapter/{slug}` → extract image array
|
||||
- Parse `has_next_page` field from list responses
|
||||
|
||||
### `base/hentaihand` ⚠️
|
||||
- Config struct: `BaseURL`, `Lang`
|
||||
- `GetPopularManga` — `GET {base}/api/comics?page={n}&order_by=popularity`
|
||||
- `GetLatestUpdates` — `GET {base}/api/comics?page={n}&order_by=date`
|
||||
- `GetPageList` — `GET {base}/api/comics/{slug}/chapters/{n}` → images array
|
||||
|
||||
### `base/pizzareader` ⚠️
|
||||
- Config struct: `APIURL`, `Lang`
|
||||
- `GetPageList` — `GET {api}/comics/{comic}/{chapter}` → page URLs
|
||||
- FlareSolverr mode (CF=Yes): inject cookies from clearance
|
||||
|
||||
### `base/gmanga` ⚠️
|
||||
- `GetPopularManga` — `GET {base}/api/releases?page={n}`
|
||||
- `GetSearchManga` — `POST {base}/api/mangas/search` with JSON body `{q, page}`
|
||||
- `GetPageList` — extract chapter page data (may require resolving CDN token)
|
||||
- Handle Arabic/RTL title fields
|
||||
|
||||
### `base/mangadventure` ⚠️
|
||||
- Django manga reader REST API
|
||||
- `GET {base}/api/v2/series/` — list; `GET {base}/api/v2/series/{slug}/` — detail
|
||||
- `GET {base}/api/v2/chapters/?series={slug}` — chapters; `GET {base}/api/v2/chapters/{vol}/{num}/` — pages
|
||||
|
||||
### `base/madara` ⚠️
|
||||
- Config struct: `BaseURL`, `Lang`, `DateFormat`, overridable CSS selectors
|
||||
- `GetPopularManga` — `POST {base}/wp-admin/admin-ajax.php` with `action=madara_load_more`, `vars[paged]={n}`
|
||||
- `GetLatestUpdates` — same AJAX with `vars[meta_key]=_latest_update`
|
||||
- `GetSearchManga` — `GET {base}/?s={q}&post_type=wp-manga`
|
||||
- `GetMangaDetails` — `GET {url}` → parse `.post-title`, `.author-content`, `.genres-content`, `.manga-summary`, `.post-image img`
|
||||
- `GetChapterList` — `POST {url}/ajax/chapters/` → parse chapter list HTML
|
||||
- `GetPageList` — `GET {chapter_url}` → parse `div.reading-content img`
|
||||
- FlareSolverr required; configurable selectors struct (child sources override individual selectors)
|
||||
- Date parsing: relative ("X days ago") + absolute ("MMMM dd, yyyy") formats
|
||||
|
||||
### `base/mangathemesia` ⚠️
|
||||
- Config struct: `BaseURL`, `Lang`, `MangaDir` (e.g. "manga"/"manhwa"), overridable selectors
|
||||
- `GetPopularManga` — `GET {base}/{dir}/?page={n}&order=popular`
|
||||
- `GetChapterList` — parse `#chapterlist li` elements
|
||||
- `GetPageList` — extract `ts_reader.run({...})` JS JSON blob, parse `sources[].images`
|
||||
- FlareSolverr required
|
||||
|
||||
### `base/madtheme` ⚠️
|
||||
- All list types via `GET {base}/search?page={n}&sort=...`
|
||||
- `GetPageList` — parse JSON blob in `<script>` tag containing image array
|
||||
- FlareSolverr required
|
||||
|
||||
### `base/wpcomics` ⚠️
|
||||
- Config struct: `BaseURL`, `Lang`, `PopularPath` (default "tim-kiem"), `DateFormat`
|
||||
- `GetPopularManga` — `GET {base}/{popularPath}?page={n}`
|
||||
- FlareSolverr required
|
||||
|
||||
### `base/fmreader` ⚠️
|
||||
- Config struct: `BaseURL`, `Lang`, `RequestPath`, overridable selectors
|
||||
- `GetPopularManga` — `GET {base}/{requestPath}?page={n}&sort=views`
|
||||
- FlareSolverr required
|
||||
|
||||
### `base/mmrcms` ⚠️
|
||||
- `GetPopularManga` — `GET {base}/filterList?page={n}&sortBy=views&asc=false`
|
||||
- `GetSearchManga` — `POST {base}/advSearchFilter` with form data
|
||||
- JSON response; no FlareSolverr required
|
||||
|
||||
### `base/mangareader` ⚠️
|
||||
- Config struct: `BaseURL`, `Lang`, `TypeParam` (comic/manga/manhwa)
|
||||
- `GetPopularManga` — `GET {base}/?page={n}&type={t}&status=all&order=popular`
|
||||
- FlareSolverr required
|
||||
|
||||
### `base/zmanga` ⚠️
|
||||
- `GetPopularManga` — `GET {base}/advanced-search/page/{n}/?order=popular`
|
||||
- FlareSolverr required
|
||||
|
||||
### `base/mangaworld` ⚠️
|
||||
- `GetPopularManga` — `GET {base}/archive?sort=most_read&page={n}`
|
||||
- FlareSolverr required
|
||||
|
||||
### `base/grouple` ⚠️
|
||||
- `GetPopularManga` — `GET {base}/list?sortType=rate&offset={50*(n-1)}`
|
||||
- No FlareSolverr
|
||||
|
||||
### `base/foolslide` ⚠️
|
||||
- `GetPopularManga` — `GET {base}/directory/{n}/`
|
||||
- `GetChapterList` — JSON API: `GET {base}/api/reader/chapters?comic={slug}`
|
||||
- `GetPageList` — JSON API: `GET {base}/api/reader/images?chapter={id}`
|
||||
- No FlareSolverr
|
||||
|
||||
### `base/liliana` ⚠️
|
||||
- `GetPopularManga` — `GET {base}/ranking/week/{n}`
|
||||
- FlareSolverr required
|
||||
|
||||
### `base/scanreader` ⚠️
|
||||
- `GetPopularManga` — `GET {base}/bibliotheque/page/{n}/?sort=views`
|
||||
- No FlareSolverr
|
||||
|
||||
### `base/gigaviewer` ⚠️
|
||||
- `GET {base}/series` returns all manga at once; no pagination (`HasNextPage` always false)
|
||||
- `GetLatestUpdates` = same request as popular (no separate endpoint)
|
||||
- FlareSolverr required; client-side filtering replicated in Go
|
||||
|
||||
### `base/mangahub` ⚠️ (GraphQL)
|
||||
- Cookie acquisition: `GET` to any chapter URL to set `mhub_access` cookie, then reuse for GraphQL
|
||||
- All operations via GraphQL POST to `{api}/graphql`
|
||||
- `GetPopularManga` — `searchManga(x:POPULAR, genre:"all", page:N)`
|
||||
- `GetPageList` — `chapter(mangaId:N, number:N)` → extract pages
|
||||
- Inject `mhub_access` cookie header on all GraphQL requests
|
||||
|
||||
### `base/senkuro` ⚠️ (GraphQL)
|
||||
- Config struct: `APIURL` (configurable per source preferences)
|
||||
- Standard GraphQL list/detail/chapter/page queries
|
||||
- No CF requirement
|
||||
|
||||
### `base/mangotheme` ⚠️ (encrypted)
|
||||
- JSON list + detail (standard REST)
|
||||
- Page URL decryption: extract encryption key from embedded JS, then XOR/AES decrypt each page URL
|
||||
- Use `crypto/aes` or manual XOR depending on algorithm found in extension source
|
||||
|
||||
### `base/mmlook` ⚠️ (encrypted + CF)
|
||||
- Same page URL decryption as mangotheme
|
||||
- FlareSolverr required
|
||||
|
||||
### `base/guya` ⚠️
|
||||
- `GET {base}/api/get_all_series/` returns all manga as a map; no pagination
|
||||
- `HasNextPage` always false
|
||||
- Scanlation group filter applied client-side
|
||||
|
||||
### `base/bakkin` ⚠️
|
||||
- No list/search; all manga from a single JSON URL; enumerate from object keys
|
||||
- `GetSearchManga` does client-side title filtering only
|
||||
|
||||
### `base/iken` ⚠️
|
||||
- CF-protected; FlareSolverr required for cookie acquisition
|
||||
- JSON REST after clearance
|
||||
|
||||
### `base/lectormoe` ⚠️
|
||||
- CF-protected; Token auth required
|
||||
- Obtain token via FlareSolverr session
|
||||
|
||||
### `base/libgroup` ⚠️
|
||||
- FlareSolverr required to acquire WebView auth token
|
||||
- `GET {api}/api/latest-updates`, `GET {api}/api/auth/me`
|
||||
- Store acquired token in `source_meta` table for reuse
|
||||
|
||||
### `base/kemono` ⚠️
|
||||
- `GET {base}/api/v1/creators` — all creators (= manga list)
|
||||
- `GET {base}/api/v1/{service}/{creator}/posts?o={offset}` — paginate in 50-post increments
|
||||
- File URLs: prefix relative paths with `{base}/data`
|
||||
- FlareSolverr required
|
||||
|
||||
---
|
||||
|
||||
## Shared Helpers (implement once in `sources/base/util/`)
|
||||
|
||||
- [ ] `parseRelativeDate(s string) int64` — "2 days ago" → unix ms
|
||||
- [ ] `parseAbsoluteDate(s, format string) int64` — "January 01, 2024" → unix ms
|
||||
- [ ] `slugFromURL(url string) string` — trailing path segment
|
||||
- [ ] `cleanText(s string) string` — HTML entity decode + whitespace normalize
|
||||
- [ ] `statusFromString(s string) int` — "ongoing"/"completed"/etc. → int constant
|
||||
- [ ] `extractNextDataJSON(html string) ([]byte, error)` — pull `__NEXT_DATA__` JSON from NextJS pages
|
||||
|
||||
---
|
||||
|
||||
## Checklist: Phase 3 Done When
|
||||
|
||||
- [ ] All 68 bases compile: `go build ./sources/base/...`
|
||||
- [ ] `base/heancms` — `GetPopularManga` returns ≥1 manga from a live site
|
||||
- [ ] `base/madara` — `GetChapterList` returns chapters via AJAX endpoint
|
||||
- [ ] `base/mangathemesia` — `GetPageList` extracts images from `ts_reader.run()` JS blob
|
||||
- [ ] `base/mangahub` — GraphQL popular list works with cookie acquisition
|
||||
- [ ] `base/mangotheme` — decrypted page URL returns HTTP 200 image
|
||||
- [ ] FlareSolverr path — a CF-protected base returns data when FlareSolverr is running
|
||||
@@ -0,0 +1,665 @@
|
||||
# Phase 4 — Standalone Sources
|
||||
|
||||
Complete port checklist. Check a box when the source passes a basic smoke test
|
||||
(popular/latest list returns ≥1 result, or detail+pages resolve for a known URL).
|
||||
|
||||
Reference:
|
||||
- `/Users/achmad/Documents/Belajar/Android/extensions-source/src/all/`
|
||||
- `/Users/achmad/Documents/Belajar/Android/extensions-source/src/en/`
|
||||
|
||||
Detailed implementation notes for complex sources are in the **Notes** section at the bottom.
|
||||
|
||||
---
|
||||
|
||||
## `sources/all/` — 125 sources
|
||||
|
||||
- [ ] `all/ahottie`
|
||||
- [ ] `all/akuma`
|
||||
- [ ] `all/allporncomicsco`
|
||||
- [ ] `all/asmhentai`
|
||||
- [ ] `all/baobua`
|
||||
- [ ] `all/beauty3600000`
|
||||
- [ ] `all/buondua`
|
||||
- [ ] `all/comicfury`
|
||||
- [ ] `all/comicgrowl`
|
||||
- [ ] `all/comicklive`
|
||||
- [ ] `all/comicskingdom`
|
||||
- [ ] `all/comicsvalley`
|
||||
- [ ] `all/comikey`
|
||||
- [ ] `all/commitstrip`
|
||||
- [ ] `all/coomer`
|
||||
- [ ] `all/coronaex`
|
||||
- [ ] `all/cosplaytele`
|
||||
- [ ] `all/cubari`
|
||||
- [ ] `all/danbooru` ⚠️ see notes
|
||||
- [ ] `all/deviantart`
|
||||
- [ ] `all/dragonballmultiverse`
|
||||
- [ ] `all/e621` ⚠️ see notes
|
||||
- [ ] `all/elitebabes`
|
||||
- [ ] `all/everiaclub`
|
||||
- [ ] `all/everiaclubcom`
|
||||
- [ ] `all/femjoyhunter`
|
||||
- [ ] `all/foamgirl`
|
||||
- [ ] `all/foolslidecustomizable`
|
||||
- [ ] `all/fourkhd`
|
||||
- [ ] `all/ftvhunter`
|
||||
- [ ] `all/fuwayomi`
|
||||
- [ ] `all/globalcomix` ⚠️ see notes
|
||||
- [ ] `all/grabberzone`
|
||||
- [ ] `all/hdoujin`
|
||||
- [ ] `all/hennojin`
|
||||
- [ ] `all/hentai3`
|
||||
- [ ] `all/hentaicosplay`
|
||||
- [ ] `all/hentaienvy`
|
||||
- [ ] `all/hentaiera`
|
||||
- [ ] `all/hentaifox`
|
||||
- [ ] `all/hentaihand`
|
||||
- [ ] `all/hentairox`
|
||||
- [ ] `all/hentaizap`
|
||||
- [ ] `all/hniscantrad`
|
||||
- [ ] `all/holonometria`
|
||||
- [ ] `all/honeytoon`
|
||||
- [ ] `all/imhentai`
|
||||
- [ ] `all/izneo`
|
||||
- [ ] `all/jjcos`
|
||||
- [ ] `all/joymiihub`
|
||||
- [ ] `all/junmeitu`
|
||||
- [ ] `all/kemono` ⚠️ see notes
|
||||
- [ ] `all/kiutaku`
|
||||
- [ ] `all/kodokustudio`
|
||||
- [ ] `all/koharu`
|
||||
- [ ] `all/komga` ⚠️ see notes
|
||||
- [ ] `all/lanraragi`
|
||||
- [ ] `all/leagueoflegends`
|
||||
- [ ] `all/lunaranime`
|
||||
- [ ] `all/luscious` ⚠️ see notes
|
||||
- [ ] `all/magicaltranslators`
|
||||
- [ ] `all/manga18me`
|
||||
- [ ] `all/mangaball`
|
||||
- [ ] `all/mangacrazy`
|
||||
- [ ] `all/mangadex` ⚠️ see notes
|
||||
- [ ] `all/mangadraft`
|
||||
- [ ] `all/mangafire`
|
||||
- [ ] `all/mangaforfree`
|
||||
- [ ] `all/mangaplus` ⚠️ see notes
|
||||
- [ ] `all/mangapluscreators`
|
||||
- [ ] `all/mangataro`
|
||||
- [ ] `all/mangatoon`
|
||||
- [ ] `all/mangaup`
|
||||
- [ ] `all/mango`
|
||||
- [ ] `all/manhuarm`
|
||||
- [ ] `all/manhwa18cc`
|
||||
- [ ] `all/manhwa18net`
|
||||
- [ ] `all/manhwa18uncensored`
|
||||
- [ ] `all/manhwaclubnet`
|
||||
- [ ] `all/manhwadashraw`
|
||||
- [ ] `all/mayotune`
|
||||
- [ ] `all/metarthunter`
|
||||
- [ ] `all/miauscan`
|
||||
- [ ] `all/misskon`
|
||||
- [ ] `all/mitaku`
|
||||
- [ ] `all/myreadingmanga`
|
||||
- [ ] `all/namicomi`
|
||||
- [ ] `all/nhentaicom` ⚠️ see notes
|
||||
- [ ] `all/nhentaixxx`
|
||||
- [ ] `all/niadd`
|
||||
- [ ] `all/ninemanga`
|
||||
- [ ] `all/novelcool`
|
||||
- [ ] `all/ososedki`
|
||||
- [ ] `all/pandachaika`
|
||||
- [ ] `all/peppercarrot`
|
||||
- [ ] `all/photos18`
|
||||
- [ ] `all/pixiv` ⚠️ see notes
|
||||
- [ ] `all/playmatehunter`
|
||||
- [ ] `all/pornpics`
|
||||
- [ ] `all/projectsuki`
|
||||
- [ ] `all/qtoon`
|
||||
- [ ] `all/rokuhentai`
|
||||
- [ ] `all/sakuramanhwa`
|
||||
- [ ] `all/sandraandwoo`
|
||||
- [ ] `all/seraphicdeviltry`
|
||||
- [ ] `all/simplycosplay`
|
||||
- [ ] `all/simplyhentai`
|
||||
- [ ] `all/stashapp` ⚠️ see notes
|
||||
- [ ] `all/taddyink`
|
||||
- [ ] `all/tappytoon`
|
||||
- [ ] `all/thelibraryofohara`
|
||||
- [ ] `all/thunderscans`
|
||||
- [ ] `all/toomics`
|
||||
- [ ] `all/twicomi`
|
||||
- [ ] `all/uncensoredmanhwa`
|
||||
- [ ] `all/vinnieVeritas`
|
||||
- [ ] `all/webtoons` ⚠️ see notes
|
||||
- [ ] `all/xarthunter`
|
||||
- [ ] `all/xasiatalbums`
|
||||
- [ ] `all/xgmn`
|
||||
- [ ] `all/xinmeitulu`
|
||||
- [ ] `all/xiutaku`
|
||||
- [ ] `all/xkcd`
|
||||
- [ ] `all/yabai`
|
||||
- [ ] `all/yaoimangaonline`
|
||||
- [ ] `all/yellownote`
|
||||
- [ ] `all/yskcomics`
|
||||
|
||||
---
|
||||
|
||||
## `sources/en/` — 430 sources
|
||||
|
||||
- [ ] `en/akaicomic`
|
||||
- [ ] `en/alandal`
|
||||
- [ ] `en/allanime` ⚠️ see notes
|
||||
- [ ] `en/allporncomic`
|
||||
- [ ] `en/allporncomicio`
|
||||
- [ ] `en/anisascans`
|
||||
- [ ] `en/apcomics`
|
||||
- [ ] `en/aquamanga`
|
||||
- [ ] `en/arcrelight`
|
||||
- [ ] `en/arenascans`
|
||||
- [ ] `en/armageddon`
|
||||
- [ ] `en/artlapsa`
|
||||
- [ ] `en/arvencomics`
|
||||
- [ ] `en/arvenscans`
|
||||
- [ ] `en/aryascans`
|
||||
- [ ] `en/asiatoon`
|
||||
- [ ] `en/asmotoon`
|
||||
- [ ] `en/assortedscans`
|
||||
- [ ] `en/asurascans` ⚠️ see notes
|
||||
- [ ] `en/athreascans`
|
||||
- [ ] `en/atsumaru`
|
||||
- [ ] `en/aurora`
|
||||
- [ ] `en/azcomic`
|
||||
- [ ] `en/azuki`
|
||||
- [ ] `en/baektoons`
|
||||
- [ ] `en/bakkin` ⚠️ see notes
|
||||
- [ ] `en/bakkinselfhosted`
|
||||
- [ ] `en/batcave`
|
||||
- [ ] `en/battleinfivesecondsaftermeeting`
|
||||
- [ ] `en/beehentai`
|
||||
- [ ] `en/bookwalker` ⚠️ see notes
|
||||
- [ ] `en/boratscans`
|
||||
- [ ] `en/boxmanhwa`
|
||||
- [ ] `en/broccolisoup`
|
||||
- [ ] `en/bunmanga`
|
||||
- [ ] `en/buttsmithy`
|
||||
- [ ] `en/clonemanga`
|
||||
- [ ] `en/clowncorps`
|
||||
- [ ] `en/cmanhua`
|
||||
- [ ] `en/cocomic`
|
||||
- [ ] `en/coffeemanga`
|
||||
- [ ] `en/collectedcurios`
|
||||
- [ ] `en/comicasura`
|
||||
- [ ] `en/comiccx`
|
||||
- [ ] `en/comichubfree`
|
||||
- [ ] `en/comickfan`
|
||||
- [ ] `en/comickiba`
|
||||
- [ ] `en/comicland`
|
||||
- [ ] `en/comicsland`
|
||||
- [ ] `en/comix`
|
||||
- [ ] `en/crowscans`
|
||||
- [ ] `en/cucumbermanga`
|
||||
- [ ] `en/culturedworks`
|
||||
- [ ] `en/cutiecomics`
|
||||
- [ ] `en/dankefurslesen`
|
||||
- [ ] `en/darklegacycomics`
|
||||
- [ ] `en/darkscans`
|
||||
- [ ] `en/darkscience`
|
||||
- [ ] `en/darthsdroids`
|
||||
- [ ] `en/deathtollscans`
|
||||
- [ ] `en/decadencescans`
|
||||
- [ ] `en/dflowscans`
|
||||
- [ ] `en/digitalcomicmuseum`
|
||||
- [ ] `en/divascans`
|
||||
- [ ] `en/doujinio`
|
||||
- [ ] `en/doujins`
|
||||
- [ ] `en/dragontea`
|
||||
- [ ] `en/drakescans`
|
||||
- [ ] `en/dynasty`
|
||||
- [ ] `en/eggporncomics`
|
||||
- [ ] `en/egscomics`
|
||||
- [ ] `en/eighteenporncomic`
|
||||
- [ ] `en/eightmuses`
|
||||
- [ ] `en/elanschool`
|
||||
- [ ] `en/elftoon`
|
||||
- [ ] `en/epicmanga`
|
||||
- [ ] `en/erisscans`
|
||||
- [ ] `en/ero18x`
|
||||
- [ ] `en/erofus`
|
||||
- [ ] `en/erosscans`
|
||||
- [ ] `en/evascans`
|
||||
- [ ] `en/evilflowers`
|
||||
- [ ] `en/existentialcomics`
|
||||
- [ ] `en/explosm`
|
||||
- [ ] `en/ezmanga`
|
||||
- [ ] `en/fablescans`
|
||||
- [ ] `en/fairyscans`
|
||||
- [ ] `en/firescans`
|
||||
- [ ] `en/flamecomics`
|
||||
- [ ] `en/frierenonline`
|
||||
- [ ] `en/gakamangas`
|
||||
- [ ] `en/galaxydegenscans`
|
||||
- [ ] `en/galaxymanga`
|
||||
- [ ] `en/gedecomix`
|
||||
- [ ] `en/gingertoon`
|
||||
- [ ] `en/goda`
|
||||
- [ ] `en/gourmetscans`
|
||||
- [ ] `en/greedscans`
|
||||
- [ ] `en/grimscans`
|
||||
- [ ] `en/grrlpower`
|
||||
- [ ] `en/gunnerkriggcourt`
|
||||
- [ ] `en/guya` ⚠️ see notes
|
||||
- [ ] `en/gwtb`
|
||||
- [ ] `en/hachirumi`
|
||||
- [ ] `en/hadesscans`
|
||||
- [ ] `en/harimanga`
|
||||
- [ ] `en/hentai3zcc`
|
||||
- [ ] `en/hentai4free`
|
||||
- [ ] `en/hentaidex`
|
||||
- [ ] `en/hentaihere`
|
||||
- [ ] `en/hentaikun`
|
||||
- [ ] `en/hentainexus`
|
||||
- [ ] `en/hentairead`
|
||||
- [ ] `en/hentaireadio`
|
||||
- [ ] `en/hentaisco`
|
||||
- [ ] `en/hentaixcomic`
|
||||
- [ ] `en/hentaixdickgirl`
|
||||
- [ ] `en/hentaixyuri`
|
||||
- [ ] `en/hentara`
|
||||
- [ ] `en/heytoon`
|
||||
- [ ] `en/hijalascans`
|
||||
- [ ] `en/hiperdex`
|
||||
- [ ] `en/hiveworks`
|
||||
- [ ] `en/hm2d`
|
||||
- [ ] `en/honkaiimpact`
|
||||
- [ ] `en/hotcomics`
|
||||
- [ ] `en/hyakuro`
|
||||
- [ ] `en/infernalvoidscans`
|
||||
- [ ] `en/infinityscans`
|
||||
- [ ] `en/irovedout`
|
||||
- [ ] `en/isekaiscantop`
|
||||
- [ ] `en/jinmangas`
|
||||
- [ ] `en/jnovel`
|
||||
- [ ] `en/kagane`
|
||||
- [ ] `en/kaizenscan`
|
||||
- [ ] `en/kaliscancom`
|
||||
- [ ] `en/kaliscanio`
|
||||
- [ ] `en/kaliscanme`
|
||||
- [ ] `en/kappabeast`
|
||||
- [ ] `en/kaynscans`
|
||||
- [ ] `en/keenspot`
|
||||
- [ ] `en/kenscans`
|
||||
- [ ] `en/kewnscans`
|
||||
- [ ] `en/killsixbilliondemons`
|
||||
- [ ] `en/kingcomix`
|
||||
- [ ] `en/kingofshojo`
|
||||
- [ ] `en/kissmangain`
|
||||
- [ ] `en/kmanga`
|
||||
- [ ] `en/kodansha`
|
||||
- [ ] `en/ksgroupscans`
|
||||
- [ ] `en/kunmanga`
|
||||
- [ ] `en/kuramanga`
|
||||
- [ ] `en/lagoonscans`
|
||||
- [ ] `en/leslievictims`
|
||||
- [ ] `en/lhtranslation`
|
||||
- [ ] `en/likemanga`
|
||||
- [ ] `en/likemangain`
|
||||
- [ ] `en/lilymanga`
|
||||
- [ ] `en/linkmanga`
|
||||
- [ ] `en/loadingartist`
|
||||
- [ ] `en/luascans`
|
||||
- [ ] `en/luminaretranslations`
|
||||
- [ ] `en/lunatoons`
|
||||
- [ ] `en/madaradex`
|
||||
- [ ] `en/madarascans`
|
||||
- [ ] `en/madokami`
|
||||
- [ ] `en/magusmanga`
|
||||
- [ ] `en/mahouirexnohentaikarte`
|
||||
- [ ] `en/manga18club`
|
||||
- [ ] `en/manga18free`
|
||||
- [ ] `en/manga18fx`
|
||||
- [ ] `en/manga18x`
|
||||
- [ ] `en/mangabat`
|
||||
- [ ] `en/mangablaze`
|
||||
- [ ] `en/mangabolt`
|
||||
- [ ] `en/mangabtt`
|
||||
- [ ] `en/mangabuddy`
|
||||
- [ ] `en/mangabuddyme`
|
||||
- [ ] `en/mangaclash`
|
||||
- [ ] `en/mangacloud`
|
||||
- [ ] `en/mangacute`
|
||||
- [ ] `en/mangadass`
|
||||
- [ ] `en/mangade`
|
||||
- [ ] `en/mangademon`
|
||||
- [ ] `en/mangadia`
|
||||
- [ ] `en/mangadistrict`
|
||||
- [ ] `en/mangadotnet`
|
||||
- [ ] `en/mangadrama`
|
||||
- [ ] `en/mangafab`
|
||||
- [ ] `en/mangaforest`
|
||||
- [ ] `en/mangaforfreecom`
|
||||
- [ ] `en/mangafox`
|
||||
- [ ] `en/mangafoxfun`
|
||||
- [ ] `en/mangafreak`
|
||||
- [ ] `en/mangafree`
|
||||
- [ ] `en/mangagg`
|
||||
- [ ] `en/mangago`
|
||||
- [ ] `en/mangagofun`
|
||||
- [ ] `en/mangahe`
|
||||
- [ ] `en/mangahen`
|
||||
- [ ] `en/mangahentai`
|
||||
- [ ] `en/mangahere`
|
||||
- [ ] `en/mangahereonl`
|
||||
- [ ] `en/mangahubio`
|
||||
- [ ] `en/mangaka`
|
||||
- [ ] `en/mangakakalot`
|
||||
- [ ] `en/mangakakalotfun`
|
||||
- [ ] `en/mangakatana`
|
||||
- [ ] `en/mangakiss`
|
||||
- [ ] `en/mangamaniacs`
|
||||
- [ ] `en/mangamo`
|
||||
- [ ] `en/mangamob`
|
||||
- [ ] `en/mangamonk`
|
||||
- [ ] `en/manganel`
|
||||
- [ ] `en/manganelo`
|
||||
- [ ] `en/manganow`
|
||||
- [ ] `en/mangaonlinefun`
|
||||
- [ ] `en/mangaowlio`
|
||||
- [ ] `en/mangapandaonl`
|
||||
- [ ] `en/mangapill`
|
||||
- [ ] `en/mangapuma`
|
||||
- [ ] `en/mangarawclub`
|
||||
- [ ] `en/mangaread`
|
||||
- [ ] `en/mangareadercc`
|
||||
- [ ] `en/mangareadersite`
|
||||
- [ ] `en/mangareadorg`
|
||||
- [ ] `en/mangasaga`
|
||||
- [ ] `en/mangasect`
|
||||
- [ ] `en/mangaspin`
|
||||
- [ ] `en/mangasushi`
|
||||
- [ ] `en/mangatellers`
|
||||
- [ ] `en/mangatoday`
|
||||
- [ ] `en/mangatown`
|
||||
- [ ] `en/mangatrend`
|
||||
- [ ] `en/mangatx`
|
||||
- [ ] `en/mangaxyz`
|
||||
- [ ] `en/manhuafast`
|
||||
- [ ] `en/manhuafastnet`
|
||||
- [ ] `en/manhuahot`
|
||||
- [ ] `en/manhuanext`
|
||||
- [ ] `en/manhuanow`
|
||||
- [ ] `en/manhuaplus`
|
||||
- [ ] `en/manhuaplusorg`
|
||||
- [ ] `en/manhuarush`
|
||||
- [ ] `en/manhuascanus`
|
||||
- [ ] `en/manhuasite`
|
||||
- [ ] `en/manhuatop`
|
||||
- [ ] `en/manhuaus`
|
||||
- [ ] `en/manhuazonghe`
|
||||
- [ ] `en/manhwa18`
|
||||
- [ ] `en/manhwa18org`
|
||||
- [ ] `en/manhwa68`
|
||||
- [ ] `en/manhwabuddy`
|
||||
- [ ] `en/manhwaclan`
|
||||
- [ ] `en/manhwacomics`
|
||||
- [ ] `en/manhwaden`
|
||||
- [ ] `en/manhwaget`
|
||||
- [ ] `en/manhwahub`
|
||||
- [ ] `en/manhwajoy`
|
||||
- [ ] `en/manhwalike`
|
||||
- [ ] `en/manhwalover`
|
||||
- [ ] `en/manhwamanhua`
|
||||
- [ ] `en/manhwaread`
|
||||
- [ ] `en/manhwareads`
|
||||
- [ ] `en/manhwatoon`
|
||||
- [ ] `en/manhwatop`
|
||||
- [ ] `en/manhwax`
|
||||
- [ ] `en/manhwaxxl`
|
||||
- [ ] `en/manhwaz`
|
||||
- [ ] `en/manhwazone`
|
||||
- [ ] `en/manta`
|
||||
- [ ] `en/megatokyo`
|
||||
- [ ] `en/mehgazone`
|
||||
- [ ] `en/meitoon`
|
||||
- [ ] `en/mgjinx`
|
||||
- [ ] `en/mgreadio`
|
||||
- [ ] `en/milftoon`
|
||||
- [ ] `en/mistscans`
|
||||
- [ ] `en/mlbblore`
|
||||
- [ ] `en/monochromecustom`
|
||||
- [ ] `en/monochromescans`
|
||||
- [ ] `en/multporn`
|
||||
- [ ] `en/murimscan`
|
||||
- [ ] `en/myhentaicomics`
|
||||
- [ ] `en/myhentaigallery`
|
||||
- [ ] `en/necroscans`
|
||||
- [ ] `en/newmanhwa`
|
||||
- [ ] `en/nexcomic`
|
||||
- [ ] `en/nikatoons`
|
||||
- [ ] `en/nineanime`
|
||||
- [ ] `en/ninehentai`
|
||||
- [ ] `en/ninekon`
|
||||
- [ ] `en/novel24h`
|
||||
- [ ] `en/novelcrow`
|
||||
- [ ] `en/noxenscans`
|
||||
- [ ] `en/nuxscans`
|
||||
- [ ] `en/nyanukafe`
|
||||
- [ ] `en/nyrascans`
|
||||
- [ ] `en/nyxscans`
|
||||
- [ ] `en/octopusmanga`
|
||||
- [ ] `en/oglaf`
|
||||
- [ ] `en/ohjoysextoy`
|
||||
- [ ] `en/omegascans`
|
||||
- [ ] `en/onemangaco`
|
||||
- [ ] `en/onemangainfo`
|
||||
- [ ] `en/onepunchmanonline`
|
||||
- [ ] `en/onlythebesthentai`
|
||||
- [ ] `en/oots`
|
||||
- [ ] `en/oppaistream`
|
||||
- [ ] `en/orchisasia`
|
||||
- [ ] `en/orionscans`
|
||||
- [ ] `en/paradisescans`
|
||||
- [ ] `en/paragonscans`
|
||||
- [ ] `en/paritehaber`
|
||||
- [ ] `en/patchfriday`
|
||||
- [ ] `en/pawmanga`
|
||||
- [ ] `en/petrotechsociety`
|
||||
- [ ] `en/philiascans`
|
||||
- [ ] `en/plutoscans`
|
||||
- [ ] `en/pmscans`
|
||||
- [ ] `en/porncomix`
|
||||
- [ ] `en/qiscans`
|
||||
- [ ] `en/questionablecontent`
|
||||
- [ ] `en/ragescans`
|
||||
- [ ] `en/randowiz`
|
||||
- [ ] `en/ravenscans`
|
||||
- [ ] `en/razure`
|
||||
- [ ] `en/rdscans`
|
||||
- [ ] `en/readallcomicscom`
|
||||
- [ ] `en/readattackontitanshingekinokyojinmanga`
|
||||
- [ ] `en/readberserkmanga`
|
||||
- [ ] `en/readblackclovermangaonline`
|
||||
- [ ] `en/readbokunoheroacademiamyheroacademiamanga`
|
||||
- [ ] `en/readchainsawmanmangaonline`
|
||||
- [ ] `en/readcomiconline`
|
||||
- [ ] `en/readcomicsonline`
|
||||
- [ ] `en/readfairytailedenszeromangaonline`
|
||||
- [ ] `en/readhaikyuumangaonline`
|
||||
- [ ] `en/readjujutsukaisenmangaonline`
|
||||
- [ ] `en/readkingdommangaonline`
|
||||
- [ ] `en/readnanatsunotaizai7deadlysinsmangaonline`
|
||||
- [ ] `en/readnarutoborutosamurai8mangaonline`
|
||||
- [ ] `en/readonepiecemangaonline`
|
||||
- [ ] `en/readonepunchmanmangaonlinetwo`
|
||||
- [ ] `en/readsololevelingmangamanhwaonline`
|
||||
- [ ] `en/readtokyoghoulretokyoghoulmangaonline`
|
||||
- [ ] `en/readvagabondmanga`
|
||||
- [ ] `en/reallifecomics`
|
||||
- [ ] `en/reimanga`
|
||||
- [ ] `en/renascans`
|
||||
- [ ] `en/resetscans`
|
||||
- [ ] `en/restscans`
|
||||
- [ ] `en/retsu`
|
||||
- [ ] `en/revivalscans`
|
||||
- [ ] `en/rinkocomics`
|
||||
- [ ] `en/ritharscans`
|
||||
- [ ] `en/rizzcomic`
|
||||
- [ ] `en/rizzcomicunoriginal`
|
||||
- [ ] `en/rokaricomics`
|
||||
- [ ] `en/roliascan`
|
||||
- [ ] `en/rosesquadscans`
|
||||
- [ ] `en/ryumanga`
|
||||
- [ ] `en/s2manga`
|
||||
- [ ] `en/sanascans`
|
||||
- [ ] `en/saturdaymorningbreakfastcomics`
|
||||
- [ ] `en/schlockmercenary`
|
||||
- [ ] `en/setsuscans`
|
||||
- [ ] `en/shibamanga`
|
||||
- [ ] `en/shojoscans`
|
||||
- [ ] `en/sirenscans`
|
||||
- [ ] `en/skymanga`
|
||||
- [ ] `en/sleepytranslations`
|
||||
- [ ] `en/solarandsundry`
|
||||
- [ ] `en/spmanhwa`
|
||||
- [ ] `en/spyfakku`
|
||||
- [ ] `en/stonescape`
|
||||
- [ ] `en/sunshinebutterflyscans`
|
||||
- [ ] `en/supermega`
|
||||
- [ ] `en/suryascans`
|
||||
- [ ] `en/swordscomic`
|
||||
- [ ] `en/tapastic`
|
||||
- [ ] `en/tcbscans`
|
||||
- [ ] `en/tcbscansunoriginal`
|
||||
- [ ] `en/teamshadowi`
|
||||
- [ ] `en/templescan`
|
||||
- [ ] `en/theblank`
|
||||
- [ ] `en/theduckwebcomics`
|
||||
- [ ] `en/thepropertyofhate`
|
||||
- [ ] `en/timelesstoons`
|
||||
- [ ] `en/todaymanga`
|
||||
- [ ] `en/toon18`
|
||||
- [ ] `en/toongod`
|
||||
- [ ] `en/toonily`
|
||||
- [ ] `en/toonilyme`
|
||||
- [ ] `en/toonitube`
|
||||
- [ ] `en/toonizy`
|
||||
- [ ] `en/topmanhua`
|
||||
- [ ] `en/topmanhuafan`
|
||||
- [ ] `en/topmanhuanet`
|
||||
- [ ] `en/tritiniascans`
|
||||
- [ ] `en/utoon`
|
||||
- [ ] `en/valirscans`
|
||||
- [ ] `en/vanillascans`
|
||||
- [ ] `en/vgperson`
|
||||
- [ ] `en/vizshonenjump`
|
||||
- [ ] `en/voyceme`
|
||||
- [ ] `en/vyvymanga`
|
||||
- [ ] `en/vyvymangaorg`
|
||||
- [ ] `en/warforrayuba`
|
||||
- [ ] `en/wearehunger`
|
||||
- [ ] `en/webcomics`
|
||||
- [ ] `en/webdexscans`
|
||||
- [ ] `en/webnovel`
|
||||
- [ ] `en/webtoonscan`
|
||||
- [ ] `en/webtoonxyz`
|
||||
- [ ] `en/weebcentral`
|
||||
- [ ] `en/whalemanga`
|
||||
- [ ] `en/witchscans`
|
||||
- [ ] `en/woopread`
|
||||
- [ ] `en/writerscans`
|
||||
- [ ] `en/wuxiaworld`
|
||||
- [ ] `en/xlecx`
|
||||
- [ ] `en/xomanga`
|
||||
- [ ] `en/xoxocomics`
|
||||
- [ ] `en/yakshacomics`
|
||||
- [ ] `en/yaoihot`
|
||||
- [ ] `en/yaoihub`
|
||||
- [ ] `en/yaoiscan`
|
||||
- [ ] `en/yaoitoon`
|
||||
- [ ] `en/yorai`
|
||||
- [ ] `en/zazamanga`
|
||||
- [ ] `en/zinchanmanga`
|
||||
- [ ] `en/zinchanmangacom`
|
||||
- [ ] `en/zinmanga`
|
||||
- [ ] `en/zinmanganet`
|
||||
|
||||
---
|
||||
|
||||
## Notes — Complex Sources
|
||||
|
||||
### `all/mangadex` ⚠️
|
||||
- Rate limit: 5 req/s (`golang.org/x/time/rate`)
|
||||
- `GetPopularManga` — `GET /manga?order[rating]=desc&includes[]=cover_art&limit=20&offset={(n-1)*20}`
|
||||
- `GetLatestUpdates` — `GET /manga?order[updatedAt]=desc&includes[]=cover_art`
|
||||
- `GetSearchManga` — tag filters (`includedTags[]`, `excludedTags[]`), demographic, content rating, status, sort
|
||||
- `GetMangaDetails` — `GET /manga/{id}?includes[]=cover_art&includes[]=author&includes[]=artist`; cover URL from relationships
|
||||
- `GetChapterList` — `GET /manga/{id}/feed?translatedLanguage[]=en&limit=500`; paginate until all chapters fetched
|
||||
- `GetPageList` — `GET /at-home/server/{chapterId}` → `{baseUrl}/{quality}/{hash}/{filename}`; quality = `data` or `data-saver`
|
||||
- `GetFilterList` — `GET /manga/tag`; cache tag list
|
||||
- Language configurable via `MANGADEX_LANG` env var (default `en`)
|
||||
|
||||
### `all/nhentaicom` ⚠️
|
||||
- FlareSolverr required
|
||||
- `GetPageList` — extract JSON blob from `document.getElementById(...)` in `<script>` tag; reconstruct URLs as `{imageServer}/galleries/{mediaId}/{pageNum}.{ext}`
|
||||
|
||||
### `all/kemono` ⚠️
|
||||
- FlareSolverr required
|
||||
- Service + creator = manga; posts = chapters; attachment files = pages
|
||||
- File URLs: prefix relative paths with `{base}/data`
|
||||
- `GetChapterList` — paginate in 50-post increments via `?o={offset}`
|
||||
|
||||
### `all/komga` ⚠️
|
||||
- Self-hosted; `KOMGA_BASE_URL`, `KOMGA_USERNAME`, `KOMGA_PASSWORD` env vars; Basic Auth
|
||||
- Series = manga; Books = chapters; pages via `GET /api/v1/books/{bookId}/pages/{n}/thumbnail`
|
||||
|
||||
### `all/e621` ⚠️
|
||||
- Optional Basic Auth (`E621_USERNAME`, `E621_API_KEY`); required for Gold content
|
||||
- Pools = manga; posts = pages; `GET /pools.json`, `GET /posts.json?tags=pool:{id}`
|
||||
|
||||
### `all/danbooru` ⚠️
|
||||
- Optional Basic Auth; free accounts limited to 2 tags per search
|
||||
- Pools = manga; pool post_ids = pages; image URL from `file_url` field
|
||||
|
||||
### `all/pixiv` ⚠️
|
||||
- Auth: `PIXIV_PHPSESSID` env var → Cookie header
|
||||
- `GetPageList` — `GET /ajax/illust/{illustId}/pages`; extract `urls.original`
|
||||
- **Must set `Referer: https://www.pixiv.net/`** on all image requests or get 403
|
||||
|
||||
### `all/luscious` ⚠️
|
||||
- GraphQL POST to `https://members.luscious.net/graphql/playground`
|
||||
- Albums = manga; `SearchAlbumPictures` paginated for page list
|
||||
|
||||
### `all/mangaplus` ⚠️
|
||||
- JSON endpoint: `GET https://jumpg-webapi.tokyo-cdn.com/api/title_detail?title_id={id}`
|
||||
- `GetPageList` — XOR page URL decryption using per-page `encryptionKey` (hex string); implement in `GetImageURL`
|
||||
- Required header: `Origin: https://mangaplus.shueisha.co.jp`
|
||||
|
||||
### `all/stashapp` ⚠️
|
||||
- Self-hosted; `STASHAPP_BASE_URL`, `STASHAPP_API_KEY` env vars
|
||||
- GraphQL POST to `{base}/graphql`; `ApiKey` header
|
||||
|
||||
### `all/globalcomix` ⚠️
|
||||
- Signed CDN URLs with short expiry — **do NOT cache `image_url` in DB**; always fetch fresh page URLs
|
||||
|
||||
### `all/webtoons` ⚠️
|
||||
- HMAC-SHA256 `Sec-Webtoon-Client-Data` header: `HMAC-SHA256(fixedKey, "{url}_{timestamp}")` → base64; fixed key in extension source
|
||||
- Web API: `https://global.apis.naver.com/webtoonSvc/v1/...`
|
||||
|
||||
### `en/allanime` ⚠️
|
||||
- GraphQL POST to `https://api.allanime.day/api`
|
||||
- Episode-based chapters; `episodeString` used as chapter number
|
||||
- Multiple CDN quality tiers in page response; pick highest
|
||||
|
||||
### `en/asurascans` ⚠️
|
||||
- FlareSolverr required
|
||||
- Page selector varies by layout version; check extension source for current selector
|
||||
|
||||
### `en/bakkin` ⚠️
|
||||
- No list/search; all manga from a single JSON URL; enumerate from object keys
|
||||
- `GetSearchManga` does client-side title filtering only
|
||||
|
||||
### `en/bookwalker` ⚠️
|
||||
- DRM-protected; metadata only
|
||||
- `GetPageList` returns empty slice; pages not accessible without purchase
|
||||
|
||||
### `en/guya` ⚠️
|
||||
- `GET {base}/api/get_all_series/` returns all manga at once; no pagination
|
||||
- Scanlation group filter applied client-side in response parsing
|
||||
@@ -0,0 +1,180 @@
|
||||
# Phase 5 — HTTP API (REST)
|
||||
|
||||
Plain REST/JSON API over HTTP using `github.com/go-chi/chi/v5`. No GraphQL on our end.
|
||||
All handlers live in `api/handler.go`. Server bootstrap in `cmd/server/main.go`.
|
||||
|
||||
Reference:
|
||||
- Suwayomi API patterns: `/Users/achmad/Documents/Belajar/Web/Suwayomi-Server/server/src/main/kotlin/suwayomi/tachidesk/manga/api/`
|
||||
|
||||
---
|
||||
|
||||
## 5.1 Server Bootstrap — `cmd/server/main.go`
|
||||
|
||||
- [ ] Parse config from env vars:
|
||||
- [ ] `DATABASE_URL` (required)
|
||||
- [ ] `PORT` (default `8080`)
|
||||
- [ ] `FLARESOLVERR_URL` (optional; disables CF-dependent sources if empty)
|
||||
- [ ] TTL vars: `MANGA_LIST_TTL_SECONDS`, `MANGA_DETAIL_TTL_SECONDS`, `CHAPTER_LIST_TTL_SECONDS`
|
||||
- [ ] Initialize `pgxpool.Pool` via `db.New()`
|
||||
- [ ] Run DB migrations via `db.Migrate()`
|
||||
- [ ] Blank-import all source packages so their `init()` registers them:
|
||||
```go
|
||||
import (
|
||||
_ "tachiyomi-go/sources/all/mangadex"
|
||||
_ "tachiyomi-go/sources/all/kemono"
|
||||
_ "tachiyomi-go/sources/en/allanime"
|
||||
// ... all sources
|
||||
)
|
||||
```
|
||||
- [ ] `UpsertSource` for every registered source on startup (sync source table with compiled sources)
|
||||
- [ ] Mount router and `http.ListenAndServe`
|
||||
- [ ] Graceful shutdown on SIGINT/SIGTERM (drain pool, close server)
|
||||
|
||||
---
|
||||
|
||||
## 5.2 Router — `api/handler.go`
|
||||
|
||||
- [ ] `NewRouter(db *pgxpool.Pool) http.Handler` using chi
|
||||
- [ ] Mount all routes under `/api`
|
||||
- [ ] Middleware:
|
||||
- [ ] `chi.Logger` — request logging
|
||||
- [ ] `chi.Recoverer` — panic recovery → 500 JSON error
|
||||
- [ ] `requestID` middleware
|
||||
- [ ] CORS headers (configurable origin, default `*`)
|
||||
|
||||
---
|
||||
|
||||
## 5.3 Source Routes
|
||||
|
||||
### `GET /api/sources`
|
||||
- [ ] Call `registry.All()`
|
||||
- [ ] Return `[]{id, name, lang, supportsLatest, isNsfw}` sorted by ID
|
||||
- [ ] No DB call required (registry is in-memory)
|
||||
|
||||
### `GET /api/sources/{id}/popular?page=1`
|
||||
- [ ] Parse `id` as int64, `page` as int (default 1), `refresh` as bool
|
||||
- [ ] Validate source exists in registry
|
||||
- [ ] Check manga list TTL (see Phase 2 cache logic)
|
||||
- [ ] If cache miss or `?refresh=true`: call `source.GetPopularManga(page)`, upsert results
|
||||
- [ ] Return `{mangas: [...], hasNextPage: bool}`
|
||||
|
||||
### `GET /api/sources/{id}/latest?page=1`
|
||||
- [ ] Same as popular but calls `source.GetLatestUpdates(page)`
|
||||
- [ ] Return 405 if `source.SupportsLatest()` is false
|
||||
|
||||
### `GET /api/sources/{id}/search?q=&page=1`
|
||||
- [ ] Parse `q` (query string), `page`, `refresh`
|
||||
- [ ] Pass `filters` from query params (see Filter parsing below)
|
||||
- [ ] Call `source.GetSearchManga(page, q, filters)`
|
||||
- [ ] Upsert results, return `{mangas: [...], hasNextPage: bool}`
|
||||
|
||||
### `GET /api/sources/{id}/filters`
|
||||
- [ ] Call `source.GetFilterList()`
|
||||
- [ ] Return filter descriptors as JSON: `[{type, name, values}]`
|
||||
- [ ] No DB involvement
|
||||
|
||||
### Filter Param Parsing
|
||||
- [ ] Parse filter values from query params: `filter_{name}={value}`
|
||||
- [ ] `SelectFilter`: single string value
|
||||
- [ ] `CheckboxFilter`: "true"/"false" → bool
|
||||
- [ ] `TriStateFilter`: "0"/"1"/"2" → int
|
||||
- [ ] `TextFilter`: raw string
|
||||
- [ ] `SortFilter`: `{name}:{asc|desc}`
|
||||
|
||||
---
|
||||
|
||||
## 5.4 Manga Routes (Source-aware)
|
||||
|
||||
### `GET /api/sources/{id}/manga?url={encodedURL}`
|
||||
- [ ] URL-decode `url` param
|
||||
- [ ] Look up manga in DB by `(source_id, url)` — `GetMangaBySourceURL`
|
||||
- [ ] If not found or `initialized=false` or TTL expired or `?refresh=true`:
|
||||
- [ ] Call `source.GetMangaDetails(SManga{URL: url})`
|
||||
- [ ] `UpsertManga` + `UpdateMangaDetails` + set `initialized=true`
|
||||
- [ ] Return full manga JSON (all fields)
|
||||
|
||||
### `GET /api/sources/{id}/manga?url={encodedURL}&resource=chapters`
|
||||
OR `GET /api/sources/{id}/chapters?url={encodedURL}`
|
||||
- [ ] Look up manga_id from DB
|
||||
- [ ] Check `chapters_last_fetched_at` TTL
|
||||
- [ ] If miss: call `source.GetChapterList(manga)`, `UpsertChapter` for each
|
||||
- [ ] Return `[{url, name, chapterNumber, dateUpload, scanlator, sourceOrder}]`
|
||||
|
||||
### `GET /api/sources/{id}/pages?chapterUrl={encodedURL}`
|
||||
- [ ] Look up chapter by URL
|
||||
- [ ] Check if pages exist in DB for chapter
|
||||
- [ ] If miss: call `source.GetPageList(chapter)`, then optionally `source.GetImageURL(page)` for each
|
||||
- [ ] `UpsertPage` for each; store `image_url` (except sources with expiring URLs)
|
||||
- [ ] Return `[{index, url, imageUrl}]`
|
||||
|
||||
---
|
||||
|
||||
## 5.5 DB-Backed Routes (no source call)
|
||||
|
||||
### `GET /api/manga/{id}`
|
||||
- [ ] `GetMangaByID` from DB
|
||||
- [ ] Return manga JSON or 404
|
||||
|
||||
### `GET /api/manga/{id}/chapters`
|
||||
- [ ] `ListChaptersByManga` from DB
|
||||
- [ ] Return chapter array ordered by `source_order`
|
||||
|
||||
### `GET /api/chapters/{id}`
|
||||
- [ ] `GetChapterByID` from DB
|
||||
- [ ] Return chapter JSON or 404
|
||||
|
||||
### `GET /api/chapters/{id}/pages`
|
||||
- [ ] `ListPagesByChapter` from DB
|
||||
- [ ] Return page array ordered by `index`
|
||||
|
||||
---
|
||||
|
||||
## 5.6 Image Proxy — `GET /api/image?url={encodedURL}&source_id={id}`
|
||||
|
||||
- [ ] URL-decode `url` param
|
||||
- [ ] Validate `source_id` exists in registry
|
||||
- [ ] Fetch image using source's configured HTTP client (respects source-specific headers/cookies)
|
||||
- [ ] Set `Referer` if required (e.g. Pixiv)
|
||||
- [ ] Use FlareSolverr-cleared cookies if source needs CF bypass
|
||||
- [ ] Stream response bytes directly to client
|
||||
- [ ] Forward `Content-Type` header from upstream
|
||||
- [ ] Set `Cache-Control: public, max-age=86400` for non-expiring images
|
||||
- [ ] **Do not** set long cache for signed/expiring URLs (globalcomix, etc.)
|
||||
- [ ] Return 502 on upstream error
|
||||
|
||||
---
|
||||
|
||||
## 5.7 Error Handling
|
||||
|
||||
- [ ] All errors: `{"error": "human-readable message"}` with correct HTTP status
|
||||
- [ ] 400: invalid params (bad ID, missing required param, bad page number)
|
||||
- [ ] 404: source not found, manga not found, chapter not found
|
||||
- [ ] 405: method not supported (e.g. `GetLatestUpdates` on source with `SupportsLatest=false`)
|
||||
- [ ] 500: DB error, unexpected source error
|
||||
- [ ] 502: upstream source returned error (connection refused, non-200, parse failure)
|
||||
- [ ] 503: FlareSolverr unavailable (when source requires it but `FLARESOLVERR_URL` is unset)
|
||||
- [ ] Structured error logging: include `source_id`, `url`, `error` in log fields
|
||||
|
||||
---
|
||||
|
||||
## 5.8 `?refresh=true` Bypass
|
||||
|
||||
- [ ] Accepted on all source-facing routes: `/popular`, `/latest`, `/search`, `/manga`, `/chapters`, `/pages`
|
||||
- [ ] Bypasses TTL — always calls source regardless of `last_fetched_at`
|
||||
- [ ] Updates DB with fresh data after fetch
|
||||
- [ ] NOT accepted on DB-only routes (`/api/manga/{id}`, `/api/chapters/{id}`, etc.)
|
||||
|
||||
---
|
||||
|
||||
## Checklist: Phase 5 Done When
|
||||
|
||||
- [ ] `GET /api/sources` returns all registered sources
|
||||
- [ ] `GET /api/sources/{heancmsID}/popular?page=1` returns ≥1 manga; rows in `manga` table confirmed via `psql`
|
||||
- [ ] Second call to same URL returns from DB (no source HTTP call — confirm via log)
|
||||
- [ ] `?refresh=true` forces re-fetch and updates `last_fetched_at` in DB
|
||||
- [ ] `GET /api/sources/{madaraID}/pages?chapterUrl=...` resolves image URLs via FlareSolverr
|
||||
- [ ] `GET /api/image?url=...&source_id=...` proxies image bytes with correct `Content-Type`
|
||||
- [ ] 404 response for unknown source ID: `{"error": "source not found"}`
|
||||
- [ ] 405 response when `GetLatestUpdates` called on source with `SupportsLatest=false`
|
||||
- [ ] `go build ./cmd/server/` succeeds
|
||||
- [ ] Server starts, logs migration version, logs registered source count
|
||||
@@ -0,0 +1,13 @@
|
||||
module goyomi
|
||||
|
||||
go 1.26.2
|
||||
|
||||
require (
|
||||
github.com/PuerkitoBio/goquery v1.12.0
|
||||
golang.org/x/time v0.15.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/andybalholm/cascadia v1.3.3 // indirect
|
||||
golang.org/x/net v0.52.0 // indirect
|
||||
)
|
||||
@@ -0,0 +1,73 @@
|
||||
github.com/PuerkitoBio/goquery v1.12.0 h1:pAcL4g3WRXekcB9AU/y1mbKez2dbY2AajVhtkO8RIBo=
|
||||
github.com/PuerkitoBio/goquery v1.12.0/go.mod h1:802ej+gV2y7bbIhOIoPY5sT183ZW0YFofScC4q/hIpQ=
|
||||
github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM=
|
||||
github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA=
|
||||
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
|
||||
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
|
||||
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
|
||||
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
|
||||
golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
|
||||
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
|
||||
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
|
||||
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
|
||||
golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0=
|
||||
golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
|
||||
golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
|
||||
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
|
||||
golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
|
||||
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
|
||||
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
|
||||
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
|
||||
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
|
||||
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
|
||||
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
|
||||
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
||||
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
|
||||
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
|
||||
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
@@ -0,0 +1,147 @@
|
||||
package httpclient
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/cookiejar"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"golang.org/x/time/rate"
|
||||
)
|
||||
|
||||
const defaultUserAgent = "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Mobile Safari/537.36"
|
||||
|
||||
type Client struct {
|
||||
http *http.Client
|
||||
rateLimit float64
|
||||
burst int
|
||||
referer string
|
||||
|
||||
mu sync.Mutex
|
||||
limiters map[string]*rate.Limiter
|
||||
}
|
||||
|
||||
type Option func(*Client)
|
||||
|
||||
func WithRateLimit(rps float64, burst int) Option {
|
||||
return func(c *Client) {
|
||||
c.rateLimit = rps
|
||||
c.burst = burst
|
||||
}
|
||||
}
|
||||
|
||||
func WithTimeout(d time.Duration) Option {
|
||||
return func(c *Client) { c.http.Timeout = d }
|
||||
}
|
||||
|
||||
func WithReferer(referer string) Option {
|
||||
return func(c *Client) { c.referer = referer }
|
||||
}
|
||||
|
||||
func NewClient(opts ...Option) *Client {
|
||||
jar, _ := cookiejar.New(nil)
|
||||
c := &Client{
|
||||
http: &http.Client{
|
||||
Timeout: 30 * time.Second,
|
||||
Jar: jar,
|
||||
},
|
||||
rateLimit: 1,
|
||||
burst: 1,
|
||||
limiters: map[string]*rate.Limiter{},
|
||||
}
|
||||
for _, o := range opts {
|
||||
o(c)
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
func (c *Client) limiter(host string) *rate.Limiter {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
l, ok := c.limiters[host]
|
||||
if !ok {
|
||||
l = rate.NewLimiter(rate.Limit(c.rateLimit), c.burst)
|
||||
c.limiters[host] = l
|
||||
}
|
||||
return l
|
||||
}
|
||||
|
||||
func (c *Client) Do(req *http.Request) (*http.Response, error) {
|
||||
if err := c.limiter(req.URL.Host).Wait(req.Context()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if c.referer != "" && req.Header.Get("Referer") == "" {
|
||||
req.Header.Set("Referer", c.referer)
|
||||
}
|
||||
if req.Header.Get("User-Agent") == "" {
|
||||
req.Header.Set("User-Agent", defaultUserAgent)
|
||||
}
|
||||
|
||||
const maxRetries = 3
|
||||
for attempt := 0; attempt <= maxRetries; attempt++ {
|
||||
resp, err := c.http.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resp.StatusCode != http.StatusTooManyRequests {
|
||||
return resp, nil
|
||||
}
|
||||
resp.Body.Close()
|
||||
if attempt == maxRetries {
|
||||
return resp, nil
|
||||
}
|
||||
sleep := retryAfter(resp)
|
||||
select {
|
||||
case <-req.Context().Done():
|
||||
return nil, req.Context().Err()
|
||||
case <-time.After(sleep):
|
||||
}
|
||||
}
|
||||
panic("unreachable")
|
||||
}
|
||||
|
||||
func retryAfter(resp *http.Response) time.Duration {
|
||||
ra := resp.Header.Get("Retry-After")
|
||||
if ra == "" {
|
||||
return 5 * time.Second
|
||||
}
|
||||
if secs, err := strconv.ParseFloat(ra, 64); err == nil {
|
||||
return time.Duration(secs * float64(time.Second))
|
||||
}
|
||||
if t, err := http.ParseTime(ra); err == nil {
|
||||
d := time.Until(t)
|
||||
if d > 0 {
|
||||
return d
|
||||
}
|
||||
}
|
||||
return 5 * time.Second
|
||||
}
|
||||
|
||||
func (c *Client) Get(ctx context.Context, url string, headers map[string]string) (*http.Response, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for k, v := range headers {
|
||||
req.Header.Set(k, v)
|
||||
}
|
||||
return c.Do(req)
|
||||
}
|
||||
|
||||
func (c *Client) Post(ctx context.Context, url string, body io.Reader, contentType string, headers map[string]string) (*http.Response, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("Content-Type", contentType)
|
||||
for k, v := range headers {
|
||||
req.Header.Set(k, v)
|
||||
}
|
||||
return c.Do(req)
|
||||
}
|
||||
|
||||
// HTTPClient returns the underlying *http.Client (for passing to graphql helper etc.)
|
||||
func (c *Client) HTTPClient() *http.Client { return c.http }
|
||||
@@ -0,0 +1,95 @@
|
||||
package httpclient
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
)
|
||||
|
||||
type FlareSolverrClient struct {
|
||||
endpoint string
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
func NewFlareSolverrClient() (*FlareSolverrClient, error) {
|
||||
ep := os.Getenv("FLARESOLVERR_URL")
|
||||
if ep == "" {
|
||||
return nil, fmt.Errorf("FLARESOLVERR_URL not set")
|
||||
}
|
||||
return &FlareSolverrClient{
|
||||
endpoint: ep,
|
||||
client: &http.Client{},
|
||||
}, nil
|
||||
}
|
||||
|
||||
type flareSolverrRequest struct {
|
||||
Cmd string `json:"cmd"`
|
||||
URL string `json:"url"`
|
||||
MaxTimeout int `json:"maxTimeout"`
|
||||
}
|
||||
|
||||
type FlareSolverrResponse struct {
|
||||
Status string `json:"status"`
|
||||
Solution struct {
|
||||
Response string `json:"response"`
|
||||
Cookies []fsCookie `json:"cookies"`
|
||||
Headers map[string]any `json:"headers"`
|
||||
URL string `json:"url"`
|
||||
Status int `json:"status"`
|
||||
} `json:"solution"`
|
||||
}
|
||||
|
||||
type fsCookie struct {
|
||||
Name string `json:"name"`
|
||||
Value string `json:"value"`
|
||||
Domain string `json:"domain"`
|
||||
Path string `json:"path"`
|
||||
Expires float64 `json:"expires"`
|
||||
HTTPOnly bool `json:"httpOnly"`
|
||||
Secure bool `json:"secure"`
|
||||
}
|
||||
|
||||
// Get fetches a Cloudflare-protected URL via FlareSolverr.
|
||||
// Returns rendered HTML and extracted cookies.
|
||||
func (f *FlareSolverrClient) Get(ctx context.Context, url string) (html string, cookies []*http.Cookie, err error) {
|
||||
payload, _ := json.Marshal(flareSolverrRequest{
|
||||
Cmd: "request.get",
|
||||
URL: url,
|
||||
MaxTimeout: 60000,
|
||||
})
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, f.endpoint+"/v1", bytes.NewReader(payload))
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
resp, err := f.client.Do(req)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
var fsResp FlareSolverrResponse
|
||||
if err := json.NewDecoder(resp.Body).Decode(&fsResp); err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
if fsResp.Status != "ok" {
|
||||
return "", nil, fmt.Errorf("flaresolverr: status %q", fsResp.Status)
|
||||
}
|
||||
|
||||
for _, c := range fsResp.Solution.Cookies {
|
||||
cookies = append(cookies, &http.Cookie{
|
||||
Name: c.Name,
|
||||
Value: c.Value,
|
||||
Domain: c.Domain,
|
||||
Path: c.Path,
|
||||
HttpOnly: c.HTTPOnly,
|
||||
Secure: c.Secure,
|
||||
})
|
||||
}
|
||||
return fsResp.Solution.Response, cookies, nil
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
package httpclient
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
type GraphQLRequest struct {
|
||||
Query string `json:"query"`
|
||||
Variables any `json:"variables,omitempty"`
|
||||
}
|
||||
|
||||
type graphQLResponse[T any] struct {
|
||||
Data T `json:"data"`
|
||||
Errors []struct {
|
||||
Message string `json:"message"`
|
||||
} `json:"errors"`
|
||||
}
|
||||
|
||||
// Post sends a GraphQL request and unmarshals the `data` field into T.
|
||||
func Post[T any](ctx context.Context, client *http.Client, url string, req GraphQLRequest, headers map[string]string) (T, error) {
|
||||
var zero T
|
||||
body, err := json.Marshal(req)
|
||||
if err != nil {
|
||||
return zero, err
|
||||
}
|
||||
|
||||
httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return zero, err
|
||||
}
|
||||
httpReq.Header.Set("Content-Type", "application/json")
|
||||
httpReq.Header.Set("Accept", "application/json")
|
||||
for k, v := range headers {
|
||||
httpReq.Header.Set(k, v)
|
||||
}
|
||||
|
||||
resp, err := client.Do(httpReq)
|
||||
if err != nil {
|
||||
return zero, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
var gqlResp graphQLResponse[T]
|
||||
if err := json.NewDecoder(resp.Body).Decode(&gqlResp); err != nil {
|
||||
return zero, err
|
||||
}
|
||||
if len(gqlResp.Errors) > 0 {
|
||||
return zero, fmt.Errorf("graphql: %s", gqlResp.Errors[0].Message)
|
||||
}
|
||||
return gqlResp.Data, nil
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
package httpclient
|
||||
|
||||
const (
|
||||
androidUA = "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Mobile Safari/537.36"
|
||||
desktopUA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
|
||||
)
|
||||
|
||||
func AndroidUA() string { return androidUA }
|
||||
func DesktopUA() string { return desktopUA }
|
||||
|
||||
func JSONHeaders() map[string]string {
|
||||
return map[string]string{
|
||||
"Content-Type": "application/json",
|
||||
"Accept": "application/json",
|
||||
}
|
||||
}
|
||||
|
||||
func FormHeaders() map[string]string {
|
||||
return map[string]string{
|
||||
"Content-Type": "application/x-www-form-urlencoded",
|
||||
}
|
||||
}
|
||||
|
||||
func WithRefererHeader(headers map[string]string, referer string) map[string]string {
|
||||
out := clone(headers)
|
||||
out["Referer"] = referer
|
||||
return out
|
||||
}
|
||||
|
||||
func WithOrigin(headers map[string]string, origin string) map[string]string {
|
||||
out := clone(headers)
|
||||
out["Origin"] = origin
|
||||
return out
|
||||
}
|
||||
|
||||
func clone(m map[string]string) map[string]string {
|
||||
out := make(map[string]string, len(m)+1)
|
||||
for k, v := range m {
|
||||
out[k] = v
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
package parser
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
)
|
||||
|
||||
func Parse(html string) (*goquery.Document, error) {
|
||||
return goquery.NewDocumentFromReader(strings.NewReader(html))
|
||||
}
|
||||
|
||||
func ParseResponse(resp *http.Response) (*goquery.Document, error) {
|
||||
defer resp.Body.Close()
|
||||
return goquery.NewDocumentFromReader(resp.Body)
|
||||
}
|
||||
|
||||
func Select(doc *goquery.Document, css string) *goquery.Selection {
|
||||
return doc.Find(css)
|
||||
}
|
||||
|
||||
func SelectFrom(sel *goquery.Selection, css string) *goquery.Selection {
|
||||
return sel.Find(css)
|
||||
}
|
||||
|
||||
func Attr(sel *goquery.Selection, name string) string {
|
||||
val, _ := sel.Attr(name)
|
||||
return val
|
||||
}
|
||||
|
||||
// AbsURL resolves a relative URL attribute against baseURL.
|
||||
func AbsURL(sel *goquery.Selection, attr string, baseURL string) string {
|
||||
val := Attr(sel, attr)
|
||||
if val == "" {
|
||||
return ""
|
||||
}
|
||||
base, err := url.Parse(baseURL)
|
||||
if err != nil {
|
||||
return val
|
||||
}
|
||||
ref, err := url.Parse(val)
|
||||
if err != nil {
|
||||
return val
|
||||
}
|
||||
return base.ResolveReference(ref).String()
|
||||
}
|
||||
|
||||
// OwnText returns the text content of the element excluding child elements.
|
||||
func OwnText(sel *goquery.Selection) string {
|
||||
clone := sel.Clone()
|
||||
clone.Children().Remove()
|
||||
return strings.TrimSpace(clone.Text())
|
||||
}
|
||||
|
||||
func TextTrim(sel *goquery.Selection) string {
|
||||
return strings.TrimSpace(sel.Text())
|
||||
}
|
||||
|
||||
func First(sel *goquery.Selection) *goquery.Selection {
|
||||
return sel.First()
|
||||
}
|
||||
|
||||
func Each(sel *goquery.Selection, fn func(i int, s *goquery.Selection)) {
|
||||
sel.Each(fn)
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
package registry
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"sync"
|
||||
|
||||
"goyomi/internal/source"
|
||||
)
|
||||
|
||||
var (
|
||||
mu sync.RWMutex
|
||||
sources = map[int64]source.CatalogueSource{}
|
||||
)
|
||||
|
||||
// Register adds a source. Panics on duplicate ID — caught at startup.
|
||||
func Register(s source.CatalogueSource) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if _, exists := sources[s.ID()]; exists {
|
||||
panic(fmt.Sprintf("registry: duplicate source ID %d (%s/%s)", s.ID(), s.Name(), s.Lang()))
|
||||
}
|
||||
sources[s.ID()] = s
|
||||
}
|
||||
|
||||
func Get(id int64) (source.CatalogueSource, bool) {
|
||||
mu.RLock()
|
||||
defer mu.RUnlock()
|
||||
s, ok := sources[id]
|
||||
return s, ok
|
||||
}
|
||||
|
||||
// All returns all registered sources sorted by ID.
|
||||
func All() []source.CatalogueSource {
|
||||
mu.RLock()
|
||||
defer mu.RUnlock()
|
||||
out := make([]source.CatalogueSource, 0, len(sources))
|
||||
for _, s := range sources {
|
||||
out = append(out, s)
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].ID() < out[j].ID() })
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
package registry_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"goyomi/internal/registry"
|
||||
"goyomi/internal/source"
|
||||
)
|
||||
|
||||
type mockSource struct {
|
||||
id int64
|
||||
name string
|
||||
lang string
|
||||
}
|
||||
|
||||
func (m *mockSource) ID() int64 { return m.id }
|
||||
func (m *mockSource) Name() string { return m.name }
|
||||
func (m *mockSource) Lang() string { return m.lang }
|
||||
func (m *mockSource) SupportsLatest() bool { return false }
|
||||
func (m *mockSource) GetPopularManga(page int) (source.MangasPage, error) { return source.MangasPage{}, nil }
|
||||
func (m *mockSource) GetLatestUpdates(page int) (source.MangasPage, error) { return source.MangasPage{}, nil }
|
||||
func (m *mockSource) GetSearchManga(page int, query string, filters []source.Filter) (source.MangasPage, error) {
|
||||
return source.MangasPage{}, nil
|
||||
}
|
||||
func (m *mockSource) GetMangaDetails(manga source.SManga) (source.SManga, error) { return manga, nil }
|
||||
func (m *mockSource) GetChapterList(manga source.SManga) ([]source.SChapter, error) { return nil, nil }
|
||||
func (m *mockSource) GetPageList(chapter source.SChapter) ([]source.Page, error) { return nil, nil }
|
||||
func (m *mockSource) GetImageURL(page source.Page) (string, error) { return page.ImageURL, nil }
|
||||
func (m *mockSource) GetFilterList() []source.Filter { return nil }
|
||||
|
||||
func TestDuplicateIDPanics(t *testing.T) {
|
||||
defer func() {
|
||||
if r := recover(); r == nil {
|
||||
t.Error("expected panic on duplicate source ID, got none")
|
||||
}
|
||||
}()
|
||||
registry.Register(&mockSource{id: 9999, name: "A", lang: "en"})
|
||||
registry.Register(&mockSource{id: 9999, name: "B", lang: "en"})
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
package source
|
||||
|
||||
import (
|
||||
"crypto/md5"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Source is the base interface for all sources.
|
||||
type Source interface {
|
||||
ID() int64
|
||||
Name() string
|
||||
Lang() string
|
||||
}
|
||||
|
||||
// CatalogueSource is the full interface every source must implement.
|
||||
type CatalogueSource interface {
|
||||
Source
|
||||
SupportsLatest() bool
|
||||
GetPopularManga(page int) (MangasPage, error)
|
||||
GetLatestUpdates(page int) (MangasPage, error)
|
||||
GetSearchManga(page int, query string, filters []Filter) (MangasPage, error)
|
||||
GetMangaDetails(manga SManga) (SManga, error)
|
||||
GetChapterList(manga SManga) ([]SChapter, error)
|
||||
GetPageList(chapter SChapter) ([]Page, error)
|
||||
// GetImageURL resolves the final image URL for a page.
|
||||
// Sources that embed image URLs directly in pages return page.ImageURL unchanged.
|
||||
GetImageURL(page Page) (string, error)
|
||||
GetFilterList() []Filter
|
||||
}
|
||||
|
||||
// GenerateSourceID replicates Tachiyomi/Suwayomi HttpSource.generateId:
|
||||
//
|
||||
// key = "${name.lowercase()}/$lang/$versionId"
|
||||
// MD5(key) → first 8 bytes as big-endian int64, sign bit cleared (& Long.MAX_VALUE)
|
||||
func GenerateSourceID(name, lang string) int64 {
|
||||
return GenerateSourceIDv(name, lang, 1)
|
||||
}
|
||||
|
||||
func GenerateSourceIDv(name, lang string, versionID int) int64 {
|
||||
key := strings.ToLower(name) + "/" + lang + "/" + itoa(versionID)
|
||||
b := md5.Sum([]byte(key))
|
||||
var id int64
|
||||
for i := 0; i < 8; i++ {
|
||||
id |= int64(b[i]) << (8 * (7 - i))
|
||||
}
|
||||
return id & int64(^uint64(0)>>1) // clear sign bit (& Long.MAX_VALUE)
|
||||
}
|
||||
|
||||
func itoa(n int) string {
|
||||
if n == 0 {
|
||||
return "0"
|
||||
}
|
||||
digits := []byte{}
|
||||
for n > 0 {
|
||||
digits = append([]byte{byte('0' + n%10)}, digits...)
|
||||
n /= 10
|
||||
}
|
||||
return string(digits)
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
package source_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"goyomi/internal/source"
|
||||
)
|
||||
|
||||
func TestGenerateSourceID(t *testing.T) {
|
||||
// IDs computed from the same MD5 formula as Tachiyomi/Suwayomi HttpSource.generateId:
|
||||
// key = "${name.lowercase()}/$lang/1", MD5 → first 8 bytes big-endian, sign bit cleared.
|
||||
cases := []struct {
|
||||
name string
|
||||
lang string
|
||||
want int64
|
||||
}{
|
||||
{"MangaDex", "en", 2499283573021220255},
|
||||
{"MangaDex", "all", 6404943692147160087},
|
||||
{"HeanCms", "en", 6473152836656709188},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
got := source.GenerateSourceID(tc.name, tc.lang)
|
||||
if got != tc.want {
|
||||
t.Errorf("GenerateSourceID(%q, %q) = %d, want %d", tc.name, tc.lang, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
package source
|
||||
|
||||
const (
|
||||
StatusUnknown = 0
|
||||
StatusOngoing = 1
|
||||
StatusCompleted = 2
|
||||
StatusLicensed = 3
|
||||
StatusHiatus = 5
|
||||
StatusCancelled = 6
|
||||
)
|
||||
|
||||
type SManga struct {
|
||||
URL string
|
||||
Title string
|
||||
Artist string
|
||||
Author string
|
||||
Description string
|
||||
Genre string // comma-separated
|
||||
Status int
|
||||
ThumbnailURL string
|
||||
Initialized bool
|
||||
}
|
||||
|
||||
type SChapter struct {
|
||||
URL string
|
||||
Name string
|
||||
DateUpload int64 // unix milliseconds
|
||||
ChapterNumber float32
|
||||
Scanlator string
|
||||
}
|
||||
|
||||
type Page struct {
|
||||
Index int
|
||||
URL string
|
||||
ImageURL string
|
||||
}
|
||||
|
||||
type MangasPage struct {
|
||||
Mangas []SManga
|
||||
HasNextPage bool
|
||||
}
|
||||
|
||||
// Filter interface
|
||||
|
||||
type Filter interface {
|
||||
Name() string
|
||||
Value() any
|
||||
}
|
||||
|
||||
// TextFilter — free-text input
|
||||
|
||||
type TextFilter struct {
|
||||
FilterName string
|
||||
Text string
|
||||
}
|
||||
|
||||
func (f *TextFilter) Name() string { return f.FilterName }
|
||||
func (f *TextFilter) Value() any { return f.Text }
|
||||
|
||||
// CheckboxFilter — boolean
|
||||
|
||||
type CheckboxFilter struct {
|
||||
FilterName string
|
||||
State bool
|
||||
}
|
||||
|
||||
func (f *CheckboxFilter) Name() string { return f.FilterName }
|
||||
func (f *CheckboxFilter) Value() any { return f.State }
|
||||
|
||||
// TriStateFilter — 0=ignore, 1=include, 2=exclude
|
||||
|
||||
type TriStateFilter struct {
|
||||
FilterName string
|
||||
State int
|
||||
}
|
||||
|
||||
func (f *TriStateFilter) Name() string { return f.FilterName }
|
||||
func (f *TriStateFilter) Value() any { return f.State }
|
||||
|
||||
// SelectFilter — dropdown
|
||||
|
||||
type SelectFilter struct {
|
||||
FilterName string
|
||||
Values []string
|
||||
Selected int
|
||||
}
|
||||
|
||||
func (f *SelectFilter) Name() string { return f.FilterName }
|
||||
func (f *SelectFilter) Value() any { return f.Selected }
|
||||
|
||||
// SortFilter
|
||||
|
||||
type SortSelection struct {
|
||||
Index int
|
||||
Ascending bool
|
||||
}
|
||||
|
||||
type SortFilter struct {
|
||||
FilterName string
|
||||
Values []string
|
||||
Selection SortSelection
|
||||
}
|
||||
|
||||
func (f *SortFilter) Name() string { return f.FilterName }
|
||||
func (f *SortFilter) Value() any { return f.Selection }
|
||||
|
||||
// GroupFilter — container of sub-filters
|
||||
|
||||
type GroupFilter struct {
|
||||
FilterName string
|
||||
Filters []Filter
|
||||
}
|
||||
|
||||
func (f *GroupFilter) Name() string { return f.FilterName }
|
||||
func (f *GroupFilter) Value() any { return f.Filters }
|
||||
Reference in New Issue
Block a user