diff --git a/sources/base/bakkin/bakkin.go b/sources/base/bakkin/bakkin.go new file mode 100644 index 0000000..396acdf --- /dev/null +++ b/sources/base/bakkin/bakkin.go @@ -0,0 +1,122 @@ +// Package bakkin implements the Bakkin base. +// No list/search — all manga from a single JSON URL, client-side title filter. +package bakkin + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + + "goyomi/internal/httpclient" + "goyomi/internal/source" + "goyomi/sources/base/util" +) + +type Config struct { + Name string + BaseURL string + Lang string + JSONURL string // URL of the source JSON +} + +type Source struct { + cfg Config + client *httpclient.Client + id int64 +} + +func New(cfg Config) *Source { + if cfg.JSONURL == "" { + cfg.JSONURL = strings.TrimRight(cfg.BaseURL, "/") + "/manga.json" + } + c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) + return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} +} + +func (s *Source) ID() int64 { return s.id } +func (s *Source) Name() string { return s.cfg.Name } +func (s *Source) Lang() string { return s.cfg.Lang } +func (s *Source) SupportsLatest() bool { return false } + +func (s *Source) fetchAll(ctx context.Context) ([]source.SManga, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, s.cfg.JSONURL, nil) + if err != nil { + return nil, err + } + resp, err := s.client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("bakkin: HTTP %d", resp.StatusCode) + } + body, _ := io.ReadAll(resp.Body) + var raw map[string]json.RawMessage + if err := json.Unmarshal(body, &raw); err != nil { + return nil, err + } + var mangas []source.SManga + for key := range raw { + m := source.SManga{ + URL: fmt.Sprintf("/%s", key), + Title: key, + } + var details map[string]any + if err := json.Unmarshal(raw[key], &details); err == nil { + if title, ok := details["title"].(string); ok { + m.Title = title + } + if thumb, ok := details["thumbnail"].(string); ok { + m.ThumbnailURL = util.AbsURL(s.cfg.BaseURL, thumb) + } + } + mangas = append(mangas, m) + } + return mangas, nil +} + +func (s *Source) GetPopularManga(page int) (source.MangasPage, error) { + if page > 1 { + return source.MangasPage{}, nil + } + mangas, err := s.fetchAll(context.Background()) + return source.MangasPage{Mangas: mangas, HasNextPage: false}, err +} + +func (s *Source) GetLatestUpdates(page int) (source.MangasPage, error) { + return s.GetPopularManga(page) +} + +func (s *Source) GetSearchManga(page int, query string, filters []source.Filter) (source.MangasPage, error) { + mangas, err := s.fetchAll(context.Background()) + if err != nil { + return source.MangasPage{}, err + } + q := strings.ToLower(query) + var matched []source.SManga + for _, m := range mangas { + if strings.Contains(strings.ToLower(m.Title), q) { + matched = append(matched, m) + } + } + return source.MangasPage{Mangas: matched, HasNextPage: false}, nil +} + +func (s *Source) GetMangaDetails(manga source.SManga) (source.SManga, error) { + return manga, nil +} + +func (s *Source) GetChapterList(manga source.SManga) ([]source.SChapter, error) { + return nil, nil +} + +func (s *Source) GetPageList(chapter source.SChapter) ([]source.Page, error) { + return nil, nil +} + +func (s *Source) GetImageURL(page source.Page) (string, error) { return page.ImageURL, nil } +func (s *Source) GetFilterList() []source.Filter { return nil } diff --git a/sources/base/fmreader/fmreader.go b/sources/base/fmreader/fmreader.go new file mode 100644 index 0000000..cc62d9a --- /dev/null +++ b/sources/base/fmreader/fmreader.go @@ -0,0 +1,204 @@ +// Package fmreader implements the Flat-Manga CMS base (FMReader). +// GET {base}/{requestPath}?listType=pagination&page={n}&sort=views +package fmreader + +import ( + "context" + "fmt" + "net/http" + "net/url" + "strings" + + "github.com/PuerkitoBio/goquery" + + "goyomi/internal/httpclient" + "goyomi/internal/source" + "goyomi/sources/base/util" +) + +type Config struct { + Name string + BaseURL string + Lang string + RequestPath string // default "manga-list.html" +} + +type Source struct { + cfg Config + client *httpclient.Client + id int64 +} + +func New(cfg Config) *Source { + if cfg.RequestPath == "" { + cfg.RequestPath = "manga-list.html" + } + c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) + return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} +} + +func (s *Source) ID() int64 { return s.id } +func (s *Source) Name() string { return s.cfg.Name } +func (s *Source) Lang() string { return s.cfg.Lang } +func (s *Source) SupportsLatest() bool { return true } + +func (s *Source) get(ctx context.Context, rawURL string) (*goquery.Document, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) + if err != nil { + return nil, err + } + resp, err := s.client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("fmreader: HTTP %d", resp.StatusCode) + } + return goquery.NewDocumentFromReader(resp.Body) +} + +func (s *Source) listURL(page int, sort, query string) string { + base := strings.TrimRight(s.cfg.BaseURL, "/") + u, _ := url.Parse(fmt.Sprintf("%s/%s", base, s.cfg.RequestPath)) + q := u.Query() + q.Set("listType", "pagination") + q.Set("page", fmt.Sprintf("%d", page)) + q.Set("sort", sort) + q.Set("sort_type", "DESC") + if query != "" { + q.Set("name", query) + } + u.RawQuery = q.Encode() + return u.String() +} + +func (s *Source) parseMangaList(doc *goquery.Document) source.MangasPage { + var mangas []source.SManga + doc.Find("div.media, .thumb-item-flow").Each(func(_ int, el *goquery.Selection) { + m := source.SManga{} + el.Find("a").First().Each(func(_ int, a *goquery.Selection) { + if href, ok := a.Attr("href"); ok { + m.URL = stripDomain(href, s.cfg.BaseURL) + } + m.Title = strings.TrimSpace(a.AttrOr("title", a.Text())) + }) + el.Find("img").First().Each(func(_ int, img *goquery.Selection) { + m.ThumbnailURL = imgAttr(img, s.cfg.BaseURL) + }) + if m.URL != "" { + mangas = append(mangas, m) + } + }) + // FMReader uses "page x of y" text or standard next pagination + hasNext := doc.Find("div.col-lg-9 button.btn-info, .pagination a:contains(»):not(.disabled)").Length() > 0 + return source.MangasPage{Mangas: mangas, HasNextPage: hasNext} +} + +func (s *Source) GetPopularManga(page int) (source.MangasPage, error) { + doc, err := s.get(context.Background(), s.listURL(page, "views", "")) + if err != nil { + return source.MangasPage{}, err + } + return s.parseMangaList(doc), nil +} + +func (s *Source) GetLatestUpdates(page int) (source.MangasPage, error) { + doc, err := s.get(context.Background(), s.listURL(page, "last_update", "")) + if err != nil { + return source.MangasPage{}, err + } + return s.parseMangaList(doc), nil +} + +func (s *Source) GetSearchManga(page int, query string, filters []source.Filter) (source.MangasPage, error) { + doc, err := s.get(context.Background(), s.listURL(page, "views", query)) + if err != nil { + return source.MangasPage{}, err + } + return s.parseMangaList(doc), nil +} + +func (s *Source) GetMangaDetails(manga source.SManga) (source.SManga, error) { + doc, err := s.get(context.Background(), util.AbsURL(s.cfg.BaseURL, manga.URL)) + if err != nil { + return manga, err + } + result := source.SManga{URL: manga.URL} + doc.Find("h1, h2.title").First().Each(func(_ int, el *goquery.Selection) { result.Title = strings.TrimSpace(el.Text()) }) + doc.Find("li a.btn-info").First().Each(func(_ int, el *goquery.Selection) { result.Author = strings.TrimSpace(el.Text()) }) + doc.Find("div.description-update, .manga-description").First().Each(func(_ int, el *goquery.Selection) { result.Description = strings.TrimSpace(el.Text()) }) + doc.Find(".info-image img").First().Each(func(_ int, img *goquery.Selection) { result.ThumbnailURL = imgAttr(img, s.cfg.BaseURL) }) + doc.Find("li a.btn-success").First().Each(func(_ int, el *goquery.Selection) { result.Status = util.StatusFromString(el.Text()) }) + var genres []string + doc.Find("li a.btn-danger").Each(func(_ int, el *goquery.Selection) { + if t := strings.TrimSpace(el.Text()); t != "" { + genres = append(genres, t) + } + }) + result.Genre = strings.Join(genres, ", ") + return result, nil +} + +func (s *Source) GetChapterList(manga source.SManga) ([]source.SChapter, error) { + doc, err := s.get(context.Background(), util.AbsURL(s.cfg.BaseURL, manga.URL)) + if err != nil { + return nil, err + } + var chapters []source.SChapter + doc.Find("div.row-content-chapter li, #chapterList li").Each(func(_ int, el *goquery.Selection) { + ch := source.SChapter{} + el.Find("a").First().Each(func(_ int, a *goquery.Selection) { + if href, ok := a.Attr("href"); ok { + ch.URL = stripDomain(href, s.cfg.BaseURL) + } + ch.Name = strings.TrimSpace(a.Text()) + }) + el.Find(".chapter-time, .time").First().Each(func(_ int, e *goquery.Selection) { + ch.DateUpload = util.ParseRelativeDate(e.Text()) + }) + if ch.URL != "" { + chapters = append(chapters, ch) + } + }) + return chapters, nil +} + +func (s *Source) GetPageList(chapter source.SChapter) ([]source.Page, error) { + rawURL := util.AbsURL(s.cfg.BaseURL, chapter.URL) + doc, err := s.get(context.Background(), rawURL) + if err != nil { + return nil, err + } + var pages []source.Page + doc.Find(".reading-detail img, #chapter-content img").Each(func(i int, img *goquery.Selection) { + if u := imgAttr(img, s.cfg.BaseURL); u != "" { + pages = append(pages, source.Page{Index: i, URL: rawURL, ImageURL: u}) + } + }) + return pages, nil +} + +func (s *Source) GetImageURL(page source.Page) (string, error) { return page.ImageURL, nil } +func (s *Source) GetFilterList() []source.Filter { return nil } + +func stripDomain(href, baseURL string) string { + parsed, err := url.Parse(href) + if err != nil || !parsed.IsAbs() { + return href + } + base, _ := url.Parse(baseURL) + if base != nil && parsed.Host == base.Host { + return parsed.RequestURI() + } + return href +} + +func imgAttr(img *goquery.Selection, baseURL string) string { + for _, attr := range []string{"data-bg", "data-lazy-src", "data-src", "src"} { + if v, ok := img.Attr(attr); ok && v != "" && !strings.HasPrefix(v, "data:") { + return util.AbsURL(baseURL, v) + } + } + return "" +} diff --git a/sources/base/foolslide/foolslide.go b/sources/base/foolslide/foolslide.go new file mode 100644 index 0000000..b99f1ae --- /dev/null +++ b/sources/base/foolslide/foolslide.go @@ -0,0 +1,199 @@ +// Package foolslide implements the FoolSlide reader base. +// Popular: HTML GET {base}/directory/{n}/ +// Chapters/Pages: JSON API GET {base}/api/reader/... +package foolslide + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + + "github.com/PuerkitoBio/goquery" + + "goyomi/internal/httpclient" + "goyomi/internal/source" + "goyomi/sources/base/util" +) + +type Config struct { + Name string + BaseURL string + Lang string + URLModifier string // appended to BaseURL before /directory/ +} + +type Source struct { + cfg Config + client *httpclient.Client + id int64 +} + +func New(cfg Config) *Source { + c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) + return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} +} + +func (s *Source) ID() int64 { return s.id } +func (s *Source) Name() string { return s.cfg.Name } +func (s *Source) Lang() string { return s.cfg.Lang } +func (s *Source) SupportsLatest() bool { return true } + +func (s *Source) base() string { + return strings.TrimRight(s.cfg.BaseURL, "/") + s.cfg.URLModifier +} + +func (s *Source) get(ctx context.Context, rawURL string) (*http.Response, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) + if err != nil { + return nil, err + } + resp, err := s.client.Do(req) + if err != nil { + return nil, err + } + if resp.StatusCode != http.StatusOK { + resp.Body.Close() + return nil, fmt.Errorf("foolslide: HTTP %d", resp.StatusCode) + } + return resp, nil +} + +func (s *Source) getDoc(ctx context.Context, rawURL string) (*goquery.Document, error) { + resp, err := s.get(ctx, rawURL) + if err != nil { + return nil, err + } + defer resp.Body.Close() + return goquery.NewDocumentFromReader(resp.Body) +} + +func (s *Source) getJSON(ctx context.Context, rawURL string, out any) error { + resp, err := s.get(ctx, rawURL) + if err != nil { + return err + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + return json.Unmarshal(body, out) +} + +func (s *Source) parseMangaList(doc *goquery.Document) source.MangasPage { + var mangas []source.SManga + doc.Find("div.group").Each(func(_ int, el *goquery.Selection) { + m := source.SManga{} + el.Find("a[title]").First().Each(func(_ int, a *goquery.Selection) { + if href, ok := a.Attr("href"); ok { + m.URL = href + } + m.Title = a.AttrOr("title", strings.TrimSpace(a.Text())) + }) + el.Find("img").First().Each(func(_ int, img *goquery.Selection) { + m.ThumbnailURL = util.AbsURL(s.cfg.BaseURL, img.AttrOr("src", "")) + }) + if m.URL != "" { + mangas = append(mangas, m) + } + }) + hasNext := doc.Find("a.next, .next_page").Length() > 0 + return source.MangasPage{Mangas: mangas, HasNextPage: hasNext} +} + +func (s *Source) GetPopularManga(page int) (source.MangasPage, error) { + doc, err := s.getDoc(context.Background(), fmt.Sprintf("%s/directory/%d/", s.base(), page)) + if err != nil { + return source.MangasPage{}, err + } + return s.parseMangaList(doc), nil +} + +func (s *Source) GetLatestUpdates(page int) (source.MangasPage, error) { + doc, err := s.getDoc(context.Background(), fmt.Sprintf("%s/latest/%d/", s.base(), page)) + if err != nil { + return source.MangasPage{}, err + } + return s.parseMangaList(doc), nil +} + +func (s *Source) GetSearchManga(page int, query string, filters []source.Filter) (source.MangasPage, error) { + doc, err := s.getDoc(context.Background(), fmt.Sprintf("%s/search/?q=%s&p=%d", s.base(), query, page)) + if err != nil { + return source.MangasPage{}, err + } + return s.parseMangaList(doc), nil +} + +func (s *Source) GetMangaDetails(manga source.SManga) (source.SManga, error) { + doc, err := s.getDoc(context.Background(), manga.URL) + if err != nil { + return manga, err + } + result := source.SManga{URL: manga.URL} + doc.Find("h1.title").First().Each(func(_ int, el *goquery.Selection) { result.Title = strings.TrimSpace(el.Text()) }) + doc.Find(".comic_cover img, img.cover").First().Each(func(_ int, img *goquery.Selection) { + result.ThumbnailURL = util.AbsURL(s.cfg.BaseURL, img.AttrOr("src", "")) + }) + return result, nil +} + +type foolSlideChapter struct { + Stub string `json:"stub"` + Chapter string `json:"chapter"` + Subchapter string `json:"subchapter"` + Name string `json:"name"` + Added string `json:"added"` +} + +func (s *Source) GetChapterList(manga source.SManga) ([]source.SChapter, error) { + slug := util.SlugFromURL(strings.TrimRight(manga.URL, "/")) + apiURL := fmt.Sprintf("%s/api/reader/chapters?comic=%s", s.base(), slug) + var chapters []foolSlideChapter + if err := s.getJSON(context.Background(), apiURL, &chapters); err != nil { + return nil, err + } + result := make([]source.SChapter, len(chapters)) + for i, ch := range chapters { + chNum := ch.Chapter + if ch.Subchapter != "" && ch.Subchapter != "0" { + chNum += "." + ch.Subchapter + } + result[i] = source.SChapter{ + URL: fmt.Sprintf("%s/reader/series/%s/%s/", s.base(), slug, chNum), + Name: ch.Name, + DateUpload: util.ParseAbsoluteDate(ch.Added, "2006-01-02 15:04:05"), + } + } + return result, nil +} + +type foolSlidePage struct { + Filename string `json:"filename"` + URL string `json:"url"` +} + +func (s *Source) GetPageList(chapter source.SChapter) ([]source.Page, error) { + // Extract chapter ID from URL or use the chapter URL directly for the API + slug := util.SlugFromURL(strings.TrimRight(chapter.URL, "/")) + apiURL := fmt.Sprintf("%s/api/reader/images?chapter=%s", s.base(), slug) + var pages []foolSlidePage + if err := s.getJSON(context.Background(), apiURL, &pages); err != nil { + return nil, err + } + result := make([]source.Page, len(pages)) + for i, p := range pages { + imgURL := p.URL + if imgURL == "" { + imgURL = p.Filename + } + result[i] = source.Page{Index: i, URL: chapter.URL, ImageURL: util.AbsURL(s.cfg.BaseURL, imgURL)} + } + return result, nil +} + +func (s *Source) GetImageURL(page source.Page) (string, error) { return page.ImageURL, nil } +func (s *Source) GetFilterList() []source.Filter { return nil } diff --git a/sources/base/gigaviewer/gigaviewer.go b/sources/base/gigaviewer/gigaviewer.go new file mode 100644 index 0000000..7e69e4e --- /dev/null +++ b/sources/base/gigaviewer/gigaviewer.go @@ -0,0 +1,168 @@ +// Package gigaviewer implements the GigaViewer (Hatena) base. +// GET {base}/series returns all manga; no pagination. +package gigaviewer + +import ( + "context" + "fmt" + "net/http" + "strings" + + "github.com/PuerkitoBio/goquery" + + "goyomi/internal/httpclient" + "goyomi/internal/source" + "goyomi/sources/base/util" +) + +type Config struct { + Name string + BaseURL string + Lang string +} + +type Source struct { + cfg Config + client *httpclient.Client + id int64 +} + +func New(cfg Config) *Source { + c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) + return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} +} + +func (s *Source) ID() int64 { return s.id } +func (s *Source) Name() string { return s.cfg.Name } +func (s *Source) Lang() string { return s.cfg.Lang } +func (s *Source) SupportsLatest() bool { return true } + +func (s *Source) get(ctx context.Context, rawURL string) (*goquery.Document, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) + if err != nil { + return nil, err + } + req.Header.Set("Origin", s.cfg.BaseURL) + req.Header.Set("Referer", s.cfg.BaseURL+"/") + resp, err := s.client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("gigaviewer: HTTP %d", resp.StatusCode) + } + return goquery.NewDocumentFromReader(resp.Body) +} + +func (s *Source) fetchAllManga() ([]source.SManga, error) { + doc, err := s.get(context.Background(), strings.TrimRight(s.cfg.BaseURL, "/")+"/series") + if err != nil { + return nil, err + } + var mangas []source.SManga + doc.Find("ul.series-list li a").Each(func(_ int, el *goquery.Selection) { + m := source.SManga{} + if href, ok := el.Attr("href"); ok { + m.URL = href + } + el.Find("h2.series-list-title").First().Each(func(_ int, e *goquery.Selection) { + m.Title = strings.TrimSpace(e.Text()) + }) + el.Find("div.series-list-thumb img").First().Each(func(_ int, img *goquery.Selection) { + if v, ok := img.Attr("data-src"); ok && v != "" { + m.ThumbnailURL = util.AbsURL(s.cfg.BaseURL, v) + } else if v, ok := img.Attr("src"); ok && v != "" { + m.ThumbnailURL = util.AbsURL(s.cfg.BaseURL, v) + } + }) + if m.URL != "" { + mangas = append(mangas, m) + } + }) + return mangas, nil +} + +func (s *Source) GetPopularManga(page int) (source.MangasPage, error) { + if page > 1 { + return source.MangasPage{HasNextPage: false}, nil + } + mangas, err := s.fetchAllManga() + return source.MangasPage{Mangas: mangas, HasNextPage: false}, err +} + +func (s *Source) GetLatestUpdates(page int) (source.MangasPage, error) { + return s.GetPopularManga(page) +} + +func (s *Source) GetSearchManga(page int, query string, filters []source.Filter) (source.MangasPage, error) { + mangas, err := s.fetchAllManga() + if err != nil { + return source.MangasPage{}, err + } + q := strings.ToLower(query) + var matched []source.SManga + for _, m := range mangas { + if strings.Contains(strings.ToLower(m.Title), q) { + matched = append(matched, m) + } + } + return source.MangasPage{Mangas: matched, HasNextPage: false}, nil +} + +func (s *Source) GetMangaDetails(manga source.SManga) (source.SManga, error) { + doc, err := s.get(context.Background(), util.AbsURL(s.cfg.BaseURL, manga.URL)) + if err != nil { + return manga, err + } + result := source.SManga{URL: manga.URL} + doc.Find("section.series-information div.series-header").First().Each(func(_ int, el *goquery.Selection) { + el.Find("h1.series-header-title").First().Each(func(_ int, e *goquery.Selection) { result.Title = strings.TrimSpace(e.Text()) }) + el.Find("h2.series-header-author").First().Each(func(_ int, e *goquery.Selection) { result.Author = strings.TrimSpace(e.Text()) }) + el.Find("p.series-header-description").First().Each(func(_ int, e *goquery.Selection) { result.Description = strings.TrimSpace(e.Text()) }) + el.Find("div.series-header-image img").First().Each(func(_ int, img *goquery.Selection) { + result.ThumbnailURL = util.AbsURL(s.cfg.BaseURL, img.AttrOr("src", "")) + }) + }) + return result, nil +} + +func (s *Source) GetChapterList(manga source.SManga) ([]source.SChapter, error) { + doc, err := s.get(context.Background(), util.AbsURL(s.cfg.BaseURL, manga.URL)) + if err != nil { + return nil, err + } + var chapters []source.SChapter + doc.Find("ul.series-episode-list li a, li.episode a").Each(func(_ int, a *goquery.Selection) { + ch := source.SChapter{} + if href, ok := a.Attr("href"); ok { + ch.URL = href + } + a.Find(".series-episode-list-title, .episode-title").First().Each(func(_ int, e *goquery.Selection) { + ch.Name = strings.TrimSpace(e.Text()) + }) + if ch.URL != "" { + chapters = append(chapters, ch) + } + }) + return chapters, nil +} + +func (s *Source) GetPageList(chapter source.SChapter) ([]source.Page, error) { + doc, err := s.get(context.Background(), util.AbsURL(s.cfg.BaseURL, chapter.URL)) + if err != nil { + return nil, err + } + rawURL := util.AbsURL(s.cfg.BaseURL, chapter.URL) + var pages []source.Page + doc.Find("div.js-page-viewer img, .page-image img").Each(func(i int, img *goquery.Selection) { + imgURL := util.AbsURL(s.cfg.BaseURL, img.AttrOr("src", img.AttrOr("data-src", ""))) + if imgURL != "" { + pages = append(pages, source.Page{Index: i, URL: rawURL, ImageURL: imgURL}) + } + }) + return pages, nil +} + +func (s *Source) GetImageURL(page source.Page) (string, error) { return page.ImageURL, nil } +func (s *Source) GetFilterList() []source.Filter { return nil } diff --git a/sources/base/gmanga/gmanga.go b/sources/base/gmanga/gmanga.go new file mode 100644 index 0000000..e5a5b79 --- /dev/null +++ b/sources/base/gmanga/gmanga.go @@ -0,0 +1,225 @@ +// Package gmanga implements the GManga Arabic manga base. +// GET {base}/api/releases?page={n}; POST search. +package gmanga + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + + "goyomi/internal/httpclient" + "goyomi/internal/source" + "goyomi/sources/base/util" +) + +type Config struct { + Name string + BaseURL string + Lang string +} + +type Source struct { + cfg Config + client *httpclient.Client + id int64 +} + +func New(cfg Config) *Source { + c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) + return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} +} + +func (s *Source) ID() int64 { return s.id } +func (s *Source) Name() string { return s.cfg.Name } +func (s *Source) Lang() string { return s.cfg.Lang } +func (s *Source) SupportsLatest() bool { return true } + +func (s *Source) base() string { return strings.TrimRight(s.cfg.BaseURL, "/") } + +type releaseDTO struct { + Manga struct { + ID int `json:"id"` + Slug string `json:"slug"` + Title string `json:"title"` + ArTitle string `json:"arabic_title"` + Thumbnail string `json:"thumbnail"` + } `json:"manga"` +} + +type releasesDTO struct { + Data []releaseDTO `json:"data"` + Meta struct { + LastPage int `json:"last_page"` + CurrentPage int `json:"current_page"` + } `json:"meta"` +} + +func (s *Source) getJSON(ctx context.Context, rawURL string, out any) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) + if err != nil { + return err + } + req.Header.Set("Accept", "application/json") + resp, err := s.client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("gmanga: HTTP %d", resp.StatusCode) + } + body, _ := io.ReadAll(resp.Body) + return json.Unmarshal(body, out) +} + +func (s *Source) GetPopularManga(page int) (source.MangasPage, error) { + var result releasesDTO + if err := s.getJSON(context.Background(), fmt.Sprintf("%s/api/releases?page=%d", s.base(), page), &result); err != nil { + return source.MangasPage{}, err + } + seen := map[int]bool{} + var mangas []source.SManga + for _, r := range result.Data { + if seen[r.Manga.ID] { + continue + } + seen[r.Manga.ID] = true + title := r.Manga.Title + if title == "" { + title = r.Manga.ArTitle + } + mangas = append(mangas, source.SManga{ + URL: fmt.Sprintf("/mangas/%s", r.Manga.Slug), + Title: title, + ThumbnailURL: util.AbsURL(s.cfg.BaseURL, r.Manga.Thumbnail), + }) + } + hasNext := result.Meta.CurrentPage < result.Meta.LastPage + return source.MangasPage{Mangas: mangas, HasNextPage: hasNext}, nil +} + +func (s *Source) GetLatestUpdates(page int) (source.MangasPage, error) { + return s.GetPopularManga(page) +} + +func (s *Source) GetSearchManga(page int, query string, filters []source.Filter) (source.MangasPage, error) { + body, _ := json.Marshal(map[string]any{"q": query, "page": page}) + req, _ := http.NewRequestWithContext(context.Background(), http.MethodPost, + s.base()+"/api/mangas/search", bytes.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json") + resp, err := s.client.Do(req) + if err != nil { + return source.MangasPage{}, err + } + defer resp.Body.Close() + raw, _ := io.ReadAll(resp.Body) + var result struct { + Data []struct { + ID int `json:"id"` + Slug string `json:"slug"` + Title string `json:"title"` + Thumbnail string `json:"thumbnail"` + } `json:"data"` + HasMore bool `json:"has_more"` + } + if err := json.Unmarshal(raw, &result); err != nil { + return source.MangasPage{}, err + } + mangas := make([]source.SManga, len(result.Data)) + for i, m := range result.Data { + mangas[i] = source.SManga{ + URL: fmt.Sprintf("/mangas/%s", m.Slug), + Title: m.Title, + ThumbnailURL: util.AbsURL(s.cfg.BaseURL, m.Thumbnail), + } + } + return source.MangasPage{Mangas: mangas, HasNextPage: result.HasMore}, nil +} + +func (s *Source) GetMangaDetails(manga source.SManga) (source.SManga, error) { + slug := util.SlugFromURL(manga.URL) + var result struct { + Manga struct { + Title string `json:"title"` + ArTitle string `json:"arabic_title"` + Thumbnail string `json:"thumbnail"` + Description string `json:"description"` + Authors []string `json:"authors"` + Status string `json:"status"` + Tags []struct{ Name string `json:"name"` } `json:"tags"` + } `json:"manga"` + } + if err := s.getJSON(context.Background(), fmt.Sprintf("%s/api/mangas/%s", s.base(), slug), &result); err != nil { + return manga, err + } + title := result.Manga.Title + if title == "" { + title = result.Manga.ArTitle + } + genres := make([]string, len(result.Manga.Tags)) + for i, t := range result.Manga.Tags { + genres[i] = t.Name + } + return source.SManga{ + URL: manga.URL, + Title: title, + Author: strings.Join(result.Manga.Authors, ", "), + Description: result.Manga.Description, + Genre: strings.Join(genres, ", "), + Status: util.StatusFromString(result.Manga.Status), + ThumbnailURL: util.AbsURL(s.cfg.BaseURL, result.Manga.Thumbnail), + }, nil +} + +func (s *Source) GetChapterList(manga source.SManga) ([]source.SChapter, error) { + slug := util.SlugFromURL(manga.URL) + var result struct { + Chapters []struct { + ID int `json:"id"` + Number float32 `json:"number"` + Title string `json:"title"` + CreatedAt string `json:"created_at"` + } `json:"chapters"` + } + if err := s.getJSON(context.Background(), fmt.Sprintf("%s/api/mangas/%s/chapters", s.base(), slug), &result); err != nil { + return nil, err + } + chapters := make([]source.SChapter, len(result.Chapters)) + for i, ch := range result.Chapters { + name := fmt.Sprintf("Chapter %.1f", ch.Number) + if ch.Title != "" { + name += " - " + ch.Title + } + chapters[i] = source.SChapter{ + URL: fmt.Sprintf("/mangas/%s/%d", slug, ch.ID), + Name: name, + DateUpload: util.ParseAbsoluteDate(ch.CreatedAt, "2006-01-02T15:04:05Z"), + } + } + return chapters, nil +} + +func (s *Source) GetPageList(chapter source.SChapter) ([]source.Page, error) { + chID := util.SlugFromURL(chapter.URL) + var result struct { + Pages []struct { + URL string `json:"url"` + } `json:"chapter_data"` + } + if err := s.getJSON(context.Background(), fmt.Sprintf("%s/api/chapters/%s", s.base(), chID), &result); err != nil { + return nil, err + } + pages := make([]source.Page, len(result.Pages)) + for i, p := range result.Pages { + pages[i] = source.Page{Index: i, ImageURL: util.AbsURL(s.cfg.BaseURL, p.URL)} + } + return pages, nil +} + +func (s *Source) GetImageURL(page source.Page) (string, error) { return page.ImageURL, nil } +func (s *Source) GetFilterList() []source.Filter { return nil } diff --git a/sources/base/grouple/grouple.go b/sources/base/grouple/grouple.go new file mode 100644 index 0000000..10e3ccd --- /dev/null +++ b/sources/base/grouple/grouple.go @@ -0,0 +1,155 @@ +// Package grouple implements the Grouple reader base. +// GET {base}/list?sortType=rate&offset={50*(n-1)} +package grouple + +import ( + "context" + "fmt" + "net/http" + "strings" + + "github.com/PuerkitoBio/goquery" + + "goyomi/internal/httpclient" + "goyomi/internal/source" + "goyomi/sources/base/util" +) + +type Config struct { + Name string + BaseURL string + Lang string +} + +type Source struct { + cfg Config + client *httpclient.Client + id int64 +} + +func New(cfg Config) *Source { + c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) + return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} +} + +func (s *Source) ID() int64 { return s.id } +func (s *Source) Name() string { return s.cfg.Name } +func (s *Source) Lang() string { return s.cfg.Lang } +func (s *Source) SupportsLatest() bool { return true } + +func (s *Source) get(ctx context.Context, rawURL string) (*goquery.Document, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) + if err != nil { + return nil, err + } + resp, err := s.client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("grouple: HTTP %d", resp.StatusCode) + } + return goquery.NewDocumentFromReader(resp.Body) +} + +func (s *Source) base() string { return strings.TrimRight(s.cfg.BaseURL, "/") } + +func (s *Source) parseMangaList(doc *goquery.Document) source.MangasPage { + var mangas []source.SManga + doc.Find("div.tile.hovertr").Each(func(_ int, el *goquery.Selection) { + m := source.SManga{} + el.Find("a.non-existent, h3 a").First().Each(func(_ int, a *goquery.Selection) { + if href, ok := a.Attr("href"); ok { + m.URL = href + } + m.Title = strings.TrimSpace(a.Text()) + }) + el.Find("img.lazy").First().Each(func(_ int, img *goquery.Selection) { + m.ThumbnailURL = util.AbsURL(s.cfg.BaseURL, img.AttrOr("data-original", img.AttrOr("src", ""))) + }) + if m.URL != "" { + mangas = append(mangas, m) + } + }) + hasNext := doc.Find(".pagination .next, li.next a").Length() > 0 + return source.MangasPage{Mangas: mangas, HasNextPage: hasNext} +} + +func (s *Source) GetPopularManga(page int) (source.MangasPage, error) { + offset := 50 * (page - 1) + doc, err := s.get(context.Background(), fmt.Sprintf("%s/list?sortType=rate&offset=%d", s.base(), offset)) + if err != nil { + return source.MangasPage{}, err + } + return s.parseMangaList(doc), nil +} + +func (s *Source) GetLatestUpdates(page int) (source.MangasPage, error) { + offset := 50 * (page - 1) + doc, err := s.get(context.Background(), fmt.Sprintf("%s/list?sortType=update&offset=%d", s.base(), offset)) + if err != nil { + return source.MangasPage{}, err + } + return s.parseMangaList(doc), nil +} + +func (s *Source) GetSearchManga(page int, query string, filters []source.Filter) (source.MangasPage, error) { + doc, err := s.get(context.Background(), fmt.Sprintf("%s/search?q=%s&page=%d", s.base(), query, page)) + if err != nil { + return source.MangasPage{}, err + } + return s.parseMangaList(doc), nil +} + +func (s *Source) GetMangaDetails(manga source.SManga) (source.SManga, error) { + doc, err := s.get(context.Background(), util.AbsURL(s.cfg.BaseURL, manga.URL)) + if err != nil { + return manga, err + } + result := source.SManga{URL: manga.URL} + doc.Find("h1.names span.name").First().Each(func(_ int, el *goquery.Selection) { result.Title = strings.TrimSpace(el.Text()) }) + doc.Find("div.elementList div.subject-meta").First().Each(func(_ int, el *goquery.Selection) { result.Description = strings.TrimSpace(el.Text()) }) + doc.Find("img.hidden-xs").First().Each(func(_ int, img *goquery.Selection) { + result.ThumbnailURL = util.AbsURL(s.cfg.BaseURL, img.AttrOr("src", "")) + }) + return result, nil +} + +func (s *Source) GetChapterList(manga source.SManga) ([]source.SChapter, error) { + doc, err := s.get(context.Background(), util.AbsURL(s.cfg.BaseURL, manga.URL)) + if err != nil { + return nil, err + } + var chapters []source.SChapter + doc.Find("div.chapters-link a, ul.chapters li a").Each(func(_ int, a *goquery.Selection) { + ch := source.SChapter{} + if href, ok := a.Attr("href"); ok { + ch.URL = href + } + ch.Name = strings.TrimSpace(a.Text()) + if ch.URL != "" { + chapters = append(chapters, ch) + } + }) + return chapters, nil +} + +func (s *Source) GetPageList(chapter source.SChapter) ([]source.Page, error) { + doc, err := s.get(context.Background(), util.AbsURL(s.cfg.BaseURL, chapter.URL)) + if err != nil { + return nil, err + } + rawURL := util.AbsURL(s.cfg.BaseURL, chapter.URL) + var pages []source.Page + doc.Find("img.reader-images, div#sps img").Each(func(i int, img *goquery.Selection) { + imgURL := util.AbsURL(s.cfg.BaseURL, img.AttrOr("data-src", img.AttrOr("src", ""))) + if imgURL != "" { + pages = append(pages, source.Page{Index: i, URL: rawURL, ImageURL: imgURL}) + } + }) + return pages, nil +} + +func (s *Source) GetImageURL(page source.Page) (string, error) { return page.ImageURL, nil } +func (s *Source) GetFilterList() []source.Filter { return nil } diff --git a/sources/base/guya/guya.go b/sources/base/guya/guya.go new file mode 100644 index 0000000..e025561 --- /dev/null +++ b/sources/base/guya/guya.go @@ -0,0 +1,179 @@ +// Package guya implements the Guya/Manga4Life reader base. +// GET {base}/api/get_all_series/ returns all manga as a JSON map. +package guya + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + + "goyomi/internal/httpclient" + "goyomi/internal/source" +) + +type Config struct { + Name string + BaseURL string + Lang string +} + +type seriesEntry struct { + Title string `json:"title"` + CoverVol string `json:"cover_vol"` + Groups map[string]string `json:"groups"` + Chapters map[string]chapterEntry `json:"chapters"` +} + +type chapterEntry struct { + Title string `json:"title"` + Date int64 `json:"date"` + Groups map[string][]string `json:"groups"` +} + +type Source struct { + cfg Config + client *httpclient.Client + id int64 +} + +func New(cfg Config) *Source { + c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) + return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} +} + +func (s *Source) ID() int64 { return s.id } +func (s *Source) Name() string { return s.cfg.Name } +func (s *Source) Lang() string { return s.cfg.Lang } +func (s *Source) SupportsLatest() bool { return true } + +func (s *Source) base() string { return strings.TrimRight(s.cfg.BaseURL, "/") } + +func (s *Source) getAllSeries(ctx context.Context) (map[string]seriesEntry, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, s.base()+"/api/get_all_series/", nil) + if err != nil { + return nil, err + } + resp, err := s.client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("guya: HTTP %d", resp.StatusCode) + } + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + var result map[string]seriesEntry + if err := json.Unmarshal(body, &result); err != nil { + return nil, err + } + return result, nil +} + +func (s *Source) toSManga(slug string, entry seriesEntry) source.SManga { + thumb := "" + if entry.CoverVol != "" { + thumb = fmt.Sprintf("%s/media/manga/%s/volume-covers/%s", s.base(), slug, entry.CoverVol) + } + return source.SManga{ + URL: fmt.Sprintf("/reader/series/%s/", slug), + Title: entry.Title, + ThumbnailURL: thumb, + } +} + +func (s *Source) GetPopularManga(page int) (source.MangasPage, error) { + if page > 1 { + return source.MangasPage{}, nil + } + series, err := s.getAllSeries(context.Background()) + if err != nil { + return source.MangasPage{}, err + } + var mangas []source.SManga + for slug, entry := range series { + mangas = append(mangas, s.toSManga(slug, entry)) + } + return source.MangasPage{Mangas: mangas, HasNextPage: false}, nil +} + +func (s *Source) GetLatestUpdates(page int) (source.MangasPage, error) { + return s.GetPopularManga(page) +} + +func (s *Source) GetSearchManga(page int, query string, filters []source.Filter) (source.MangasPage, error) { + mp, err := s.GetPopularManga(1) + if err != nil { + return source.MangasPage{}, err + } + q := strings.ToLower(query) + var matched []source.SManga + for _, m := range mp.Mangas { + if strings.Contains(strings.ToLower(m.Title), q) { + matched = append(matched, m) + } + } + return source.MangasPage{Mangas: matched, HasNextPage: false}, nil +} + +func (s *Source) GetMangaDetails(manga source.SManga) (source.SManga, error) { + slug := strings.Trim(strings.TrimPrefix(manga.URL, "/reader/series/"), "/") + apiURL := fmt.Sprintf("%s/api/series/%s/", s.base(), slug) + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, apiURL, nil) + if err != nil { + return manga, err + } + resp, err := s.client.Do(req) + if err != nil { + return manga, err + } + defer resp.Body.Close() + body, _ := io.ReadAll(resp.Body) + var entry seriesEntry + if err := json.Unmarshal(body, &entry); err != nil { + return manga, err + } + return s.toSManga(slug, entry), nil +} + +func (s *Source) GetChapterList(manga source.SManga) ([]source.SChapter, error) { + slug := strings.Trim(strings.TrimPrefix(manga.URL, "/reader/series/"), "/") + apiURL := fmt.Sprintf("%s/api/series/%s/", s.base(), slug) + req, _ := http.NewRequestWithContext(context.Background(), http.MethodGet, apiURL, nil) + resp, err := s.client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + body, _ := io.ReadAll(resp.Body) + var entry seriesEntry + if err := json.Unmarshal(body, &entry); err != nil { + return nil, err + } + var chapters []source.SChapter + for chNum, ch := range entry.Chapters { + name := "Chapter " + chNum + if ch.Title != "" { + name += " - " + ch.Title + } + chapters = append(chapters, source.SChapter{ + URL: fmt.Sprintf("%s/reader/series/%s/%s/", s.base(), slug, chNum), + Name: name, + DateUpload: ch.Date * 1000, + }) + } + return chapters, nil +} + +func (s *Source) GetPageList(chapter source.SChapter) ([]source.Page, error) { + // Pages are served as images at known paths; minimal impl returns empty + return nil, nil +} + +func (s *Source) GetImageURL(page source.Page) (string, error) { return page.ImageURL, nil } +func (s *Source) GetFilterList() []source.Filter { return nil } diff --git a/sources/base/heancms/heancms.go b/sources/base/heancms/heancms.go new file mode 100644 index 0000000..eec340c --- /dev/null +++ b/sources/base/heancms/heancms.go @@ -0,0 +1,397 @@ +// Package heancms implements the HeanCMS multi-source base. +// API: JSON REST, endpoint at api.{baseURL} by default. +package heancms + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" + + "goyomi/internal/httpclient" + "goyomi/internal/source" + "goyomi/sources/base/util" +) + +// Config holds per-source configuration. +type Config struct { + Name string + BaseURL string + Lang string + APIURL string // defaults to api.{BaseURL} + CoverPath string // path prefix for cover images + CdnURL string // CDN base, defaults to APIURL + MangaSubDirectory string // e.g. "series" or "manga" +} + +type Source struct { + cfg Config + client *httpclient.Client + id int64 +} + +func New(cfg Config) *Source { + if cfg.APIURL == "" { + parsed, err := url.Parse(cfg.BaseURL) + if err == nil { + parsed.Host = "api." + parsed.Host + cfg.APIURL = parsed.String() + } else { + cfg.APIURL = strings.Replace(cfg.BaseURL, "://", "://api.", 1) + } + } + if cfg.CdnURL == "" { + cfg.CdnURL = cfg.APIURL + } + if cfg.MangaSubDirectory == "" { + cfg.MangaSubDirectory = "series" + } + c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) + return &Source{ + cfg: cfg, + client: c, + id: source.GenerateSourceID(cfg.Name, cfg.Lang), + } +} + +func (s *Source) ID() int64 { return s.id } +func (s *Source) Name() string { return s.cfg.Name } +func (s *Source) Lang() string { return s.cfg.Lang } +func (s *Source) SupportsLatest() bool { return true } + +// --- JSON DTOs --- + +type seriesListDTO struct { + Data []seriesDTO `json:"data"` + Meta *seriesMetaDTO `json:"meta"` +} + +type seriesMetaDTO struct { + CurrentPage int `json:"current_page"` + LastPage int `json:"last_page"` +} + +func (m *seriesMetaDTO) hasNextPage() bool { + if m == nil { + return false + } + return m.CurrentPage < m.LastPage +} + +type seriesDTO struct { + ID int `json:"id"` + Slug string `json:"series_slug"` + Author *string `json:"author"` + Description *string `json:"description"` + Studio *string `json:"studio"` + Status *string `json:"status"` + Thumbnail string `json:"thumbnail"` + Title string `json:"title"` + Tags []tagDTO `json:"tags"` + Seasons []seasonDTO `json:"seasons"` +} + +type tagDTO struct { + Name string `json:"name"` +} + +type seasonDTO struct { + Chapters []chapterDTO `json:"chapters"` +} + +type chapterDTO struct { + ID int `json:"id"` + Name string `json:"chapter_name"` + Title *string `json:"chapter_title"` + Slug string `json:"chapter_slug"` + CreatedAt *string `json:"created_at"` + Price int `json:"price"` +} + +type chapterPayloadDTO struct { + Data []chapterDTO `json:"data"` + Meta chapterMetaDTO `json:"meta"` +} + +type chapterMetaDTO struct { + CurrentPage int `json:"current_page"` + LastPage int `json:"last_page"` +} + +func (m chapterMetaDTO) hasNextPage() bool { + return m.CurrentPage < m.LastPage +} + +type pagePayloadDTO struct { + Chapter struct { + ChapterData *struct { + Images []string `json:"images"` + } `json:"chapter_data"` + } `json:"chapter"` + Data []string `json:"data"` +} + +// --- helpers --- + +var dateLayouts = []string{ + "2006-01-02T15:04:05.999Z07:00", + "2006-01-02T15:04:05Z07:00", + "2006-01-02T15:04:05", + "2006-01-02", +} + +func parseDate(s string) int64 { + if s == "" { + return 0 + } + for _, layout := range dateLayouts { + t, err := time.Parse(layout, s) + if err == nil { + return t.UnixMilli() + } + } + return 0 +} + +func (s *Source) thumbnailURL(thumb string) string { + if thumb == "" { + return "" + } + if strings.HasPrefix(thumb, "http") { + return thumb + } + base := strings.TrimRight(s.cfg.CdnURL, "/") + prefix := "" + if s.cfg.CoverPath != "" { + prefix = "/" + strings.Trim(s.cfg.CoverPath, "/") + } + return base + prefix + "/" + strings.TrimLeft(thumb, "/") +} + +func (s *Source) seriesURL(slug string, id int) string { + return fmt.Sprintf("/%s/%s#%d", s.cfg.MangaSubDirectory, slug, id) +} + +func (s *Source) toSManga(dto seriesDTO) source.SManga { + genres := make([]string, 0, len(dto.Tags)) + for _, t := range dto.Tags { + genres = append(genres, t.Name) + } + var desc string + if dto.Description != nil { + desc = util.CleanText(*dto.Description) + } + var author string + if dto.Author != nil { + author = *dto.Author + } + var artist string + if dto.Studio != nil { + artist = *dto.Studio + } + var status int + if dto.Status != nil { + status = util.StatusFromString(*dto.Status) + } + return source.SManga{ + URL: s.seriesURL(dto.Slug, dto.ID), + Title: dto.Title, + Author: author, + Artist: artist, + Description: desc, + Genre: strings.Join(genres, ", "), + Status: status, + ThumbnailURL: s.thumbnailURL(dto.Thumbnail), + } +} + +func (s *Source) toSChapter(dto chapterDTO, seriesSlug string) source.SChapter { + name := "Chapter " + dto.Name + if dto.Title != nil && *dto.Title != "" { + name += " - " + *dto.Title + } + var date int64 + if dto.CreatedAt != nil { + date = parseDate(*dto.CreatedAt) + } + return source.SChapter{ + URL: fmt.Sprintf("/%s/%s/%s", s.cfg.MangaSubDirectory, seriesSlug, dto.Slug), + Name: name, + DateUpload: date, + } +} + +// --- API calls --- + +func (s *Source) getJSON(ctx context.Context, rawURL string, out any) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) + if err != nil { + return err + } + req.Header.Set("Accept", "application/json") + resp, err := s.client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("heancms: HTTP %d for %s", resp.StatusCode, rawURL) + } + body, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + return json.Unmarshal(body, out) +} + +func (s *Source) fetchSeriesList(ctx context.Context, page int, order, query string) (source.MangasPage, error) { + endpoint := strings.TrimRight(s.cfg.APIURL, "/") + "/series" + u, _ := url.Parse(endpoint) + q := u.Query() + q.Set("page", fmt.Sprintf("%d", page)) + q.Set("order", order) + q.Set("series_type", "Comic") + if query != "" { + q.Set("query_string", query) + } + u.RawQuery = q.Encode() + + var result seriesListDTO + if err := s.getJSON(ctx, u.String(), &result); err != nil { + return source.MangasPage{}, err + } + mangas := make([]source.SManga, 0, len(result.Data)) + for _, d := range result.Data { + mangas = append(mangas, s.toSManga(d)) + } + return source.MangasPage{Mangas: mangas, HasNextPage: result.Meta.hasNextPage()}, nil +} + +func (s *Source) GetPopularManga(page int) (source.MangasPage, error) { + return s.fetchSeriesList(context.Background(), page, "total_views", "") +} + +func (s *Source) GetLatestUpdates(page int) (source.MangasPage, error) { + return s.fetchSeriesList(context.Background(), page, "latest", "") +} + +func (s *Source) GetSearchManga(page int, query string, filters []source.Filter) (source.MangasPage, error) { + return s.fetchSeriesList(context.Background(), page, "total_views", query) +} + +func (s *Source) GetMangaDetails(manga source.SManga) (source.SManga, error) { + // URL format: /series/{slug}#{id} + urlStr := manga.URL + slug := util.SlugFromURL(strings.Split(urlStr, "#")[0]) + apiURL := strings.TrimRight(s.cfg.APIURL, "/") + "/series/" + slug + + var result seriesDTO + if err := s.getJSON(context.Background(), apiURL, &result); err != nil { + return manga, err + } + return s.toSManga(result), nil +} + +func (s *Source) GetChapterList(manga source.SManga) ([]source.SChapter, error) { + urlStr := manga.URL + parts := strings.SplitN(urlStr, "#", 2) + slug := util.SlugFromURL(parts[0]) + + // Try paginated chapter endpoint first + if len(parts) == 2 { + seriesID := parts[1] + return s.fetchChaptersPaginated(context.Background(), slug, seriesID) + } + + // Fall back to series detail endpoint + apiURL := strings.TrimRight(s.cfg.APIURL, "/") + "/series/" + slug + var result seriesDTO + if err := s.getJSON(context.Background(), apiURL, &result); err != nil { + return nil, err + } + var chapters []source.SChapter + for _, season := range result.Seasons { + for _, ch := range season.Chapters { + if ch.Price == 0 { + chapters = append(chapters, s.toSChapter(ch, result.Slug)) + } + } + } + return chapters, nil +} + +func (s *Source) fetchChaptersPaginated(ctx context.Context, slug, seriesID string) ([]source.SChapter, error) { + base := strings.TrimRight(s.cfg.APIURL, "/") + "/chapter/query" + var all []source.SChapter + page := 1 + for { + u, _ := url.Parse(base) + q := u.Query() + q.Set("page", fmt.Sprintf("%d", page)) + q.Set("perPage", "1000") + q.Set("series_id", seriesID) + u.RawQuery = q.Encode() + + var payload chapterPayloadDTO + if err := s.getJSON(ctx, u.String(), &payload); err != nil { + return nil, err + } + for _, ch := range payload.Data { + if ch.Price == 0 { + all = append(all, s.toSChapter(ch, slug)) + } + } + if !payload.Meta.hasNextPage() { + break + } + page++ + } + return all, nil +} + +func (s *Source) GetPageList(chapter source.SChapter) ([]source.Page, error) { + // Chapter URL: /series/{slug}/{chapter_slug} + // API URL: /chapter/{chapter_slug} + chURL := chapter.URL + chSlug := util.SlugFromURL(chURL) + // Replace /series/ with /chapter/ + apiURL := strings.TrimRight(s.cfg.APIURL, "/") + "/chapter/" + chSlug + + var result pagePayloadDTO + if err := s.getJSON(context.Background(), apiURL, &result); err != nil { + return nil, err + } + + var images []string + if result.Chapter.ChapterData != nil && len(result.Chapter.ChapterData.Images) > 0 { + images = result.Chapter.ChapterData.Images + } else { + images = result.Data + } + + pages := make([]source.Page, len(images)) + for i, img := range images { + imgURL := img + if !strings.HasPrefix(imgURL, "http") { + base := strings.TrimRight(s.cfg.CdnURL, "/") + if s.cfg.CoverPath != "" { + base += "/" + strings.Trim(s.cfg.CoverPath, "/") + } + imgURL = base + "/" + strings.TrimLeft(imgURL, "/") + } + pages[i] = source.Page{Index: i, ImageURL: imgURL} + } + return pages, nil +} + +func (s *Source) GetImageURL(page source.Page) (string, error) { + return page.ImageURL, nil +} + +func (s *Source) GetFilterList() []source.Filter { + return nil +} diff --git a/sources/base/hentaihand/hentaihand.go b/sources/base/hentaihand/hentaihand.go new file mode 100644 index 0000000..53834de --- /dev/null +++ b/sources/base/hentaihand/hentaihand.go @@ -0,0 +1,180 @@ +// Package hentaihand implements the HentaiHand REST API base. +// GET {base}/api/comics?page={n}&order_by=popularity +package hentaihand + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + + "goyomi/internal/httpclient" + "goyomi/internal/source" + "goyomi/sources/base/util" +) + +type Config struct { + Name string + BaseURL string + Lang string +} + +type comicDTO struct { + ID int `json:"id"` + Slug string `json:"slug"` + Title string `json:"title"` + CoverURL string `json:"cover_url"` + Author string `json:"author"` + Description string `json:"description"` + Status string `json:"status"` + Tags []struct{ Name string `json:"name"` } `json:"tags"` +} + +type comicsListDTO struct { + Data []comicDTO `json:"data"` + Meta struct { + CurrentPage int `json:"current_page"` + LastPage int `json:"last_page"` + } `json:"meta"` +} + +type Source struct { + cfg Config + client *httpclient.Client + id int64 +} + +func New(cfg Config) *Source { + c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) + return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} +} + +func (s *Source) ID() int64 { return s.id } +func (s *Source) Name() string { return s.cfg.Name } +func (s *Source) Lang() string { return s.cfg.Lang } +func (s *Source) SupportsLatest() bool { return true } + +func (s *Source) base() string { return strings.TrimRight(s.cfg.BaseURL, "/") } + +func (s *Source) getJSON(ctx context.Context, rawURL string, out any) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) + if err != nil { + return err + } + req.Header.Set("Accept", "application/json") + resp, err := s.client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("hentaihand: HTTP %d", resp.StatusCode) + } + body, _ := io.ReadAll(resp.Body) + return json.Unmarshal(body, out) +} + +func (s *Source) toSManga(c comicDTO) source.SManga { + genres := make([]string, len(c.Tags)) + for i, t := range c.Tags { + genres[i] = t.Name + } + return source.SManga{ + URL: fmt.Sprintf("/comics/%s", c.Slug), + Title: c.Title, + Author: c.Author, + Description: c.Description, + Genre: strings.Join(genres, ", "), + Status: util.StatusFromString(c.Status), + ThumbnailURL: c.CoverURL, + } +} + +func (s *Source) fetchComics(ctx context.Context, page int, orderBy, q string) (source.MangasPage, error) { + u := fmt.Sprintf("%s/api/comics?page=%d&order_by=%s", s.base(), page, orderBy) + if q != "" { + u += "&q=" + q + } + var result comicsListDTO + if err := s.getJSON(ctx, u, &result); err != nil { + return source.MangasPage{}, err + } + mangas := make([]source.SManga, len(result.Data)) + for i, c := range result.Data { + mangas[i] = s.toSManga(c) + } + hasNext := result.Meta.CurrentPage < result.Meta.LastPage + return source.MangasPage{Mangas: mangas, HasNextPage: hasNext}, nil +} + +func (s *Source) GetPopularManga(page int) (source.MangasPage, error) { + return s.fetchComics(context.Background(), page, "popularity", "") +} +func (s *Source) GetLatestUpdates(page int) (source.MangasPage, error) { + return s.fetchComics(context.Background(), page, "date", "") +} +func (s *Source) GetSearchManga(page int, query string, filters []source.Filter) (source.MangasPage, error) { + return s.fetchComics(context.Background(), page, "popularity", query) +} + +func (s *Source) GetMangaDetails(manga source.SManga) (source.SManga, error) { + slug := util.SlugFromURL(manga.URL) + var result comicDTO + if err := s.getJSON(context.Background(), fmt.Sprintf("%s/api/comics/%s", s.base(), slug), &result); err != nil { + return manga, err + } + return s.toSManga(result), nil +} + +func (s *Source) GetChapterList(manga source.SManga) ([]source.SChapter, error) { + slug := util.SlugFromURL(manga.URL) + var result struct { + Data []struct { + Slug string `json:"slug"` + Number float32 `json:"number"` + Title string `json:"title"` + Date string `json:"date"` + } `json:"data"` + } + if err := s.getJSON(context.Background(), fmt.Sprintf("%s/api/comics/%s/chapters", s.base(), slug), &result); err != nil { + return nil, err + } + chapters := make([]source.SChapter, len(result.Data)) + for i, ch := range result.Data { + name := fmt.Sprintf("Chapter %.1f", ch.Number) + if ch.Title != "" { + name += " - " + ch.Title + } + chapters[i] = source.SChapter{ + URL: fmt.Sprintf("/comics/%s/%s", slug, ch.Slug), + Name: name, + DateUpload: util.ParseAbsoluteDate(ch.Date, "2006-01-02"), + } + } + return chapters, nil +} + +func (s *Source) GetPageList(chapter source.SChapter) ([]source.Page, error) { + parts := strings.Split(strings.Trim(chapter.URL, "/"), "/") + if len(parts) < 3 { + return nil, fmt.Errorf("hentaihand: invalid chapter URL") + } + slug := parts[1] + chSlug := parts[2] + var result struct { + Images []string `json:"images"` + } + if err := s.getJSON(context.Background(), fmt.Sprintf("%s/api/comics/%s/chapters/%s", s.base(), slug, chSlug), &result); err != nil { + return nil, err + } + pages := make([]source.Page, len(result.Images)) + for i, img := range result.Images { + pages[i] = source.Page{Index: i, ImageURL: img} + } + return pages, nil +} + +func (s *Source) GetImageURL(page source.Page) (string, error) { return page.ImageURL, nil } +func (s *Source) GetFilterList() []source.Filter { return nil } diff --git a/sources/base/kemono/kemono.go b/sources/base/kemono/kemono.go new file mode 100644 index 0000000..20f632f --- /dev/null +++ b/sources/base/kemono/kemono.go @@ -0,0 +1,206 @@ +// Package kemono implements the Kemono Party base. +// GET {base}/api/v1/creators → creator list +// GET {base}/api/v1/{service}/{creator}/posts?o={offset} → paginated posts +package kemono + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + + "goyomi/internal/httpclient" + "goyomi/internal/source" + "goyomi/sources/base/util" +) + +type Config struct { + Name string + BaseURL string + Lang string +} + +type creatorDTO struct { + ID string `json:"id"` + Name string `json:"name"` + Service string `json:"service"` + Icon string `json:"icon"` +} + +type postDTO struct { + ID string `json:"id"` + Title string `json:"title"` + User string `json:"user"` + Service string `json:"service"` + Added string `json:"added"` + Attachments []struct { + Name string `json:"name"` + Path string `json:"path"` + } `json:"attachments"` + File struct { + Name string `json:"name"` + Path string `json:"path"` + } `json:"file"` +} + +type Source struct { + cfg Config + client *httpclient.Client + id int64 +} + +func New(cfg Config) *Source { + c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) + return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} +} + +func (s *Source) ID() int64 { return s.id } +func (s *Source) Name() string { return s.cfg.Name } +func (s *Source) Lang() string { return s.cfg.Lang } +func (s *Source) SupportsLatest() bool { return true } + +func (s *Source) getJSON(ctx context.Context, rawURL string, out any) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) + if err != nil { + return err + } + resp, err := s.client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("kemono: HTTP %d for %s", resp.StatusCode, rawURL) + } + body, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + return json.Unmarshal(body, out) +} + +func (s *Source) base() string { return strings.TrimRight(s.cfg.BaseURL, "/") } + +func (s *Source) creatorToSManga(c creatorDTO) source.SManga { + icon := c.Icon + if icon != "" && !strings.HasPrefix(icon, "http") { + icon = s.base() + "/data" + icon + } + return source.SManga{ + URL: fmt.Sprintf("/%s/user/%s", c.Service, c.ID), + Title: c.Name, + ThumbnailURL: icon, + } +} + +func (s *Source) GetPopularManga(page int) (source.MangasPage, error) { + var creators []creatorDTO + if err := s.getJSON(context.Background(), s.base()+"/api/v1/creators", &creators); err != nil { + return source.MangasPage{}, err + } + // page 1 returns all; no actual pagination + mangas := make([]source.SManga, 0, len(creators)) + for _, c := range creators { + mangas = append(mangas, s.creatorToSManga(c)) + } + return source.MangasPage{Mangas: mangas, HasNextPage: false}, nil +} + +func (s *Source) GetLatestUpdates(page int) (source.MangasPage, error) { + return s.GetPopularManga(page) +} + +func (s *Source) GetSearchManga(page int, query string, filters []source.Filter) (source.MangasPage, error) { + mp, err := s.GetPopularManga(1) + if err != nil { + return source.MangasPage{}, err + } + // Client-side filter + q := strings.ToLower(query) + var matched []source.SManga + for _, m := range mp.Mangas { + if strings.Contains(strings.ToLower(m.Title), q) { + matched = append(matched, m) + } + } + return source.MangasPage{Mangas: matched, HasNextPage: false}, nil +} + +func (s *Source) GetMangaDetails(manga source.SManga) (source.SManga, error) { + return manga, nil +} + +func (s *Source) GetChapterList(manga source.SManga) ([]source.SChapter, error) { + // URL: /{service}/user/{id} + parts := strings.Split(strings.Trim(manga.URL, "/"), "/") + if len(parts) < 3 { + return nil, fmt.Errorf("kemono: invalid manga URL %s", manga.URL) + } + service := parts[0] + creatorID := parts[2] + + var all []source.SChapter + offset := 0 + const limit = 50 + for { + apiURL := fmt.Sprintf("%s/api/v1/%s/user/%s/posts?o=%d", s.base(), service, creatorID, offset) + var posts []postDTO + if err := s.getJSON(context.Background(), apiURL, &posts); err != nil { + return nil, err + } + for _, p := range posts { + all = append(all, source.SChapter{ + URL: fmt.Sprintf("/%s/user/%s/post/%s", service, creatorID, p.ID), + Name: p.Title, + DateUpload: util.ParseAbsoluteDate(p.Added, "2006-01-02 15:04:05"), + }) + } + if len(posts) < limit { + break + } + offset += limit + } + return all, nil +} + +func (s *Source) GetPageList(chapter source.SChapter) ([]source.Page, error) { + // URL: /{service}/user/{creatorID}/post/{postID} + parts := strings.Split(strings.Trim(chapter.URL, "/"), "/") + if len(parts) < 5 { + return nil, fmt.Errorf("kemono: invalid chapter URL %s", chapter.URL) + } + service := parts[0] + creatorID := parts[2] + postID := parts[4] + + apiURL := fmt.Sprintf("%s/api/v1/%s/user/%s/post/%s", s.base(), service, creatorID, postID) + var post postDTO + if err := s.getJSON(context.Background(), apiURL, &post); err != nil { + return nil, err + } + + var pages []source.Page + idx := 0 + if post.File.Path != "" { + imgURL := post.File.Path + if !strings.HasPrefix(imgURL, "http") { + imgURL = s.base() + "/data" + imgURL + } + pages = append(pages, source.Page{Index: idx, ImageURL: imgURL}) + idx++ + } + for _, att := range post.Attachments { + imgURL := att.Path + if !strings.HasPrefix(imgURL, "http") { + imgURL = s.base() + "/data" + imgURL + } + pages = append(pages, source.Page{Index: idx, ImageURL: imgURL}) + idx++ + } + return pages, nil +} + +func (s *Source) GetImageURL(page source.Page) (string, error) { return page.ImageURL, nil } +func (s *Source) GetFilterList() []source.Filter { return nil } diff --git a/sources/base/madara/madara.go b/sources/base/madara/madara.go new file mode 100644 index 0000000..145eb3e --- /dev/null +++ b/sources/base/madara/madara.go @@ -0,0 +1,436 @@ +// Package madara implements the Madara WordPress theme multi-source base. +// Uses admin-ajax.php or /ajax/chapters for chapter lists; HTML scraping throughout. +package madara + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" + + "github.com/PuerkitoBio/goquery" + + "goyomi/internal/httpclient" + "goyomi/internal/source" + "goyomi/sources/base/util" +) + +// Config holds per-source configuration and overridable CSS selectors. +type Config struct { + Name string + BaseURL string + Lang string + + // MangaSubString is the URL path segment for manga listings (default "manga"). + MangaSubString string + + // UseNewChapterEndpoint: use /ajax/chapters instead of admin-ajax.php. + UseNewChapterEndpoint bool + + // Overridable selectors — leave empty to use defaults. + PopularMangaSelector string + PopularMangaURLSelector string + SearchMangaSelector string + ChapterListSelector string + ChapterURLSelector string + ChapterDateSelector string + PageListParseSelector string + MangaDetailsSelectorTitle string + MangaDetailsSelectorAuthor string + MangaDetailsSelectorArtist string + MangaDetailsSelectorStatus string + MangaDetailsSelectorDesc string + MangaDetailsSelectorThumb string + MangaDetailsSelectorGenre string +} + +func (c *Config) setDefaults() { + if c.MangaSubString == "" { + c.MangaSubString = "manga" + } + if c.PopularMangaSelector == "" { + c.PopularMangaSelector = "div.page-item-detail, .manga__item" + } + if c.PopularMangaURLSelector == "" { + c.PopularMangaURLSelector = "div.post-title a" + } + if c.SearchMangaSelector == "" { + c.SearchMangaSelector = "div.c-tabs-item__content, div.page-item-detail, .manga__item" + } + if c.ChapterListSelector == "" { + c.ChapterListSelector = "li.wp-manga-chapter" + } + if c.ChapterURLSelector == "" { + c.ChapterURLSelector = "a" + } + if c.ChapterDateSelector == "" { + c.ChapterDateSelector = "span.chapter-release-date" + } + if c.PageListParseSelector == "" { + c.PageListParseSelector = "div.page-break img, li.blocks-gallery-item img, .reading-content img" + } + if c.MangaDetailsSelectorTitle == "" { + c.MangaDetailsSelectorTitle = "div.post-title h3, div.post-title h1, #manga-title > h1" + } + if c.MangaDetailsSelectorAuthor == "" { + c.MangaDetailsSelectorAuthor = "div.author-content > a, div.manga-authors > a" + } + if c.MangaDetailsSelectorArtist == "" { + c.MangaDetailsSelectorArtist = "div.artist-content > a" + } + if c.MangaDetailsSelectorStatus == "" { + c.MangaDetailsSelectorStatus = "div.summary-content, div.summary-heading:contains(Status) + div" + } + if c.MangaDetailsSelectorDesc == "" { + c.MangaDetailsSelectorDesc = "div.description-summary div.summary__content, div.summary_content div.post-content_item > h5 + div" + } + if c.MangaDetailsSelectorThumb == "" { + c.MangaDetailsSelectorThumb = "div.summary_image img" + } + if c.MangaDetailsSelectorGenre == "" { + c.MangaDetailsSelectorGenre = "div.genres-content a" + } +} + +// Source implements source.CatalogueSource for Madara-based sites. +type Source struct { + cfg Config + client *httpclient.Client + id int64 +} + +func New(cfg Config) *Source { + cfg.setDefaults() + c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) + return &Source{ + cfg: cfg, + client: c, + id: source.GenerateSourceID(cfg.Name, cfg.Lang), + } +} + +func (s *Source) ID() int64 { return s.id } +func (s *Source) Name() string { return s.cfg.Name } +func (s *Source) Lang() string { return s.cfg.Lang } +func (s *Source) SupportsLatest() bool { return true } + +// --- HTTP helpers --- + +func (s *Source) get(ctx context.Context, rawURL string) (*goquery.Document, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) + if err != nil { + return nil, err + } + req.Header.Set("Referer", s.cfg.BaseURL+"/") + resp, err := s.client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("madara: HTTP %d for %s", resp.StatusCode, rawURL) + } + return goquery.NewDocumentFromReader(resp.Body) +} + +func (s *Source) post(ctx context.Context, rawURL string, form url.Values) (*goquery.Document, error) { + encoded := form.Encode() + req, err := http.NewRequestWithContext(ctx, http.MethodPost, rawURL, + strings.NewReader(encoded)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + req.Header.Set("X-Requested-With", "XMLHttpRequest") + req.Header.Set("Referer", s.cfg.BaseURL+"/") + resp, err := s.client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("madara: HTTP %d: %s", resp.StatusCode, string(body)) + } + return goquery.NewDocumentFromReader(resp.Body) +} + +// searchPage returns the URL path component for page n. +func (s *Source) searchPage(page int) string { + if page == 1 { + return "" + } + return fmt.Sprintf("page/%d/", page) +} + +func (s *Source) parseMangaFromElement(el *goquery.Selection) source.SManga { + manga := source.SManga{} + el.Find(s.cfg.PopularMangaURLSelector).First().Each(func(_ int, a *goquery.Selection) { + if href, ok := a.Attr("href"); ok { + manga.URL = stripDomain(href, s.cfg.BaseURL) + } + manga.Title = strings.TrimSpace(a.Text()) + }) + el.Find("img").First().Each(func(_ int, img *goquery.Selection) { + manga.ThumbnailURL = imgAttr(img, s.cfg.BaseURL) + }) + return manga +} + +func (s *Source) parseSearchMangaFromElement(el *goquery.Selection) source.SManga { + manga := source.SManga{} + el.Find("div.post-title a, h3.h5 a").First().Each(func(_ int, a *goquery.Selection) { + if href, ok := a.Attr("href"); ok { + manga.URL = stripDomain(href, s.cfg.BaseURL) + } + manga.Title = strings.TrimSpace(a.Text()) + }) + el.Find("img").First().Each(func(_ int, img *goquery.Selection) { + manga.ThumbnailURL = imgAttr(img, s.cfg.BaseURL) + }) + return manga +} + +func (s *Source) GetPopularManga(page int) (source.MangasPage, error) { + pageStr := s.searchPage(page) + rawURL := fmt.Sprintf("%s/%s/%s?m_orderby=views", + strings.TrimRight(s.cfg.BaseURL, "/"), s.cfg.MangaSubString, pageStr) + + doc, err := s.get(context.Background(), rawURL) + if err != nil { + return source.MangasPage{}, err + } + return s.parseMangaList(doc, s.cfg.PopularMangaSelector, true), nil +} + +func (s *Source) GetLatestUpdates(page int) (source.MangasPage, error) { + pageStr := s.searchPage(page) + rawURL := fmt.Sprintf("%s/%s/%s?m_orderby=latest", + strings.TrimRight(s.cfg.BaseURL, "/"), s.cfg.MangaSubString, pageStr) + + doc, err := s.get(context.Background(), rawURL) + if err != nil { + return source.MangasPage{}, err + } + return s.parseMangaList(doc, s.cfg.PopularMangaSelector, true), nil +} + +func (s *Source) GetSearchManga(page int, query string, filters []source.Filter) (source.MangasPage, error) { + base := strings.TrimRight(s.cfg.BaseURL, "/") + searchURL := fmt.Sprintf("%s/?s=%s&post_type=wp-manga&paged=%d", + base, url.QueryEscape(query), page) + + doc, err := s.get(context.Background(), searchURL) + if err != nil { + return source.MangasPage{}, err + } + return s.parseMangaList(doc, s.cfg.SearchMangaSelector, false), nil +} + +func (s *Source) parseMangaList(doc *goquery.Document, selector string, popular bool) source.MangasPage { + var mangas []source.SManga + doc.Find(selector).Each(func(_ int, el *goquery.Selection) { + var m source.SManga + if popular { + m = s.parseMangaFromElement(el) + } else { + m = s.parseSearchMangaFromElement(el) + } + if m.URL != "" { + mangas = append(mangas, m) + } + }) + hasNext := doc.Find("div.nav-previous, nav.navigation-ajax, a.nextpostslink").Length() > 0 + return source.MangasPage{Mangas: mangas, HasNextPage: hasNext} +} + +func (s *Source) GetMangaDetails(manga source.SManga) (source.SManga, error) { + rawURL := util.AbsURL(s.cfg.BaseURL, manga.URL) + doc, err := s.get(context.Background(), rawURL) + if err != nil { + return manga, err + } + return s.parseMangaDetails(doc, manga.URL), nil +} + +func (s *Source) parseMangaDetails(doc *goquery.Document, mangaURL string) source.SManga { + manga := source.SManga{URL: mangaURL} + + doc.Find(s.cfg.MangaDetailsSelectorTitle).First().Each(func(_ int, el *goquery.Selection) { + manga.Title = strings.TrimSpace(el.Text()) + }) + var authors []string + doc.Find(s.cfg.MangaDetailsSelectorAuthor).Each(func(_ int, el *goquery.Selection) { + if t := strings.TrimSpace(el.Text()); t != "" { + authors = append(authors, t) + } + }) + manga.Author = strings.Join(authors, ", ") + + var artists []string + doc.Find(s.cfg.MangaDetailsSelectorArtist).Each(func(_ int, el *goquery.Selection) { + if t := strings.TrimSpace(el.Text()); t != "" { + artists = append(artists, t) + } + }) + manga.Artist = strings.Join(artists, ", ") + + doc.Find(s.cfg.MangaDetailsSelectorDesc).First().Each(func(_ int, el *goquery.Selection) { + manga.Description = strings.TrimSpace(el.Text()) + }) + + doc.Find(s.cfg.MangaDetailsSelectorThumb).First().Each(func(_ int, img *goquery.Selection) { + manga.ThumbnailURL = imgAttr(img, s.cfg.BaseURL) + }) + + doc.Find(s.cfg.MangaDetailsSelectorStatus).Last().Each(func(_ int, el *goquery.Selection) { + manga.Status = util.StatusFromString(el.Text()) + }) + + var genres []string + doc.Find(s.cfg.MangaDetailsSelectorGenre).Each(func(_ int, el *goquery.Selection) { + if t := strings.TrimSpace(el.Text()); t != "" { + genres = append(genres, t) + } + }) + manga.Genre = strings.Join(genres, ", ") + + return manga +} + +func (s *Source) GetChapterList(manga source.SManga) ([]source.SChapter, error) { + mangaURL := strings.TrimRight(util.AbsURL(s.cfg.BaseURL, manga.URL), "/") + doc, err := s.get(context.Background(), mangaURL) + if err != nil { + return nil, err + } + + // Try inline chapter list first + chapterEls := doc.Find(s.cfg.ChapterListSelector) + + if chapterEls.Length() == 0 { + // Need AJAX fetch + chapHolder := doc.Find("div[id^=manga-chapters-holder]") + if chapHolder.Length() > 0 { + mangaID, _ := chapHolder.Attr("data-id") + ajaxDoc, ajaxErr := s.fetchChaptersAJAX(mangaURL, mangaID) + if ajaxErr != nil { + return nil, ajaxErr + } + chapterEls = ajaxDoc.Find(s.cfg.ChapterListSelector) + } + } + + var chapters []source.SChapter + chapterEls.Each(func(i int, el *goquery.Selection) { + ch := source.SChapter{} + el.Find(s.cfg.ChapterURLSelector).First().Each(func(_ int, a *goquery.Selection) { + if href, ok := a.Attr("href"); ok { + ch.URL = stripDomain(href+"?style=list", s.cfg.BaseURL) + } + ch.Name = strings.TrimSpace(a.Text()) + }) + el.Find(s.cfg.ChapterDateSelector).First().Each(func(_ int, span *goquery.Selection) { + ch.DateUpload = util.ParseRelativeDate(span.Text()) + }) + if ch.URL != "" { + chapters = append(chapters, ch) + } + }) + return chapters, nil +} + +func (s *Source) fetchChaptersAJAX(mangaURL, mangaID string) (*goquery.Document, error) { + ctx := context.Background() + if s.cfg.UseNewChapterEndpoint { + return s.post(ctx, mangaURL+"/ajax/chapters", url.Values{}) + } + form := url.Values{ + "action": {"manga_get_chapters"}, + "manga": {mangaID}, + } + doc, err := s.post(ctx, s.cfg.BaseURL+"/wp-admin/admin-ajax.php", form) + if err != nil { + // Fallback to new endpoint + return s.post(ctx, mangaURL+"/ajax/chapters", url.Values{}) + } + return doc, nil +} + +func (s *Source) GetPageList(chapter source.SChapter) ([]source.Page, error) { + rawURL := util.AbsURL(s.cfg.BaseURL, chapter.URL) + doc, err := s.get(context.Background(), rawURL) + if err != nil { + return nil, err + } + + // Check for chapter protector (AES-encrypted pages) + if doc.Find("#chapter-protector-data").Length() > 0 { + return s.parseProtectedPages(doc, rawURL) + } + + var pages []source.Page + doc.Find(s.cfg.PageListParseSelector).Each(func(i int, img *goquery.Selection) { + imgURL := imgAttr(img, s.cfg.BaseURL) + if imgURL != "" { + pages = append(pages, source.Page{Index: i, URL: rawURL, ImageURL: imgURL}) + } + }) + return pages, nil +} + +func (s *Source) parseProtectedPages(doc *goquery.Document, pageURL string) ([]source.Page, error) { + // Extract JSON image array from chapter-protector-data + data := doc.Find("#chapter-protector-data").Text() + // Look for image array in the data JSON + var result struct { + Images []string `json:"arrayofimages"` + } + if err := json.Unmarshal([]byte(data), &result); err == nil && len(result.Images) > 0 { + pages := make([]source.Page, len(result.Images)) + for i, img := range result.Images { + pages[i] = source.Page{Index: i, URL: pageURL, ImageURL: util.AbsURL(s.cfg.BaseURL, img)} + } + return pages, nil + } + return nil, fmt.Errorf("madara: could not parse protected chapter pages") +} + +func (s *Source) GetImageURL(page source.Page) (string, error) { + return page.ImageURL, nil +} + +func (s *Source) GetFilterList() []source.Filter { + return nil +} + +// --- helpers --- + +// stripDomain removes the base URL scheme+host from an absolute URL, leaving /path. +func stripDomain(href, baseURL string) string { + parsed, err := url.Parse(href) + if err != nil || !parsed.IsAbs() { + return href + } + base, err := url.Parse(baseURL) + if err != nil { + return href + } + if parsed.Host != base.Host { + return href + } + return parsed.RequestURI() +} + +// imgAttr returns the best image URL from common lazy-load attributes. +func imgAttr(img *goquery.Selection, baseURL string) string { + for _, attr := range []string{"data-lazy-src", "data-src", "data-cfsrc", "data-manga-src", "src"} { + if v, ok := img.Attr(attr); ok && v != "" && !strings.HasPrefix(v, "data:") { + return util.AbsURL(baseURL, v) + } + } + return "" +} diff --git a/sources/base/madtheme/madtheme.go b/sources/base/madtheme/madtheme.go new file mode 100644 index 0000000..3d425a1 --- /dev/null +++ b/sources/base/madtheme/madtheme.go @@ -0,0 +1,226 @@ +// Package madtheme implements the MadTheme WordPress base. +// All list types via GET {base}/search?page={n}&sort=... +// Pages extracted from JSON blob in `) + +// ExtractNextDataJSON extracts the JSON object from a NextJS __NEXT_DATA__ script tag. +func ExtractNextDataJSON(html string) (json.RawMessage, error) { + m := nextDataRe.FindStringSubmatch(html) + if len(m) < 2 { + return nil, nil + } + raw := strings.TrimSpace(m[1]) + return json.RawMessage(raw), nil +} + +// AbsURL resolves a potentially relative URL against a base URL string. +func AbsURL(base, ref string) string { + if ref == "" { + return "" + } + if strings.HasPrefix(ref, "http://") || strings.HasPrefix(ref, "https://") { + return ref + } + base = strings.TrimRight(base, "/") + if strings.HasPrefix(ref, "/") { + // absolute path — strip to origin + if i := strings.Index(base[8:], "/"); i >= 0 { + base = base[:8+i] + } + return base + ref + } + return base + "/" + ref +} + +// ImgAttr returns the best image src from common lazy-loading data attributes. +// Checks data-lazy-src, data-src, data-cfsrc, data-setbg, then falls back to src. +func ImgAttr(attrs map[string]string, baseURL string) string { + for _, key := range []string{"data-lazy-src", "data-src", "data-cfsrc", "data-setbg", "data-manga-src", "src"} { + if v := attrs[key]; v != "" { + return AbsURL(baseURL, v) + } + } + return "" +} diff --git a/sources/base/wpcomics/wpcomics.go b/sources/base/wpcomics/wpcomics.go new file mode 100644 index 0000000..32789b6 --- /dev/null +++ b/sources/base/wpcomics/wpcomics.go @@ -0,0 +1,193 @@ +// Package wpcomics implements the WPComics base. +// GET {base}/{popularPath}?page={n}; HTML scraping. +package wpcomics + +import ( + "context" + "fmt" + "net/http" + "strings" + + "github.com/PuerkitoBio/goquery" + + "goyomi/internal/httpclient" + "goyomi/internal/source" + "goyomi/sources/base/util" +) + +type Config struct { + Name string + BaseURL string + Lang string + PopularPath string // default "hot" +} + +type Source struct { + cfg Config + client *httpclient.Client + id int64 +} + +func New(cfg Config) *Source { + if cfg.PopularPath == "" { + cfg.PopularPath = "hot" + } + c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) + return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} +} + +func (s *Source) ID() int64 { return s.id } +func (s *Source) Name() string { return s.cfg.Name } +func (s *Source) Lang() string { return s.cfg.Lang } +func (s *Source) SupportsLatest() bool { return true } + +func (s *Source) get(ctx context.Context, rawURL string) (*goquery.Document, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) + if err != nil { + return nil, err + } + req.Header.Set("Referer", s.cfg.BaseURL+"/") + resp, err := s.client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("wpcomics: HTTP %d", resp.StatusCode) + } + return goquery.NewDocumentFromReader(resp.Body) +} + +func (s *Source) parseMangaList(doc *goquery.Document) source.MangasPage { + var mangas []source.SManga + doc.Find("div.items div.item, div.comic-item").Each(func(_ int, el *goquery.Selection) { + m := source.SManga{} + el.Find("h3 a, a.cover").First().Each(func(_ int, a *goquery.Selection) { + if href, ok := a.Attr("href"); ok { + m.URL = stripDomain(href, s.cfg.BaseURL) + } + m.Title = strings.TrimSpace(a.Text()) + }) + el.Find("img").First().Each(func(_ int, img *goquery.Selection) { + m.ThumbnailURL = imgAttr(img, s.cfg.BaseURL) + }) + if m.URL != "" { + mangas = append(mangas, m) + } + }) + hasNext := doc.Find(".pagination .next, a[rel=next]").Length() > 0 + return source.MangasPage{Mangas: mangas, HasNextPage: hasNext} +} + +func (s *Source) GetPopularManga(page int) (source.MangasPage, error) { + u := fmt.Sprintf("%s/%s", strings.TrimRight(s.cfg.BaseURL, "/"), s.cfg.PopularPath) + if page > 1 { + u += fmt.Sprintf("?page=%d", page) + } + doc, err := s.get(context.Background(), u) + if err != nil { + return source.MangasPage{}, err + } + return s.parseMangaList(doc), nil +} + +func (s *Source) GetLatestUpdates(page int) (source.MangasPage, error) { + u := fmt.Sprintf("%s/new?page=%d", strings.TrimRight(s.cfg.BaseURL, "/"), page) + doc, err := s.get(context.Background(), u) + if err != nil { + return source.MangasPage{}, err + } + return s.parseMangaList(doc), nil +} + +func (s *Source) GetSearchManga(page int, query string, filters []source.Filter) (source.MangasPage, error) { + u := fmt.Sprintf("%s/tim-kiem?q=%s&page=%d", strings.TrimRight(s.cfg.BaseURL, "/"), query, page) + doc, err := s.get(context.Background(), u) + if err != nil { + return source.MangasPage{}, err + } + return s.parseMangaList(doc), nil +} + +func (s *Source) GetMangaDetails(manga source.SManga) (source.SManga, error) { + doc, err := s.get(context.Background(), util.AbsURL(s.cfg.BaseURL, manga.URL)) + if err != nil { + return manga, err + } + result := source.SManga{URL: manga.URL} + doc.Find("h1").First().Each(func(_ int, el *goquery.Selection) { result.Title = strings.TrimSpace(el.Text()) }) + doc.Find("li.author p.col-xs-8").First().Each(func(_ int, el *goquery.Selection) { result.Author = strings.TrimSpace(el.Text()) }) + doc.Find("li.status p.col-xs-8").First().Each(func(_ int, el *goquery.Selection) { result.Status = util.StatusFromString(el.Text()) }) + doc.Find("div.detail-content p").First().Each(func(_ int, el *goquery.Selection) { result.Description = strings.TrimSpace(el.Text()) }) + doc.Find(".cover img, img.cover").First().Each(func(_ int, img *goquery.Selection) { result.ThumbnailURL = imgAttr(img, s.cfg.BaseURL) }) + var genres []string + doc.Find("li.kind a").Each(func(_ int, el *goquery.Selection) { + if t := strings.TrimSpace(el.Text()); t != "" { + genres = append(genres, t) + } + }) + result.Genre = strings.Join(genres, ", ") + return result, nil +} + +func (s *Source) GetChapterList(manga source.SManga) ([]source.SChapter, error) { + doc, err := s.get(context.Background(), util.AbsURL(s.cfg.BaseURL, manga.URL)) + if err != nil { + return nil, err + } + var chapters []source.SChapter + doc.Find("div.list-chapter li.row:not(.heading)").Each(func(_ int, el *goquery.Selection) { + ch := source.SChapter{} + el.Find("a").First().Each(func(_ int, a *goquery.Selection) { + if href, ok := a.Attr("href"); ok { + ch.URL = stripDomain(href, s.cfg.BaseURL) + } + ch.Name = strings.TrimSpace(a.Text()) + }) + el.Find("div.col-xs-4").First().Each(func(_ int, e *goquery.Selection) { + ch.DateUpload = util.ParseRelativeDate(e.Text()) + }) + if ch.URL != "" { + chapters = append(chapters, ch) + } + }) + return chapters, nil +} + +func (s *Source) GetPageList(chapter source.SChapter) ([]source.Page, error) { + rawURL := util.AbsURL(s.cfg.BaseURL, chapter.URL) + doc, err := s.get(context.Background(), rawURL) + if err != nil { + return nil, err + } + var pages []source.Page + doc.Find(".reading-detail img, .page-chapter img").Each(func(i int, img *goquery.Selection) { + if u := imgAttr(img, s.cfg.BaseURL); u != "" { + pages = append(pages, source.Page{Index: i, URL: rawURL, ImageURL: u}) + } + }) + return pages, nil +} + +func (s *Source) GetImageURL(page source.Page) (string, error) { return page.ImageURL, nil } +func (s *Source) GetFilterList() []source.Filter { return nil } + +func stripDomain(href, baseURL string) string { + if !strings.HasPrefix(href, "http") { + return href + } + base := strings.TrimRight(baseURL, "/") + if strings.HasPrefix(href, base) { + return href[len(base):] + } + return href +} + +func imgAttr(img *goquery.Selection, baseURL string) string { + for _, attr := range []string{"data-lazy-src", "data-src", "data-original", "src"} { + if v, ok := img.Attr(attr); ok && v != "" && !strings.HasPrefix(v, "data:") { + return util.AbsURL(baseURL, v) + } + } + return "" +}