diff --git a/docs/phase3-bases.md b/docs/phase3-bases.md index 939c7a0..f24c5a1 100644 --- a/docs/phase3-bases.md +++ b/docs/phase3-bases.md @@ -11,11 +11,11 @@ Detailed implementation notes for complex bases are in the **Notes** section at ## All Bases — 68 total - [x] `base/bakkin` ⚠️ see notes -- [ ] `base/colorlibanime` -- [ ] `base/comicaso` -- [ ] `base/comiciviewer` -- [ ] `base/eromuse` -- [ ] `base/ezmanhwa` +- [x] `base/colorlibanime` +- [x] `base/comicaso` +- [x] `base/comiciviewer` +- [x] `base/eromuse` +- [x] `base/ezmanhwa` - [ ] `base/fansubscat` - [x] `base/fmreader` ⚠️ see notes - [x] `base/foolslide` ⚠️ see notes diff --git a/sources/base/colorlibanime/colorlibanime.go b/sources/base/colorlibanime/colorlibanime.go new file mode 100644 index 0000000..3210045 --- /dev/null +++ b/sources/base/colorlibanime/colorlibanime.go @@ -0,0 +1,200 @@ +// Package colorlibanime implements the ColorlibAnime manga base. +// GET {base}/manga?page={n}&sort=view; CF-protected. +package colorlibanime + +import ( + "context" + "fmt" + "net/http" + "regexp" + "strings" + + "github.com/PuerkitoBio/goquery" + + "goyomi/internal/httpclient" + "goyomi/internal/source" + "goyomi/sources/base/util" +) + +type Config struct { + Name string + BaseURL string + Lang string +} + +type Source struct { + cfg Config + client *httpclient.Client + id int64 +} + +func New(cfg Config) *Source { + c := httpclient.NewClient(httpclient.WithRateLimit(3, 1)) + return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} +} + +func (s *Source) ID() int64 { return s.id } +func (s *Source) Name() string { return s.cfg.Name } +func (s *Source) Lang() string { return s.cfg.Lang } +func (s *Source) SupportsLatest() bool { return true } + +func (s *Source) base() string { return strings.TrimRight(s.cfg.BaseURL, "/") } + +func (s *Source) get(ctx context.Context, rawURL string) (*goquery.Document, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) + if err != nil { + return nil, err + } + req.Header.Set("Referer", s.cfg.BaseURL+"/") + resp, err := s.client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("colorlibanime: HTTP %d", resp.StatusCode) + } + return goquery.NewDocumentFromReader(resp.Body) +} + +func (s *Source) parseMangaList(doc *goquery.Document) source.MangasPage { + var mangas []source.SManga + doc.Find(".product__page__content .product__item").Each(func(_ int, el *goquery.Selection) { + m := source.SManga{} + el.Find("a.img-link").First().Each(func(_ int, a *goquery.Selection) { + m.URL, _ = a.Attr("href") + }) + m.Title = strings.TrimSpace(el.Find("h5").Text()) + // thumbnail from .set-bg data-setbg, strip query string + el.Find(".set-bg").First().Each(func(_ int, e *goquery.Selection) { + if v, ok := e.Attr("data-setbg"); ok { + if idx := strings.LastIndex(v, "?"); idx >= 0 { + v = v[:idx] + } + m.ThumbnailURL = v + } + }) + if m.URL != "" { + mangas = append(mangas, m) + } + }) + hasNext := doc.Find(".fa-angle-right").Length() > 0 + return source.MangasPage{Mangas: mangas, HasNextPage: hasNext} +} + +func (s *Source) listURL(page int, sort, search string) string { + u := fmt.Sprintf("%s/manga?page=%d&sort=%s", s.base(), page, sort) + if search != "" { + u += "&search=" + search + } + return u +} + +func (s *Source) GetPopularManga(page int) (source.MangasPage, error) { + doc, err := s.get(context.Background(), s.listURL(page, "view", "")) + if err != nil { + return source.MangasPage{}, err + } + return s.parseMangaList(doc), nil +} + +func (s *Source) GetLatestUpdates(page int) (source.MangasPage, error) { + doc, err := s.get(context.Background(), s.listURL(page, "updated", "")) + if err != nil { + return source.MangasPage{}, err + } + return s.parseMangaList(doc), nil +} + +func (s *Source) GetSearchManga(page int, query string, filters []source.Filter) (source.MangasPage, error) { + doc, err := s.get(context.Background(), s.listURL(page, "view", query)) + if err != nil { + return source.MangasPage{}, err + } + return s.parseMangaList(doc), nil +} + +func (s *Source) GetMangaDetails(manga source.SManga) (source.SManga, error) { + doc, err := s.get(context.Background(), util.AbsURL(s.cfg.BaseURL, manga.URL)) + if err != nil { + return manga, err + } + result := source.SManga{URL: manga.URL} + el := doc.Find(".anime__details__content") + result.Title = strings.TrimSpace(el.Find("h3").Text()) + if result.Title == "" { + result.Title = manga.Title + } + result.Author = strings.TrimSpace(el.Find("h3 + span").Text()) + result.Description = strings.TrimSpace(el.Find("p").Text()) + el.Find("li").Each(func(_ int, li *goquery.Selection) { + text := strings.ToLower(li.Text()) + if strings.Contains(text, "status") { + val := strings.TrimSpace(strings.SplitN(li.Text(), " ", 2)[1]) + result.Status = util.StatusFromString(val) + } + }) + // thumbnail from og:image + if v, ok := doc.Find("meta[property='og:image']").First().Attr("content"); ok { + result.ThumbnailURL = v + } + return result, nil +} + +var lastUpdatedRe = regexp.MustCompile(`Date\((\d+)\)`) + +func (s *Source) GetChapterList(manga source.SManga) ([]source.SChapter, error) { + doc, err := s.get(context.Background(), util.AbsURL(s.cfg.BaseURL, manga.URL)) + if err != nil { + return nil, err + } + // extract lastUpdated timestamp from script + var lastUpdated int64 + doc.Find("script").Each(func(_ int, el *goquery.Selection) { + if m := lastUpdatedRe.FindStringSubmatch(el.Text()); len(m) == 2 { + if ms, ok := parseInt64(m[1]); ok { + lastUpdated = ms + } + } + }) + var chapters []source.SChapter + doc.Find(".anime__details__episodes a").Each(func(_ int, a *goquery.Selection) { + ch := source.SChapter{ + Name: strings.TrimSpace(a.Text()), + DateUpload: lastUpdated, + } + ch.URL, _ = a.Attr("href") + if ch.URL != "" { + chapters = append(chapters, ch) + } + }) + return chapters, nil +} + +func (s *Source) GetPageList(chapter source.SChapter) ([]source.Page, error) { + doc, err := s.get(context.Background(), util.AbsURL(s.cfg.BaseURL, chapter.URL)) + if err != nil { + return nil, err + } + var pages []source.Page + doc.Find(".container .read-img > img").Each(func(i int, img *goquery.Selection) { + if u, ok := img.Attr("src"); ok && u != "" { + pages = append(pages, source.Page{Index: i, ImageURL: util.AbsURL(s.cfg.BaseURL, u)}) + } + }) + return pages, nil +} + +func (s *Source) GetImageURL(page source.Page) (string, error) { return page.ImageURL, nil } +func (s *Source) GetFilterList() []source.Filter { return nil } + +func parseInt64(s string) (int64, bool) { + var n int64 + for _, c := range s { + if c < '0' || c > '9' { + return 0, false + } + n = n*10 + int64(c-'0') + } + return n, true +} diff --git a/sources/base/comicaso/comicaso.go b/sources/base/comicaso/comicaso.go new file mode 100644 index 0000000..a28c2b6 --- /dev/null +++ b/sources/base/comicaso/comicaso.go @@ -0,0 +1,244 @@ +// Package comicaso implements the Comicaso manga base. +// Single static JSON index at {base}/wp-content/static/manga/index.json; paginated client-side. +package comicaso + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "sync" + + "goyomi/internal/httpclient" + "goyomi/internal/source" + "goyomi/sources/base/util" +) + +const pageSize = 20 + +type Config struct { + Name string + BaseURL string + Lang string +} + +type Source struct { + cfg Config + client *httpclient.Client + id int64 + + mu sync.Mutex + mangaList []mangaDTO +} + +type mangaDTO struct { + Slug string `json:"slug"` + Title string `json:"title"` + Thumbnail string `json:"thumbnail"` + Status string `json:"status"` + Genres []string `json:"genres"` + UpdatedAt int64 `json:"updated_at"` + MangaDate int64 `json:"manga_date"` +} + +type mangaDetailDTO struct { + Slug string `json:"slug"` + Title string `json:"title"` + Thumbnail string `json:"thumbnail"` + Synopsis string `json:"synopsis"` + Author string `json:"author"` + Artist string `json:"artist"` + Status string `json:"status"` + Genres []string `json:"genres"` + Chapters []chapterDTO `json:"chapters"` +} + +type chapterDTO struct { + Slug string `json:"slug"` + Title string `json:"title"` + Date int64 `json:"date"` +} + +type tokenResponseDTO struct { + Tokens map[string]string `json:"tokens"` + Expire int64 `json:"expire"` +} + +func New(cfg Config) *Source { + c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) + return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} +} + +func (s *Source) ID() int64 { return s.id } +func (s *Source) Name() string { return s.cfg.Name } +func (s *Source) Lang() string { return s.cfg.Lang } +func (s *Source) SupportsLatest() bool { return true } + +func (s *Source) base() string { return strings.TrimRight(s.cfg.BaseURL, "/") } + +func (s *Source) getJSON(ctx context.Context, rawURL string, out any) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) + if err != nil { + return err + } + req.Header.Set("Referer", s.cfg.BaseURL+"/") + resp, err := s.client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("comicaso: HTTP %d", resp.StatusCode) + } + body, _ := io.ReadAll(resp.Body) + return json.Unmarshal(body, out) +} + +func (s *Source) getMangaList(ctx context.Context) ([]mangaDTO, error) { + s.mu.Lock() + defer s.mu.Unlock() + if s.mangaList != nil { + return s.mangaList, nil + } + var list []mangaDTO + if err := s.getJSON(ctx, s.base()+"/wp-content/static/manga/index.json", &list); err != nil { + return nil, err + } + s.mangaList = list + return list, nil +} + +func toSManga(m mangaDTO) source.SManga { + return source.SManga{ + URL: m.Slug, + Title: m.Title, + ThumbnailURL: m.Thumbnail, + Genre: strings.Join(m.Genres, ", "), + Status: util.StatusFromString(m.Status), + } +} + +func (s *Source) paginate(list []mangaDTO, page int) source.MangasPage { + start := (page - 1) * pageSize + if start >= len(list) { + return source.MangasPage{} + } + end := start + pageSize + if end > len(list) { + end = len(list) + } + mangas := make([]source.SManga, end-start) + for i, m := range list[start:end] { + mangas[i] = toSManga(m) + } + return source.MangasPage{Mangas: mangas, HasNextPage: end < len(list)} +} + +func (s *Source) GetPopularManga(page int) (source.MangasPage, error) { + list, err := s.getMangaList(context.Background()) + if err != nil { + return source.MangasPage{}, err + } + return s.paginate(list, page), nil +} + +func (s *Source) GetLatestUpdates(page int) (source.MangasPage, error) { + list, err := s.getMangaList(context.Background()) + if err != nil { + return source.MangasPage{}, err + } + // sort by updated_at desc (already sorted by server usually, just use as-is) + return s.paginate(list, page), nil +} + +func (s *Source) GetSearchManga(page int, query string, filters []source.Filter) (source.MangasPage, error) { + list, err := s.getMangaList(context.Background()) + if err != nil { + return source.MangasPage{}, err + } + q := strings.ToLower(query) + var filtered []mangaDTO + for _, m := range list { + if strings.Contains(strings.ToLower(m.Title), q) { + filtered = append(filtered, m) + } + } + return s.paginate(filtered, page), nil +} + +func (s *Source) GetMangaDetails(manga source.SManga) (source.SManga, error) { + var detail mangaDetailDTO + if err := s.getJSON(context.Background(), fmt.Sprintf("%s/wp-content/static/manga/%s.json", s.base(), manga.URL), &detail); err != nil { + return manga, err + } + return source.SManga{ + URL: manga.URL, + Title: detail.Title, + Author: detail.Author, + Artist: detail.Artist, + Description: detail.Synopsis, + Genre: strings.Join(detail.Genres, ", "), + Status: util.StatusFromString(detail.Status), + ThumbnailURL: detail.Thumbnail, + }, nil +} + +func (s *Source) GetChapterList(manga source.SManga) ([]source.SChapter, error) { + var detail mangaDetailDTO + if err := s.getJSON(context.Background(), fmt.Sprintf("%s/wp-content/static/manga/%s.json", s.base(), manga.URL), &detail); err != nil { + return nil, err + } + chapters := make([]source.SChapter, len(detail.Chapters)) + for i, ch := range detail.Chapters { + chapters[i] = source.SChapter{ + URL: fmt.Sprintf("/komik/%s/%s/", manga.URL, ch.Slug), + Name: ch.Title, + DateUpload: ch.Date * 1000, + } + } + return chapters, nil +} + +func (s *Source) GetPageList(chapter source.SChapter) ([]source.Page, error) { + chapterURL := util.AbsURL(s.cfg.BaseURL, chapter.URL) + // acquire token for this chapter URL + tokenURL := s.base() + "/wp-json/wp/v2/token" + body, _ := json.Marshal(map[string]any{"urls": []string{chapterURL}}) + req, err := http.NewRequestWithContext(context.Background(), http.MethodPost, tokenURL, bytes.NewReader(body)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Referer", s.cfg.BaseURL+"/") + resp, err := s.client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + raw, _ := io.ReadAll(resp.Body) + var tokenResp tokenResponseDTO + pageURL := chapterURL + if json.Unmarshal(raw, &tokenResp) == nil { + if tok, ok := tokenResp.Tokens[chapterURL]; ok && tok != "" { + pageURL = chapterURL + "?t=" + tok + } + } + // GET page list JSON + var pages []struct { + URL string `json:"url"` + } + if err := s.getJSON(context.Background(), pageURL, &pages); err != nil { + return nil, err + } + result := make([]source.Page, len(pages)) + for i, p := range pages { + result[i] = source.Page{Index: i, ImageURL: util.AbsURL(s.cfg.BaseURL, p.URL)} + } + return result, nil +} + +func (s *Source) GetImageURL(page source.Page) (string, error) { return page.ImageURL, nil } +func (s *Source) GetFilterList() []source.Filter { return nil } diff --git a/sources/base/comiciviewer/comiciviewer.go b/sources/base/comiciviewer/comiciviewer.go new file mode 100644 index 0000000..9b3daad --- /dev/null +++ b/sources/base/comiciviewer/comiciviewer.go @@ -0,0 +1,279 @@ +// Package comiciviewer implements the ComiciViewer Japanese manga base. +// GET {base}/ranking/manga for popular; API viewer for pages (requires login). +package comiciviewer + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + + "github.com/PuerkitoBio/goquery" + + "goyomi/internal/httpclient" + "goyomi/internal/source" + "goyomi/sources/base/util" +) + +type Config struct { + Name string + BaseURL string + Lang string +} + +type Source struct { + cfg Config + client *httpclient.Client + id int64 +} + +func New(cfg Config) *Source { + c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) + return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} +} + +func (s *Source) ID() int64 { return s.id } +func (s *Source) Name() string { return s.cfg.Name } +func (s *Source) Lang() string { return s.cfg.Lang } +func (s *Source) SupportsLatest() bool { return true } + +func (s *Source) base() string { return strings.TrimRight(s.cfg.BaseURL, "/") } + +func (s *Source) get(ctx context.Context, rawURL string) (*goquery.Document, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) + if err != nil { + return nil, err + } + req.Header.Set("Referer", s.cfg.BaseURL+"/") + resp, err := s.client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("comiciviewer: HTTP %d", resp.StatusCode) + } + return goquery.NewDocumentFromReader(resp.Body) +} + +func srcsetToURL(s string) string { + // "//cdn.example.com/img.jpg 1x" → "https://cdn.example.com/img.jpg" + if idx := strings.Index(s, " "); idx >= 0 { + s = s[:idx] + } + if strings.HasPrefix(s, "//") { + return "https:" + s + } + return s +} + +func mangaFromElement(el *goquery.Selection) source.SManga { + m := source.SManga{} + el.Find("a").First().Each(func(_ int, a *goquery.Selection) { + m.URL, _ = a.Attr("href") + }) + m.Title = strings.TrimSpace(el.Find(".title-text").Text()) + el.Find("source").First().Each(func(_ int, e *goquery.Selection) { + if v, ok := e.Attr("data-srcset"); ok { + m.ThumbnailURL = srcsetToURL(v) + } + }) + return m +} + +func (s *Source) GetPopularManga(page int) (source.MangasPage, error) { + doc, err := s.get(context.Background(), s.base()+"/ranking/manga") + if err != nil { + return source.MangasPage{}, err + } + var mangas []source.SManga + doc.Find("div.ranking-box-vertical, div.ranking-box-vertical-top3").Each(func(_ int, el *goquery.Selection) { + m := mangaFromElement(el) + if m.URL != "" { + mangas = append(mangas, m) + } + }) + return source.MangasPage{Mangas: mangas, HasNextPage: false}, nil +} + +func (s *Source) GetLatestUpdates(page int) (source.MangasPage, error) { + doc, err := s.get(context.Background(), s.base()+"/category/manga") + if err != nil { + return source.MangasPage{}, err + } + var mangas []source.SManga + doc.Find("div.category-box-vertical").Each(func(_ int, el *goquery.Selection) { + m := mangaFromElement(el) + if m.URL != "" { + mangas = append(mangas, m) + } + }) + hasNext := doc.Find("li.mode-paging-active + li > a").Length() > 0 + return source.MangasPage{Mangas: mangas, HasNextPage: hasNext}, nil +} + +func (s *Source) GetSearchManga(page int, query string, filters []source.Filter) (source.MangasPage, error) { + doc, err := s.get(context.Background(), fmt.Sprintf("%s/search?keyword=%s", s.base(), query)) + if err != nil { + return source.MangasPage{}, err + } + var mangas []source.SManga + doc.Find("div.manga-store-item").Each(func(_ int, el *goquery.Selection) { + m := source.SManga{} + el.Find("a.c-ms-clk-article").First().Each(func(_ int, a *goquery.Selection) { + m.URL, _ = a.Attr("href") + }) + m.Title = strings.TrimSpace(el.Find("h2.manga-title").Text()) + el.Find("source").First().Each(func(_ int, e *goquery.Selection) { + if v, ok := e.Attr("data-srcset"); ok { + m.ThumbnailURL = srcsetToURL(v) + } + }) + if m.URL != "" { + mangas = append(mangas, m) + } + }) + hasNext := doc.Find("li.mode-paging-active + li > a").Length() > 0 + return source.MangasPage{Mangas: mangas, HasNextPage: hasNext}, nil +} + +func (s *Source) GetMangaDetails(manga source.SManga) (source.SManga, error) { + doc, err := s.get(context.Background(), util.AbsURL(s.cfg.BaseURL, manga.URL)) + if err != nil { + return manga, err + } + result := source.SManga{URL: manga.URL} + // title: last span in h1.series-h-title + doc.Find("h1.series-h-title span").Last().Each(func(_ int, el *goquery.Selection) { + result.Title = strings.TrimSpace(el.Text()) + }) + if result.Title == "" { + result.Title = manga.Title + } + result.Author = strings.TrimSpace(doc.Find("div.series-h-author").Text()) + result.Description = strings.TrimSpace(doc.Find("div.series-h-description, p.series-description").Text()) + var genres []string + doc.Find("a.series-h-tag-link").Each(func(_ int, el *goquery.Selection) { + t := strings.TrimPrefix(strings.TrimSpace(el.Text()), "#") + if t != "" { + genres = append(genres, t) + } + }) + result.Genre = strings.Join(genres, ", ") + doc.Find("div.series-h-img source").First().Each(func(_ int, e *goquery.Selection) { + if v, ok := e.Attr("data-srcset"); ok { + result.ThumbnailURL = srcsetToURL(v) + } + }) + return result, nil +} + +func (s *Source) GetChapterList(manga source.SManga) ([]source.SChapter, error) { + chURL := fmt.Sprintf("%s%s/list?s=1", s.base(), manga.URL) + doc, err := s.get(context.Background(), chURL) + if err != nil { + return nil, err + } + var chapters []source.SChapter + doc.Find("li.episode-item, .chapter-list li").Each(func(_ int, el *goquery.Selection) { + ch := source.SChapter{} + el.Find("a").First().Each(func(_ int, a *goquery.Selection) { + ch.URL, _ = a.Attr("href") + ch.Name = strings.TrimSpace(a.Find(".episode-title, .title-text").Text()) + if ch.Name == "" { + ch.Name = strings.TrimSpace(a.Text()) + } + }) + el.Find("time").First().Each(func(_ int, e *goquery.Selection) { + dt := e.AttrOr("datetime", e.Text()) + ch.DateUpload = util.ParseAbsoluteDate(dt, "2006-01-02 15:04:05") + }) + if ch.URL != "" { + chapters = append(chapters, ch) + } + }) + return chapters, nil +} + +func (s *Source) GetPageList(chapter source.SChapter) ([]source.Page, error) { + doc, err := s.get(context.Background(), util.AbsURL(s.cfg.BaseURL, chapter.URL)) + if err != nil { + return nil, err + } + viewer := doc.Find("#comici-viewer") + if viewer.Length() == 0 { + return nil, fmt.Errorf("comiciviewer: WebView login required to read this chapter") + } + viewerID := viewer.AttrOr("comici-viewer-id", "") + memberJWT := viewer.AttrOr("data-member-jwt", "") + if viewerID == "" { + return nil, fmt.Errorf("comiciviewer: could not find viewer ID") + } + // Step 1: get total pages + apiURL := fmt.Sprintf("%s/book/contentsInfo?comici-viewer-id=%s&user-id=%s&page-from=0&page-to=1", + s.base(), viewerID, memberJWT) + pages, err := s.fetchViewerPages(apiURL, viewerID, memberJWT) + if err != nil { + return nil, err + } + return pages, nil +} + +type viewerResponse struct { + TotalPages int `json:"totalPages"` + Result []struct { + ImageURL string `json:"imageUrl"` + Sort int `json:"sort"` + Scramble string `json:"scramble"` + } `json:"result"` +} + +func (s *Source) fetchViewerPages(initialURL, viewerID, jwt string) ([]source.Page, error) { + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, initialURL, nil) + if err != nil { + return nil, err + } + req.Header.Set("Referer", s.cfg.BaseURL+"/") + resp, err := s.client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + body, _ := io.ReadAll(resp.Body) + var vr viewerResponse + if err := json.Unmarshal(body, &vr); err != nil { + return nil, err + } + if vr.TotalPages == 0 { + return nil, fmt.Errorf("comiciviewer: no pages found") + } + // fetch all pages + allURL := fmt.Sprintf("%s/book/contentsInfo?comici-viewer-id=%s&user-id=%s&page-from=0&page-to=%d", + s.base(), viewerID, jwt, vr.TotalPages) + req2, _ := http.NewRequestWithContext(context.Background(), http.MethodGet, allURL, nil) + req2.Header.Set("Referer", s.cfg.BaseURL+"/") + resp2, err := s.client.Do(req2) + if err != nil { + return nil, err + } + defer resp2.Body.Close() + body2, _ := io.ReadAll(resp2.Body) + var vr2 viewerResponse + if err := json.Unmarshal(body2, &vr2); err != nil { + return nil, err + } + pages := make([]source.Page, len(vr2.Result)) + for i, p := range vr2.Result { + imgURL := p.ImageURL + if p.Scramble != "" { + imgURL += "#" + p.Scramble + } + pages[i] = source.Page{Index: p.Sort, ImageURL: imgURL} + } + return pages, nil +} + +func (s *Source) GetImageURL(page source.Page) (string, error) { return page.ImageURL, nil } +func (s *Source) GetFilterList() []source.Filter { return nil } diff --git a/sources/base/eromuse/eromuse.go b/sources/base/eromuse/eromuse.go new file mode 100644 index 0000000..d233bd7 --- /dev/null +++ b/sources/base/eromuse/eromuse.go @@ -0,0 +1,214 @@ +// Package eromuse implements the EroMuse adult manga base. +// GET {base}/comics/album/Various-Authors; album-based crawl. +package eromuse + +import ( + "context" + "fmt" + "net/http" + "strings" + + "github.com/PuerkitoBio/goquery" + + "goyomi/internal/httpclient" + "goyomi/internal/source" + "goyomi/sources/base/util" +) + +type Config struct { + Name string + BaseURL string + Lang string +} + +type Source struct { + cfg Config + client *httpclient.Client + id int64 +} + +func New(cfg Config) *Source { + c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) + return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} +} + +func (s *Source) ID() int64 { return s.id } +func (s *Source) Name() string { return s.cfg.Name } +func (s *Source) Lang() string { return s.cfg.Lang } +func (s *Source) SupportsLatest() bool { return true } + +func (s *Source) base() string { return strings.TrimRight(s.cfg.BaseURL, "/") } + +func (s *Source) get(ctx context.Context, rawURL string) (*goquery.Document, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) + if err != nil { + return nil, err + } + req.Header.Set("Referer", s.cfg.BaseURL+"/") + resp, err := s.client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("eromuse: HTTP %d", resp.StatusCode) + } + return goquery.NewDocumentFromReader(resp.Body) +} + +func (s *Source) parseMangaList(doc *goquery.Document) ([]source.SManga, string) { + var mangas []source.SManga + doc.Find("a.c-tile:has(img)").Each(func(_ int, el *goquery.Selection) { + // skip members-only + if el.Find(".members-only").Length() > 0 { + return + } + m := source.SManga{} + m.URL, _ = el.Attr("href") + m.Title = strings.TrimSpace(el.Find("span.title, .c-tile-title").Text()) + if m.Title == "" { + m.Title = strings.TrimSpace(el.AttrOr("title", "")) + } + el.Find("img").First().Each(func(_ int, img *goquery.Selection) { + m.ThumbnailURL = imgAttr(img, s.cfg.BaseURL) + }) + if m.URL != "" { + mangas = append(mangas, m) + } + }) + // next page selector + nextURL := "" + doc.Find(".pagination span.current + span a").First().Each(func(_ int, a *goquery.Selection) { + nextURL, _ = a.Attr("href") + }) + return mangas, nextURL +} + +func (s *Source) fetchPage(rawURL string) (source.MangasPage, error) { + doc, err := s.get(context.Background(), rawURL) + if err != nil { + return source.MangasPage{}, err + } + mangas, next := s.parseMangaList(doc) + return source.MangasPage{Mangas: mangas, HasNextPage: next != ""}, nil +} + +func (s *Source) GetPopularManga(page int) (source.MangasPage, error) { + u := fmt.Sprintf("%s/comics/album/Various-Authors", s.base()) + if page > 1 { + u = fmt.Sprintf("%s/comics/album/Various-Authors?page=%d", s.base(), page) + } + return s.fetchPage(u) +} + +func (s *Source) GetLatestUpdates(page int) (source.MangasPage, error) { + u := fmt.Sprintf("%s/comics/album/Various-Authors?sort=date", s.base()) + if page > 1 { + u = fmt.Sprintf("%s/comics/album/Various-Authors?sort=date&page=%d", s.base(), page) + } + return s.fetchPage(u) +} + +func (s *Source) GetSearchManga(page int, query string, filters []source.Filter) (source.MangasPage, error) { + u := fmt.Sprintf("%s/search?q=%s", s.base(), query) + if page > 1 { + u += fmt.Sprintf("&page=%d", page) + } + return s.fetchPage(u) +} + +func (s *Source) GetMangaDetails(manga source.SManga) (source.SManga, error) { + doc, err := s.get(context.Background(), util.AbsURL(s.cfg.BaseURL, manga.URL)) + if err != nil { + return manga, err + } + result := source.SManga{URL: manga.URL} + // breadcrumb: li[2] = author, li[3] = album + crumbs := doc.Find("div.top-menu-breadcrumb li") + if crumbs.Length() >= 2 { + result.Author = strings.TrimSpace(crumbs.Eq(1).Text()) + } + if crumbs.Length() >= 3 { + result.Title = strings.TrimSpace(crumbs.Eq(2).Text()) + } + if result.Title == "" { + result.Title = manga.Title + } + result.ThumbnailURL = imgAttr(doc.Find(".c-tile img, .album-cover img").First(), s.cfg.BaseURL) + result.Description = strings.TrimSpace(doc.Find(".album-description, .description").Text()) + return result, nil +} + +func (s *Source) GetChapterList(manga source.SManga) ([]source.SChapter, error) { + doc, err := s.get(context.Background(), util.AbsURL(s.cfg.BaseURL, manga.URL)) + if err != nil { + return nil, err + } + var chapters []source.SChapter + // linked chapters + doc.Find("a.c-tile[href*='/comics/']").Each(func(_ int, a *goquery.Selection) { + ch := source.SChapter{} + ch.URL, _ = a.Attr("href") + ch.Name = strings.TrimSpace(a.Find("span.title, .c-tile-title").Text()) + if ch.Name == "" { + ch.Name = strings.TrimSpace(a.AttrOr("title", ch.URL)) + } + if ch.URL != "" { + chapters = append(chapters, ch) + } + }) + // if no sub-chapters, the manga itself is the chapter + if len(chapters) == 0 { + chapters = append(chapters, source.SChapter{ + URL: manga.URL, + Name: manga.Title, + }) + } + return chapters, nil +} + +func (s *Source) GetPageList(chapter source.SChapter) ([]source.Page, error) { + doc, err := s.get(context.Background(), util.AbsURL(s.cfg.BaseURL, chapter.URL)) + if err != nil { + return nil, err + } + var pages []source.Page + doc.Find(".read-img img, .pages img, .comic-page img").Each(func(i int, img *goquery.Selection) { + if u := imgAttr(img, s.cfg.BaseURL); u != "" { + pages = append(pages, source.Page{Index: i, ImageURL: u}) + } + }) + // paginated pages: follow next links + nextURL := "" + doc.Find(".pagination span.current + span a").First().Each(func(_ int, a *goquery.Selection) { + nextURL, _ = a.Attr("href") + }) + for nextURL != "" && len(pages) < 500 { + nextDoc, err := s.get(context.Background(), util.AbsURL(s.cfg.BaseURL, nextURL)) + if err != nil { + break + } + nextDoc.Find(".read-img img, .pages img, .comic-page img").Each(func(_ int, img *goquery.Selection) { + if u := imgAttr(img, s.cfg.BaseURL); u != "" { + pages = append(pages, source.Page{Index: len(pages), ImageURL: u}) + } + }) + nextURL = "" + nextDoc.Find(".pagination span.current + span a").First().Each(func(_ int, a *goquery.Selection) { + nextURL, _ = a.Attr("href") + }) + } + return pages, nil +} + +func (s *Source) GetImageURL(page source.Page) (string, error) { return page.ImageURL, nil } +func (s *Source) GetFilterList() []source.Filter { return nil } + +func imgAttr(img *goquery.Selection, baseURL string) string { + for _, attr := range []string{"data-lazy-src", "data-src", "data-cfsrc", "src"} { + if v, ok := img.Attr(attr); ok && v != "" && !strings.HasPrefix(v, "data:") { + return util.AbsURL(baseURL, v) + } + } + return "" +} diff --git a/sources/base/ezmanhwa/ezmanhwa.go b/sources/base/ezmanhwa/ezmanhwa.go new file mode 100644 index 0000000..dfba481 --- /dev/null +++ b/sources/base/ezmanhwa/ezmanhwa.go @@ -0,0 +1,197 @@ +// Package ezmanhwa implements the EZManhwa JSON REST base. +// GET {apiUrl}/series?page={n}&perPage=20&sort=popular +package ezmanhwa + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + + "goyomi/internal/httpclient" + "goyomi/internal/source" + "goyomi/sources/base/util" +) + +type Config struct { + Name string + BaseURL string + APIURL string + Lang string +} + +type Source struct { + cfg Config + client *httpclient.Client + id int64 +} + +func New(cfg Config) *Source { + if cfg.APIURL == "" { + cfg.APIURL = cfg.BaseURL + } + c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) + return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} +} + +func (s *Source) ID() int64 { return s.id } +func (s *Source) Name() string { return s.cfg.Name } +func (s *Source) Lang() string { return s.cfg.Lang } +func (s *Source) SupportsLatest() bool { return true } + +func (s *Source) api() string { return strings.TrimRight(s.cfg.APIURL, "/") } + +func (s *Source) getJSON(ctx context.Context, rawURL string, out any) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) + if err != nil { + return err + } + req.Header.Set("Accept", "application/json, text/plain, */*") + req.Header.Set("Referer", s.cfg.BaseURL+"/") + resp, err := s.client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("ezmanhwa: HTTP %d", resp.StatusCode) + } + body, _ := io.ReadAll(resp.Body) + return json.Unmarshal(body, out) +} + +type seriesListDTO struct { + Data []seriesDTO `json:"data"` + TotalPages int `json:"totalPages"` + CurrentPage int `json:"current"` +} + +type seriesDTO struct { + Slug string `json:"slug"` + Title string `json:"title"` + Cover string `json:"cover"` + Synopsis string `json:"synopsis"` + Author string `json:"author"` + Status string `json:"status"` + Genres []string `json:"genres"` +} + +type chapterListDTO struct { + Data []chapterDTO `json:"data"` + TotalPages int `json:"totalPages"` + CurrentPage int `json:"current"` +} + +type chapterDTO struct { + Slug string `json:"slug"` + Number float64 `json:"number"` + Title string `json:"title"` + RequiresPurchase bool `json:"requiresPurchase"` + CreatedAt string `json:"createdAt"` +} + +type pageListDTO struct { + Images []struct{ URL string `json:"url"` } `json:"images"` + RequiresPurchase bool `json:"requiresPurchase"` +} + +func (s *Source) toSManga(m seriesDTO) source.SManga { + return source.SManga{ + URL: m.Slug, + Title: m.Title, + Author: m.Author, + Description: m.Synopsis, + Genre: strings.Join(m.Genres, ", "), + Status: util.StatusFromString(m.Status), + ThumbnailURL: m.Cover, + } +} + +func (s *Source) fetchSeries(ctx context.Context, page int, sort, search string) (source.MangasPage, error) { + var u string + if search != "" { + u = fmt.Sprintf("%s/series/search?page=%d&perPage=20&search=%s", s.api(), page, search) + } else { + u = fmt.Sprintf("%s/series?page=%d&perPage=20&sort=%s", s.api(), page, sort) + } + var result seriesListDTO + if err := s.getJSON(ctx, u, &result); err != nil { + return source.MangasPage{}, err + } + mangas := make([]source.SManga, len(result.Data)) + for i, m := range result.Data { + mangas[i] = s.toSManga(m) + } + hasNext := result.CurrentPage < result.TotalPages + return source.MangasPage{Mangas: mangas, HasNextPage: hasNext}, nil +} + +func (s *Source) GetPopularManga(page int) (source.MangasPage, error) { + return s.fetchSeries(context.Background(), page, "popular", "") +} +func (s *Source) GetLatestUpdates(page int) (source.MangasPage, error) { + return s.fetchSeries(context.Background(), page, "latest", "") +} +func (s *Source) GetSearchManga(page int, query string, filters []source.Filter) (source.MangasPage, error) { + return s.fetchSeries(context.Background(), page, "popular", query) +} + +func (s *Source) GetMangaDetails(manga source.SManga) (source.SManga, error) { + var result seriesDTO + if err := s.getJSON(context.Background(), fmt.Sprintf("%s/series/%s", s.api(), manga.URL), &result); err != nil { + return manga, err + } + out := s.toSManga(result) + out.URL = manga.URL + return out, nil +} + +func (s *Source) GetChapterList(manga source.SManga) ([]source.SChapter, error) { + var result chapterListDTO + if err := s.getJSON(context.Background(), + fmt.Sprintf("%s/series/%s/chapters?page=1&perPage=100&sort=desc", s.api(), manga.URL), + &result); err != nil { + return nil, err + } + var chapters []source.SChapter + for _, ch := range result.Data { + if ch.RequiresPurchase { + continue + } + numStr := fmt.Sprintf("%.0f", ch.Number) + if ch.Number != float64(int(ch.Number)) { + numStr = fmt.Sprintf("%g", ch.Number) + } + name := "Chapter " + numStr + if ch.Title != "" && ch.Title != numStr { + name += " - " + ch.Title + } + chapters = append(chapters, source.SChapter{ + // URL format: series/{seriesSlug}/chapters/{chSlug} + URL: fmt.Sprintf("series/%s/chapters/%s", manga.URL, ch.Slug), + Name: name, + DateUpload: util.ParseAbsoluteDate(ch.CreatedAt, "2006-01-02T15:04:05.000Z"), + }) + } + return chapters, nil +} + +func (s *Source) GetPageList(chapter source.SChapter) ([]source.Page, error) { + var result pageListDTO + if err := s.getJSON(context.Background(), fmt.Sprintf("%s/%s", s.api(), chapter.URL), &result); err != nil { + return nil, err + } + if result.RequiresPurchase { + return nil, fmt.Errorf("ezmanhwa: chapter requires purchase") + } + pages := make([]source.Page, len(result.Images)) + for i, img := range result.Images { + pages[i] = source.Page{Index: i, ImageURL: img.URL} + } + return pages, nil +} + +func (s *Source) GetImageURL(page source.Page) (string, error) { return page.ImageURL, nil } +func (s *Source) GetFilterList() []source.Filter { return nil }