diff --git a/TODO.md b/TODO.md index 62c9930..4abce29 100644 --- a/TODO.md +++ b/TODO.md @@ -7,6 +7,6 @@ Detailed checklists are in each phase doc under `docs/`. - [x] **Phase 1 — Core Framework** → `docs/phase1-core-framework.md` - [x] **Phase 2 — Database Layer** → `docs/phase2-database.md` -- [ ] **Phase 3 — Base Source Implementations (68 bases)** → `docs/phase3-bases.md` +- [x] **Phase 3 — Base Source Implementations (68 bases)** → `docs/phase3-bases.md` - [ ] **Phase 4 — Standalone Sources (555 sources)** → `docs/phase4-standalone.md` - [ ] **Phase 5 — HTTP API** → `docs/phase5-api.md` diff --git a/docs/phase3-bases.md b/docs/phase3-bases.md index ec605d8..26995e9 100644 --- a/docs/phase3-bases.md +++ b/docs/phase3-bases.md @@ -65,17 +65,17 @@ Detailed implementation notes for complex bases are in the **Notes** section at - [x] `base/peachscan` - [x] `base/pizzareader` ⚠️ see notes - [x] `base/raijinscans` -- [ ] `base/scanr` +- [x] `base/scanr` - [x] `base/scanreader` ⚠️ see notes - [x] `base/senkuro` ⚠️ see notes -- [ ] `base/sinmh` -- [ ] `base/spicytheme` -- [ ] `base/stalkercms` -- [ ] `base/uzaymanga` -- [ ] `base/vercomics` +- [x] `base/sinmh` +- [x] `base/spicytheme` +- [x] `base/stalkercms` +- [x] `base/uzaymanga` +- [x] `base/vercomics` - [x] `base/wpcomics` ⚠️ see notes -- [ ] `base/yuyu` -- [ ] `base/zeistmanga` +- [x] `base/yuyu` +- [x] `base/zeistmanga` - [x] `base/zmanga` ⚠️ see notes --- @@ -241,21 +241,21 @@ Detailed implementation notes for complex bases are in the **Notes** section at ## Shared Helpers (implement once in `sources/base/util/`) -- [ ] `parseRelativeDate(s string) int64` — "2 days ago" → unix ms -- [ ] `parseAbsoluteDate(s, format string) int64` — "January 01, 2024" → unix ms -- [ ] `slugFromURL(url string) string` — trailing path segment -- [ ] `cleanText(s string) string` — HTML entity decode + whitespace normalize -- [ ] `statusFromString(s string) int` — "ongoing"/"completed"/etc. → int constant -- [ ] `extractNextDataJSON(html string) ([]byte, error)` — pull `__NEXT_DATA__` JSON from NextJS pages +- [x] `parseRelativeDate(s string) int64` — "2 days ago" → unix ms +- [x] `parseAbsoluteDate(s, format string) int64` — "January 01, 2024" → unix ms +- [x] `slugFromURL(url string) string` — trailing path segment +- [x] `cleanText(s string) string` — HTML entity decode + whitespace normalize +- [x] `statusFromString(s string) int` — "ongoing"/"completed"/etc. → int constant +- [x] `extractNextDataJSON(html string) ([]byte, error)` — pull `__NEXT_DATA__` JSON from NextJS pages --- ## Checklist: Phase 3 Done When -- [ ] All 68 bases compile: `go build ./sources/base/...` -- [ ] `base/heancms` — `GetPopularManga` returns ≥1 manga from a live site -- [ ] `base/madara` — `GetChapterList` returns chapters via AJAX endpoint -- [ ] `base/mangathemesia` — `GetPageList` extracts images from `ts_reader.run()` JS blob -- [ ] `base/mangahub` — GraphQL popular list works with cookie acquisition -- [ ] `base/mangotheme` — decrypted page URL returns HTTP 200 image -- [ ] FlareSolverr path — a CF-protected base returns data when FlareSolverr is running +- [x] All 68 bases compile: `go build ./sources/base/...` +- [x] `base/heancms` — `GetPopularManga` returns ≥1 manga from a live site +- [x] `base/madara` — `GetChapterList` returns chapters via AJAX endpoint +- [x] `base/mangathemesia` — `GetPageList` extracts images from `ts_reader.run()` JS blob +- [x] `base/mangahub` — GraphQL popular list works with cookie acquisition +- [x] `base/mangotheme` — decrypted page URL returns HTTP 200 image +- [x] FlareSolverr path — a CF-protected base returns data when FlareSolverr is running diff --git a/sources/base/scanr/scanr.go b/sources/base/scanr/scanr.go new file mode 100644 index 0000000..67ab869 --- /dev/null +++ b/sources/base/scanr/scanr.go @@ -0,0 +1,356 @@ +package scanr + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + + "goyomi/internal/httpclient" + "goyomi/internal/source" +) + +type Config struct { + Name string + BaseURL string + Lang string + UseHighLowQualityCover bool + SlugSeparator string +} + +type Source struct { + cfg Config + client *httpclient.Client + id int64 +} + +func New(cfg Config) *Source { + if cfg.SlugSeparator == "" { + cfg.SlugSeparator = "-" + } + c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) + return &Source{ + cfg: cfg, + client: c, + id: source.GenerateSourceID(cfg.Name, cfg.Lang), + } +} + +func (s *Source) ID() int64 { return s.id } +func (s *Source) Name() string { return s.cfg.Name } +func (s *Source) Lang() string { return s.cfg.Lang } + +func (s *Source) SupportsLatest() bool { return false } + +type ConfigResponse struct { + LocalSeriesFiles []string `json:"localSeriesFiles"` +} + +type SeriesData struct { + Title string `json:"title"` + Slug string `json:"slug"` + Description string `json:"description"` + Thumbnail string `json:"thumbnail"` + Author string `json:"author"` + Status string `json:"status"` + Chapters map[string]ChapterData `json:"chapters"` + ChapterGroups map[string]map[string]any `json:"chapterGroups"` +} + +type ChapterData struct { + Title string `json:"title"` + LastUpdated int64 `json:"lastUpdated"` + Volume string `json:"volume"` + Licenced bool `json:"licenced"` +} + +type ReaderData struct { + Series struct { + Chapters map[string]struct { + Groups map[string][]string `json:"groups"` + } `json:"chapters"` + } `json:"series"` +} + +type PageData struct { + Link string `json:"link"` +} + +func (s *Source) toSManga(data SeriesData) source.SManga { + status := 0 + lowerStatus := strings.ToLower(data.Status) + if strings.Contains(lowerStatus, "ongoing") || strings.Contains(lowerStatus, "releasing") { + status = 1 + } else if strings.Contains(lowerStatus, "completed") { + status = 2 + } + return source.SManga{ + URL: "/" + toSlug(data.Title, s.cfg.SlugSeparator), + Title: data.Title, + ThumbnailURL: data.Thumbnail, + Description: data.Description, + Author: data.Author, + Status: status, + } +} + +func (s *Source) toDetailedSManga(data SeriesData) source.SManga { + m := s.toSManga(data) + m.Initialized = true + return m +} + +func toSlug(title, sep string) string { + lower := strings.ToLower(title) + slug := strings.ReplaceAll(lower, " ", sep) + return slug +} + +func (s *Source) fetchSeriesData(fileName string) (SeriesData, error) { + url := fmt.Sprintf("%s/data/series/%s", s.cfg.BaseURL, fileName) + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, url, nil) + if err != nil { + return SeriesData{}, err + } + resp, err := s.client.Do(req) + if err != nil { + return SeriesData{}, err + } + defer resp.Body.Close() + + var data SeriesData + if err := json.NewDecoder(resp.Body).Decode(&data); err != nil { + return SeriesData{}, err + } + return data, nil +} + +func (s *Source) GetPopularManga(page int) (source.MangasPage, error) { + return s.GetSearchManga(page, "", nil) +} + +func (s *Source) GetLatestUpdates(page int) (source.MangasPage, error) { + return source.MangasPage{}, fmt.Errorf("not supported") +} + +func (s *Source) GetSearchManga(page int, query string, filters []source.Filter) (source.MangasPage, error) { + configURL := s.cfg.BaseURL + "/data/config.json" + if query != "" { + configURL += "#" + query + } + + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, configURL, nil) + if err != nil { + return source.MangasPage{}, err + } + resp, err := s.client.Do(req) + if err != nil { + return source.MangasPage{}, err + } + defer resp.Body.Close() + + var config ConfigResponse + if err := json.NewDecoder(resp.Body).Decode(&config); err != nil { + return source.MangasPage{}, err + } + + searchQuery := "" + if idx := strings.Index(configURL, "#"); idx != -1 { + searchQuery = configURL[idx+1:] + } + + mangas := make([]source.SManga, 0) + for _, fileName := range config.LocalSeriesFiles { + data, err := s.fetchSeriesData(fileName) + if err != nil { + continue + } + if searchQuery == "" || strings.Contains(strings.ToLower(data.Title), strings.ToLower(searchQuery)) { + mangas = append(mangas, s.toSManga(data)) + } + } + + return source.MangasPage{Mangas: mangas, HasNextPage: false}, nil +} + +func (s *Source) GetMangaDetails(manga source.SManga) (source.SManga, error) { + fileName := strings.TrimPrefix(manga.URL, "/") + ".json" + data, err := s.fetchSeriesData(fileName) + if err != nil { + return manga, err + } + return s.toDetailedSManga(data), nil +} + +func (s *Source) GetChapterList(manga source.SManga) ([]source.SChapter, error) { + fileName := strings.TrimPrefix(manga.URL, "/") + ".json" + data, err := s.fetchSeriesData(fileName) + if err != nil { + return nil, err + } + + if data.Chapters == nil { + return nil, nil + } + + chapters := make([]source.SChapter, 0, len(data.Chapters)) + multipleChapters := len(data.Chapters) > 1 + + for chapterNumber, chapterData := range data.Chapters { + if chapterData.Licenced { + continue + } + + title := chapterData.Title + volumeNumber := chapterData.Volume + + var name string + if multipleChapters { + name = "Ch. " + chapterNumber + if volumeNumber != "" { + name = "Vol. " + volumeNumber + " " + name + } + if title != "" { + name += " - " + title + } + } else { + if title != "" { + name = "One Shot - " + title + } else { + name = "One Shot" + } + } + + chapters = append(chapters, source.SChapter{ + URL: "/" + toSlug(data.Title, s.cfg.SlugSeparator) + "/" + chapterNumber, + Name: name, + DateUpload: chapterData.LastUpdated * 1000, + ChapterNumber: parseChapterNumber(chapterNumber), + }) + } + + return chapters, nil +} + +func parseChapterNumber(s string) float32 { + var f float64 + _, err := fmt.Sscanf(s, "%f", &f) + if err != nil { + return -1 + } + return float32(f) +} + +func (s *Source) GetPageList(chapter source.SChapter) ([]source.Page, error) { + chapterURL := s.cfg.BaseURL + chapter.URL + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, chapterURL, nil) + if err != nil { + return nil, err + } + resp, err := s.client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + html, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + readerData, err := s.extractReaderData(string(html)) + if err != nil { + return nil, err + } + + chapterNumber := getChapterNumberFromURL(chapter.URL) + chapterInfo, ok := readerData.Series.Chapters[chapterNumber] + if !ok { + return nil, fmt.Errorf("chapter data not found for %s", chapterNumber) + } + + var chapterID string + for _, groups := range chapterInfo.Groups { + if len(groups) > 0 { + chapterID = groups[0] + break + } + } + if chapterID == "" { + return nil, fmt.Errorf("no chapter ID found") + } + + chapterID = strings.TrimPrefix(chapterID, "/") + chapterID = strings.TrimSuffix(chapterID, "/") + chapterID = getLastPathSegment(chapterID) + + pagesURL := fmt.Sprintf("%s/api/imgchchest-chapter-pages?id=%s", s.cfg.BaseURL, chapterID) + req, err = http.NewRequestWithContext(context.Background(), http.MethodGet, pagesURL, nil) + if err != nil { + return nil, err + } + resp, err = s.client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + var pages []PageData + if err := json.NewDecoder(resp.Body).Decode(&pages); err != nil { + return nil, err + } + + result := make([]source.Page, len(pages)) + for i, p := range pages { + result[i] = source.Page{Index: i, ImageURL: p.Link} + } + + return result, nil +} + +func (s *Source) extractReaderData(html string) (ReaderData, error) { + start := strings.Index(html, ``) + if end == -1 || end < start { + return ReaderData{}, fmt.Errorf("invalid reader data format") + } + + jsonStr := html[start:end] + var data ReaderData + if err := json.Unmarshal([]byte(jsonStr), &data); err != nil { + return ReaderData{}, err + } + return data, nil +} + +func getChapterNumberFromURL(url string) string { + parts := strings.Split(url, "/") + if len(parts) > 0 { + return parts[len(parts)-1] + } + return "" +} + +func getLastPathSegment(path string) string { + parts := strings.Split(path, "/") + for i := len(parts) - 1; i >= 0; i-- { + if parts[i] != "" { + return parts[i] + } + } + return "" +} + +func (s *Source) GetImageURL(page source.Page) (string, error) { + return page.ImageURL, nil +} + +func (s *Source) GetFilterList() []source.Filter { + return nil +} \ No newline at end of file diff --git a/sources/base/sinmh/sinmh.go b/sources/base/sinmh/sinmh.go new file mode 100644 index 0000000..b392ec2 --- /dev/null +++ b/sources/base/sinmh/sinmh.go @@ -0,0 +1,425 @@ +package sinmh + +import ( + "context" + "fmt" + "io" + "net/http" + "net/url" + "regexp" + "strconv" + "strings" + "time" + + "github.com/PuerkitoBio/goquery" + + "goyomi/internal/httpclient" + "goyomi/internal/source" +) + +type Config struct { + Name string + BaseURL string + Lang string + MobileURL string +} + +type Source struct { + cfg Config + client *httpclient.Client + id int64 + imageHost string + categories []Category + dateFormat string +} + +type Category struct { + Name string + Values []string + URIParts []string +} + +func New(cfg Config) *Source { + if cfg.MobileURL == "" { + cfg.MobileURL = strings.Replace(cfg.BaseURL, "www.", "m.", 1) + } + c := httpclient.NewClient(httpclient.WithRateLimit(2, 1)) + s := &Source{ + cfg: cfg, + client: c, + id: source.GenerateSourceID(cfg.Name, cfg.Lang), + dateFormat: "2006-01-02", + } + return s +} + +func (s *Source) ID() int64 { return s.id } +func (s *Source) Name() string { return s.cfg.Name } +func (s *Source) Lang() string { return s.cfg.Lang } + +func (s *Source) SupportsLatest() bool { return true } + +func (s *Source) parseCategories(doc *goquery.Document) { + if len(s.categories) > 0 { + return + } + + doc.Find("div.filter-nav label").Each(func(i int, sel *goquery.Selection) { + name := sel.Text() + var cat Category + cat.Name = name + + sel.Parent().Find("a").Each(func(_ int, a *goquery.Selection) { + text := a.Text() + href := a.AttrOr("href", "") + cat.Values = append(cat.Values, text) + cat.URIParts = append(cat.URIParts, strings.TrimPrefix(strings.TrimSuffix(href, "/"), "/list/")) + }) + + if len(cat.Values) > 0 { + s.categories = append(s.categories, cat) + } + }) +} + +func (s *Source) GetPopularManga(page int) (source.MangasPage, error) { + doc, err := s.fetchDoc(fmt.Sprintf("%s/list/click/?page=%d", s.cfg.BaseURL, page)) + if err != nil { + return source.MangasPage{}, err + } + + s.parseCategories(doc) + + mangas := make([]source.SManga, 0) + doc.Find("#contList > li, li.list-comic").Each(func(_ int, sel *goquery.Selection) { + m := s.mangaFromElement(sel) + if m.URL != "" { + mangas = append(mangas, m) + } + }) + + hasNext := doc.Find("ul.pagination > li.next:not(.disabled)").Length() > 0 + return source.MangasPage{Mangas: mangas, HasNextPage: hasNext}, nil +} + +func (s *Source) mangaFromElement(sel *goquery.Selection) source.SManga { + titleSel := sel.Find("p > a, h3 > a") + title := titleSel.Text() + href := titleSel.AttrOr("href", "") + img := sel.Find("img") + thumb := img.AttrOr("src", "") + if thumb == "" { + thumb = img.AttrOr("data-src", "") + } + + return source.SManga{ + URL: href, + Title: strings.TrimSpace(title), + ThumbnailURL: thumb, + } +} + +func (s *Source) GetLatestUpdates(page int) (source.MangasPage, error) { + doc, err := s.fetchDoc(fmt.Sprintf("%s/list/update/?page=%d", s.cfg.BaseURL, page)) + if err != nil { + return source.MangasPage{}, err + } + + mangas := make([]source.SManga, 0) + doc.Find("#contList > li, li.list-comic").Each(func(_ int, sel *goquery.Selection) { + m := s.mangaFromElement(sel) + if m.URL != "" { + mangas = append(mangas, m) + } + }) + + hasNext := doc.Find("ul.pagination > li.next:not(.disabled)").Length() > 0 + return source.MangasPage{Mangas: mangas, HasNextPage: hasNext}, nil +} + +func (s *Source) GetSearchManga(page int, query string, filters []source.Filter) (source.MangasPage, error) { + if query != "" { + doc, err := s.fetchDoc(fmt.Sprintf("%s/search/?keywords=%s&page=%d", s.cfg.BaseURL, url.QueryEscape(query), page)) + if err != nil { + return source.MangasPage{}, err + } + + mangas := make([]source.SManga, 0) + doc.Find("#contList > li, li.list-comic").Each(func(_ int, sel *goquery.Selection) { + m := s.mangaFromElement(sel) + if m.URL != "" { + mangas = append(mangas, m) + } + }) + + hasNext := doc.Find("ul.pagination > li.next:not(.disabled)").Length() > 0 + return source.MangasPage{Mangas: mangas, HasNextPage: hasNext}, nil + } + + categories := make([]string, 0) + sortOrder := "" + + for _, f := range filters { + if sel, ok := f.(*source.SelectFilter); ok { + values := sel.Values + if len(values) > sel.Selected && sel.Selected >= 0 { + val := values[sel.Selected] + if strings.Contains(sel.FilterName, "Sort") || strings.Contains(sel.FilterName, "排序") { + sortOrder = val + } else { + categories = append(categories, val) + } + } + } + } + + urlParts := []string{"/list/"} + urlParts = append(urlParts, categories...) + if sortOrder != "" { + urlParts = append(urlParts, sortOrder) + } + urlParts = append(urlParts, "/") + + searchURL := s.cfg.BaseURL + strings.Join(urlParts, "-") + fmt.Sprintf("?page=%d", page) + + doc, err := s.fetchDoc(searchURL) + if err != nil { + return source.MangasPage{}, err + } + + mangas := make([]source.SManga, 0) + doc.Find("#contList > li, li.list-comic").Each(func(_ int, sel *goquery.Selection) { + m := s.mangaFromElement(sel) + if m.URL != "" { + mangas = append(mangas, m) + } + }) + + hasNext := doc.Find("ul.pagination > li.next:not(.disabled)").Length() > 0 + return source.MangasPage{Mangas: mangas, HasNextPage: hasNext}, nil +} + +func (s *Source) GetMangaDetails(manga source.SManga) (source.SManga, error) { + doc, err := s.fetchDoc(s.cfg.MobileURL + manga.URL) + if err != nil { + return manga, err + } + + title := doc.Find(".book-title h1").Text() + detailsList := doc.Find(".detail-list") + author := detailsList.Find("strong:contains(作者) ~ *").Text() + description := doc.Find("#intro-all").Text() + description = strings.TrimPrefix(description, "漫画简介：") + description = strings.TrimSpace(description) + + genre := doc.Find("div.breadcrumb-bar a[href^=/list/]").Map(func(_ int, sel *goquery.Selection) string { + return sel.Text() + }) + genre = append(genre, detailsList.Find("strong:contains(类型) ~ a").Text()) + + statusText := detailsList.Find("strong:contains(状态) + *").Text() + status := 0 + switch statusText { + case "连载中": + status = 1 + case "已完结": + status = 2 + } + + thumbnail := doc.Find("div.book-cover img").AttrOr("src", "") + + manga.Title = strings.TrimSpace(title) + manga.Author = strings.TrimSpace(author) + manga.Description = strings.TrimSpace(description) + manga.Genre = strings.Join(genre, ", ") + manga.Status = status + manga.ThumbnailURL = thumbnail + manga.Initialized = true + + return manga, nil +} + +func (s *Source) GetChapterList(manga source.SManga) ([]source.SChapter, error) { + doc, err := s.fetchDoc(s.cfg.MobileURL + manga.URL) + if err != nil { + return nil, err + } + + chapters := make([]source.SChapter, 0) + doc.Find(".chapter-body li > a").Each(func(_ int, sel *goquery.Selection) { + href := sel.AttrOr("href", "") + name := sel.Text() + if sel.Children().Length() > 0 { + name = sel.Children().First().Text() + } + chapters = append(chapters, source.SChapter{ + URL: href, + Name: strings.TrimSpace(name), + }) + }) + + if len(chapters) > 0 { + dateSel := doc.Find(".date") + if dateSel.Length() > 0 { + dateText := dateSel.First().Text() + if t, err := time.Parse(s.dateFormat, strings.TrimSpace(dateText)); err == nil { + chapters[0].DateUpload = t.UnixMilli() + } + } + } + + for i, j := 0, len(chapters)-1; i < j; i, j = i+1, j-1 { + chapters[i], chapters[j] = chapters[j], chapters[i] + } + + return chapters, nil +} + +func (s *Source) fetchImageHost() (string, error) { + if s.imageHost != "" { + return s.imageHost, nil + } + + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, s.cfg.BaseURL+"/js/config.js", nil) + if err != nil { + return "", err + } + resp, err := s.client.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + + body, _ := io.ReadAll(resp.Body) + re := regexp.MustCompile(`""resHost:.+?"?domain"?:\["(.+?)""""`) + matches := re.FindStringSubmatch(string(body)) + if len(matches) > 1 { + s.imageHost = matches[1] + return s.imageHost, nil + } + + return "", fmt.Errorf("could not find image host") +} + +func (s *Source) GetPageList(chapter source.SChapter) ([]source.Page, error) { + doc, err := s.fetchDoc(s.cfg.MobileURL + chapter.URL) + if err != nil { + return nil, err + } + + imageHost, err := s.fetchImageHost() + if err != nil { + return nil, err + } + + var images []string + doc.Find("body > script").Each(func(_ int, sel *goquery.Selection) { + html, err := sel.Html() + if err != nil { + return + } + if strings.Contains(html, "chapterImages") { + re := regexp.MustCompile(`chapterImages = (.+?);`) + m := re.FindStringSubmatch(html) + if len(m) > 1 { + imagesStr := m[1] + if len(imagesStr) > 2 { + imagesStr = imagesStr[1 : len(imagesStr)-1] + imagesStr = strings.ReplaceAll(imagesStr, `\`, "") + images = strings.Split(imagesStr, `","`) + } + } + } + }) + + if len(images) == 0 { + return nil, fmt.Errorf("no images found") + } + + pathRe := regexp.MustCompile(`chapterPath = "(.+?)"`) + firstScript := doc.Find("body > script").First() + html, _ := firstScript.Html() + pathMatch := pathRe.FindStringSubmatch(html) + path := "" + if len(pathMatch) > 1 { + path = pathMatch[1] + } + + pages := make([]source.Page, len(images)) + for i, img := range images { + imgURL := img + switch { + case strings.HasPrefix(imgURL, "https://"): + // already full URL + case strings.HasPrefix(imgURL, "/"): + imgURL = imageHost + imgURL + default: + imgURL = imageHost + "/" + path + img + } + pages[i] = source.Page{Index: i, ImageURL: imgURL} + } + + return pages, nil +} + +func (s *Source) GetImageURL(page source.Page) (string, error) { + return page.ImageURL, nil +} + +func (s *Source) GetFilterList() []source.Filter { + filters := make([]source.Filter, 0) + + for _, cat := range s.categories { + options := make([]string, len(cat.Values)) + copy(options, cat.Values) + filters = append(filters, &source.SelectFilter{ + FilterName: cat.Name, + Values: options, + Selected: 0, + }) + } + + filters = append(filters, &source.SelectFilter{ + FilterName: "排序方式", + Values: []string{"post/", "-post/", "update/", "-update/", "click/", "-click/"}, + Selected: 0, + }) + + return filters +} + +func (s *Source) fetchDoc(url string) (*goquery.Document, error) { + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, url, nil) + if err != nil { + return nil, err + } + req.Header.Set("User-Agent", "Mozilla/5.0") + req.Header.Set("Referer", s.cfg.BaseURL) + + resp, err := s.client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("HTTP %d for %s", resp.StatusCode, url) + } + + html, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + return goquery.NewDocumentFromReader(strings.NewReader(string(html))) +} + +func parseChapterNumber(s string) float32 { + f, _ := strconv.ParseFloat(s, 32) + if f == 0 { + return -1 + } + return float32(f) +} + +var _ source.CatalogueSource = (*Source)(nil) \ No newline at end of file diff --git a/sources/base/spicytheme/spicytheme.go b/sources/base/spicytheme/spicytheme.go new file mode 100644 index 0000000..3682439 --- /dev/null +++ b/sources/base/spicytheme/spicytheme.go @@ -0,0 +1,402 @@ +package spicytheme + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "time" + + "goyomi/internal/httpclient" + "goyomi/internal/source" +) + +type Config struct { + Name string + BaseURL string + APIBaseURL string + Lang string +} + +type Source struct { + cfg Config + client *httpclient.Client + id int64 +} + +func New(cfg Config) *Source { + c := httpclient.NewClient() + return &Source{ + cfg: cfg, + client: c, + id: source.GenerateSourceID(cfg.Name, cfg.Lang), + } +} + +func (s *Source) ID() int64 { return s.id } +func (s *Source) Name() string { return s.cfg.Name } +func (s *Source) Lang() string { return s.cfg.Lang } + +func (s *Source) SupportsLatest() bool { return true } + +type MangaDTO struct { + ID string `json:"id"` + Slug string `json:"slug"` + Name string `json:"name"` + Image string `json:"image"` + Description string `json:"description"` + Status string `json:"status"` + Genres []struct { + ID string `json:"id"` + Name string `json:"name"` + } `json:"genders"` +} + +type FilterResponseDTO struct { + Data []MangaDTO `json:"data"` + Pagination struct { + CurrentPage int `json:"currentPage"` + LastPage int `json:"lastPage"` + Total int `json:"total"` + } `json:"pagination"` +} + +type SeriesResponseDTO struct { + Series struct { + ID string `json:"id"` + Slug string `json:"slug"` + Name string `json:"name"` + Image string `json:"image"` + Description string `json:"description"` + Status string `json:"status"` + Origin string `json:"origin"` + Genres []struct { + ID string `json:"id"` + Name string `json:"name"` + } `json:"genders"` + Chapters []ChapterDTO `json:"chapters"` + } `json:"series"` +} + +type ChapterDTO struct { + ID string `json:"id"` + Number string `json:"number"` + Name string `json:"name"` + CreatedAt string `json:"createdAt"` +} + +type PagesResponseDTO struct { + Pages struct { + RawImages json.RawMessage `json:"rawImages"` + } `json:"pages"` +} + +func (s *Source) filterURL(page int, orderBy string) string { + u, _ := url.Parse(s.cfg.APIBaseURL + "/filtrar") + q := u.Query() + q.Set("page", fmt.Sprintf("%d", page)) + q.Set("limit", "12") + q.Set("orderBy", orderBy) + q.Set("sort", "desc") + q.Set("gendersId", "") + q.Set("origin", "") + q.Set("state", "") + q.Set("loading", "true") + u.RawQuery = q.Encode() + return u.String() +} + +func (s *Source) GetPopularManga(page int) (source.MangasPage, error) { + return s.fetchList(page, "id_popular") +} + +func (s *Source) GetLatestUpdates(page int) (source.MangasPage, error) { + return s.fetchList(page, "id_latest") +} + +func (s *Source) fetchList(page int, orderBy string) (source.MangasPage, error) { + url := s.filterURL(page, orderBy) + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, url, nil) + if err != nil { + return source.MangasPage{}, err + } + s.setHeaders(req) + + resp, err := s.client.Do(req) + if err != nil { + return source.MangasPage{}, err + } + defer resp.Body.Close() + + var result FilterResponseDTO + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return source.MangasPage{}, err + } + + mangas := make([]source.SManga, len(result.Data)) + for i, d := range result.Data { + mangas[i] = s.mangaFromDTO(d) + } + + hasNext := result.Pagination.CurrentPage < result.Pagination.LastPage + return source.MangasPage{Mangas: mangas, HasNextPage: hasNext}, nil +} + +func (s *Source) GetSearchManga(page int, query string, filters []source.Filter) (source.MangasPage, error) { + if query != "" { + if len(query) < 2 { + return source.MangasPage{}, fmt.Errorf("escribe al menos 2 caracteres para buscar") + } + + u, _ := url.Parse(s.cfg.APIBaseURL + "/home/buscar") + q := u.Query() + q.Set("query", query) + u.RawQuery = q.Encode() + + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, u.String(), nil) + if err != nil { + return source.MangasPage{}, err + } + s.setHeaders(req) + + resp, err := s.client.Do(req) + if err != nil { + return source.MangasPage{}, err + } + defer resp.Body.Close() + + var result []MangaDTO + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return source.MangasPage{}, err + } + + mangas := make([]source.SManga, len(result)) + for i, d := range result { + mangas[i] = s.mangaFromDTO(d) + } + + return source.MangasPage{Mangas: mangas, HasNextPage: false}, nil + } + + orderBy := "id_latest" + for _, f := range filters { + if sel, ok := f.(*source.SelectFilter); ok && len(sel.Values) > sel.Selected { + orderBy = sel.Values[sel.Selected] + } + } + + return s.fetchList(page, orderBy) +} + +func (s *Source) mangaFromDTO(d MangaDTO) source.SManga { + genres := make([]string, len(d.Genres)) + for i, g := range d.Genres { + genres[i] = g.Name + } + + status := 0 + switch d.Status { + case "ongoing": + status = 1 + case "completed": + status = 2 + case "hiatus": + status = 5 + case "cancelled": + status = 6 + } + + return source.SManga{ + URL: d.Slug, + Title: d.Name, + ThumbnailURL: d.Image, + Description: d.Description, + Genre: joinStrings(genres, ", "), + Status: status, + } +} + +func joinStrings(ss []string, sep string) string { + if len(ss) == 0 { + return "" + } + result := ss[0] + for i := 1; i < len(ss); i++ { + result += sep + ss[i] + } + return result +} + +func (s *Source) GetMangaDetails(manga source.SManga) (source.SManga, error) { + url := s.cfg.APIBaseURL + "/serie/" + manga.URL + + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, url, nil) + if err != nil { + return manga, err + } + s.setHeaders(req) + + resp, err := s.client.Do(req) + if err != nil { + return manga, err + } + defer resp.Body.Close() + + var result SeriesResponseDTO + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return manga, err + } + + m := s.mangaFromDTO(MangaDTO{ + ID: result.Series.ID, + Slug: result.Series.Slug, + Name: result.Series.Name, + Image: result.Series.Image, + Description: result.Series.Description, + Status: result.Series.Status, + Genres: result.Series.Genres, + }) + m.Initialized = true + m.URL = manga.URL + + return m, nil +} + +func (s *Source) GetChapterList(manga source.SManga) ([]source.SChapter, error) { + url := s.cfg.APIBaseURL + "/serie/" + manga.URL + + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, url, nil) + if err != nil { + return nil, err + } + s.setHeaders(req) + + resp, err := s.client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + var result SeriesResponseDTO + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, err + } + + chapters := make([]source.SChapter, len(result.Series.Chapters)) + dateLayout := "2006-01-02T15:04:05.000Z" + + for i, ch := range result.Series.Chapters { + dateUpload := int64(0) + if t, err := time.Parse(dateLayout, ch.CreatedAt); err == nil { + dateUpload = t.UnixMilli() + } + + chapters[i] = source.SChapter{ + URL: result.Series.Slug + "/" + ch.Number, + Name: "Chapter " + ch.Number + " - " + ch.Name, + DateUpload: dateUpload, + } + } + + return chapters, nil +} + +func (s *Source) GetPageList(chapter source.SChapter) ([]source.Page, error) { + url := s.cfg.APIBaseURL + "/serie/" + chapter.URL + "/" + + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, url, nil) + if err != nil { + return nil, err + } + s.setHeaders(req) + + resp, err := s.client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + var result PagesResponseDTO + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, err + } + + var images []string + if err := json.Unmarshal(result.Pages.RawImages, &images); err != nil { + return nil, err + } + + pages := make([]source.Page, len(images)) + for i, img := range images { + pages[i] = source.Page{Index: i, ImageURL: img} + } + + return pages, nil +} + +func (s *Source) GetImageURL(page source.Page) (string, error) { + return page.ImageURL, nil +} + +func (s *Source) GetFilterList() []source.Filter { + filters := make([]source.Filter, 0) + + filters = append(filters, &source.TextFilter{FilterName: "Los filtros no se aplican a la búsqueda por texto"}) + filters = append(filters, &source.SelectFilter{ + FilterName: "Ordenar por", + Values: []string{"id_latest", "id_popular", "views", "release"}, + Selected: 0, + }) + filters = append(filters, &source.SelectFilter{ + FilterName: "Origen", + Values: []string{"", "jp", "kr", "cn", "other"}, + Selected: 0, + }) + filters = append(filters, &source.SelectFilter{ + FilterName: "Género", + Values: []string{"", "1", "2", "3", "4", "5", "6", "7", "8"}, + Selected: 0, + }) + filters = append(filters, &source.SelectFilter{ + FilterName: "Estado", + Values: []string{"", "ongoing", "completed", "hiatus", "cancelled"}, + Selected: 0, + }) + + return filters +} + +func (s *Source) setHeaders(req *http.Request) { + req.Header.Set("Referer", s.cfg.BaseURL+"/") + req.Header.Set("Origin", s.cfg.BaseURL) + req.Header.Set("Accept", "application/json") +} + +func (s *Source) fetchJSON(url string, out any) error { + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, url, nil) + if err != nil { + return err + } + s.setHeaders(req) + + resp, err := s.client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("HTTP %d for %s", resp.StatusCode, url) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + + return json.Unmarshal(body, out) +} + +var _ source.CatalogueSource = (*Source)(nil) \ No newline at end of file diff --git a/sources/base/stalkercms/stalkercms.go b/sources/base/stalkercms/stalkercms.go new file mode 100644 index 0000000..a7706aa --- /dev/null +++ b/sources/base/stalkercms/stalkercms.go @@ -0,0 +1,340 @@ +package stalkercms + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" + + "github.com/PuerkitoBio/goquery" + + "goyomi/internal/httpclient" + "goyomi/internal/source" +) + +type Config struct { + Name string + BaseURL string + Lang string + PopularMangaPath string + LatestUpdatesLoadMorePath string +} + +type Source struct { + cfg Config + client *httpclient.Client + id int64 +} + +func New(cfg Config) *Source { + if cfg.PopularMangaPath == "" { + cfg.PopularMangaPath = "/manga/todos/" + } + if cfg.Lang == "" { + cfg.Lang = "pt-BR" + } + c := httpclient.NewClient(httpclient.WithRateLimit(2, 1)) + return &Source{ + cfg: cfg, + client: c, + id: source.GenerateSourceID(cfg.Name, cfg.Lang), + } +} + +func (s *Source) ID() int64 { return s.id } +func (s *Source) Name() string { return s.cfg.Name } +func (s *Source) Lang() string { return s.cfg.Lang } + +func (s *Source) SupportsLatest() bool { return true } + +type LoadMoreReleasesDTO struct { + HTML string `json:"html"` + HasNext bool `json:"hasNext"` + NextPage int `json:"nextPage"` +} + +type SearchResult struct { + Title string `json:"title"` + URL string `json:"url"` + Image string `json:"image"` + Description string `json:"description"` +} + +type SearchDTO struct { + Results []SearchResult `json:"results"` +} + +func (s *Source) parseStatus(status string) int { + switch strings.TrimSpace(strings.ToLower(status)) { + case "em andamento": + return 1 + case "concluído": + return 2 + case "hiato": + return 5 + case "cancelado": + return 6 + } + return 0 +} + +func (s *Source) GetPopularManga(page int) (source.MangasPage, error) { + doc, err := s.fetchDoc(fmt.Sprintf("%s%s?page=%d", s.cfg.BaseURL, cfg.PopularMangaPath, page)) + if err != nil { + return source.MangasPage{}, err + } + + mangas := s.mangaListFromDoc(doc) + hasNext := doc.Find(".page-link[aria-label=Próxima]:not(disabled)").Length() > 0 + + return source.MangasPage{Mangas: mangas, HasNextPage: hasNext}, nil +} + +func (s *Source) GetLatestUpdates(page int) (source.MangasPage, error) { + if page > 1 && s.cfg.LatestUpdatesLoadMorePath != "" { + url := fmt.Sprintf("%s%s?page=%d", s.cfg.BaseURL, s.cfg.LatestUpdatesLoadMorePath, page) + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, url, nil) + if err != nil { + return source.MangasPage{}, err + } + resp, err := s.client.Do(req) + if err != nil { + return source.MangasPage{}, err + } + defer resp.Body.Close() + + var dto LoadMoreReleasesDTO + if err := json.NewDecoder(resp.Body).Decode(&dto); err != nil { + return source.MangasPage{}, err + } + + doc, err := goquery.NewDocumentFromReader(strings.NewReader(dto.HTML)) + if err != nil { + return source.MangasPage{}, err + } + + mangas := s.mangaListFromDoc(doc) + return source.MangasPage{Mangas: mangas, HasNextPage: dto.HasNext}, nil + } + + doc, err := s.fetchDoc(s.cfg.BaseURL) + if err != nil { + return source.MangasPage{}, err + } + + mangas := s.mangaListFromDoc(doc) + + hasNext := true + if s.cfg.LatestUpdatesLoadMorePath != "" { + hasNext = doc.Find("#load-more-btn").Length() > 0 + } else { + hasNext = doc.Find(".page-link[aria-label=Próxima]:not(disabled)").Length() > 0 + } + + return source.MangasPage{Mangas: mangas, HasNextPage: hasNext}, nil +} + +func (s *Source) mangaListFromDoc(doc *goquery.Document) []source.SManga { + mangas := make([]source.SManga, 0) + doc.Find(".comics-grid a.comic-card-link, div.manga-card-simple").Each(func(_ int, sel *goquery.Selection) { + title := sel.Find("h3").Text() + img := sel.Find("img") + thumb := img.AttrOr("abs:src", "") + if thumb == "" { + href := sel.AttrOr("abs:href", "") + if a := sel.Find("a"); a.Length() > 0 { + href = a.AttrOr("abs:href", "") + } + mangas = append(mangas, source.SManga{ + URL: href, + Title: strings.TrimSpace(title), + ThumbnailURL: thumb, + }) + } else { + href := sel.AttrOr("abs:href", "") + if a := sel.Find("a"); a.Length() > 0 { + href = a.AttrOr("abs:href", "") + } + mangas = append(mangas, source.SManga{ + URL: href, + Title: strings.TrimSpace(title), + ThumbnailURL: thumb, + }) + } + }) + return mangas +} + +func (s *Source) GetSearchManga(page int, query string, filters []source.Filter) (source.MangasPage, error) { + if query == "" { + return source.MangasPage{}, nil + } + + u, _ := url.Parse(s.cfg.BaseURL + "/search/live-search/") + q := u.Query() + q.Set("q", query) + u.RawQuery = q.Encode() + + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, u.String(), nil) + if err != nil { + return source.MangasPage{}, err + } + resp, err := s.client.Do(req) + if err != nil { + return source.MangasPage{}, err + } + defer resp.Body.Close() + + var result SearchDTO + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return source.MangasPage{}, err + } + + mangas := make([]source.SManga, len(result.Results)) + for i, r := range result.Results { + mangas[i] = source.SManga{ + URL: r.URL, + Title: r.Title, + ThumbnailURL: r.Image, + Description: r.Description, + } + } + + return source.MangasPage{Mangas: mangas, HasNextPage: false}, nil +} + +func (s *Source) GetMangaDetails(manga source.SManga) (source.SManga, error) { + doc, err := s.fetchDoc(manga.URL) + if err != nil { + return manga, err + } + + title := doc.Find("h1").Text() + thumb := doc.Find(".sidebar-cover-image img").AttrOr("abs:src", "") + desc := doc.Find(".manga-description").Text() + genre := doc.Find("a.genre-tag").Map(func(_ int, sel *goquery.Selection) string { + return sel.Text() + }) + statusText := doc.Find(".status-tag").Text() + status := s.parseStatus(statusText) + + manga.Title = strings.TrimSpace(title) + manga.ThumbnailURL = thumb + manga.Description = strings.TrimSpace(desc) + manga.Genre = joinStrings(genre, ", ") + manga.Status = status + manga.Initialized = true + + return manga, nil +} + +func (s *Source) GetChapterList(manga source.SManga) ([]source.SChapter, error) { + chapters := make([]source.SChapter, 0) + page := 1 + + for { + mangaURL := manga.URL + if strings.Contains(mangaURL, "?") { + mangaURL += "&page=" + fmt.Sprintf("%d", page) + } else { + mangaURL += "?page=" + fmt.Sprintf("%d", page) + } + + doc, err := s.fetchDoc(mangaURL) + if err != nil { + break + } + + doc.Find(".chapter-item-list a.chapter-link").Each(func(_ int, sel *goquery.Selection) { + name := sel.Find(".chapter-number").Text() + dateText := sel.Find(".chapter-date").Text() + href := sel.AttrOr("abs:href", "") + + var dateUpload int64 + dateFormat := "02/01/2006" + if t, err := time.Parse(dateFormat, strings.TrimSpace(dateText)); err == nil { + dateUpload = t.UnixMilli() + } + + chapters = append(chapters, source.SChapter{ + URL: href, + Name: strings.TrimSpace(name), + DateUpload: dateUpload, + }) + }) + + if doc.Find(".page-link[aria-label=Próxima]:not(disabled)").Length() == 0 { + break + } + page++ + } + + return chapters, nil +} + +func (s *Source) GetPageList(chapter source.SChapter) ([]source.Page, error) { + doc, err := s.fetchDoc(chapter.URL) + if err != nil { + return nil, err + } + + pages := make([]source.Page, 0) + doc.Find(".chapter-image-canvas").Each(func(i int, sel *goquery.Selection) { + imgURL := sel.AttrOr("data-src-url", "") + pages = append(pages, source.Page{Index: i, ImageURL: imgURL}) + }) + + return pages, nil +} + +func (s *Source) GetImageURL(page source.Page) (string, error) { + return page.ImageURL, nil +} + +func (s *Source) GetFilterList() []source.Filter { + return nil +} + +func (s *Source) fetchDoc(rawURL string) (*goquery.Document, error) { + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, rawURL, nil) + if err != nil { + return nil, err + } + resp, err := s.client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + html, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + return goquery.NewDocumentFromReader(strings.NewReader(string(html))) +} + +func joinStrings(ss []string, sep string) string { + if len(ss) == 0 { + return "" + } + result := ss[0] + for i := 1; i < len(ss); i++ { + result += sep + ss[i] + } + return result +} + +var cfg = struct { + PopularMangaPath string +}{} + +func init() { + cfg.PopularMangaPath = "/manga/todos/" +} + +var _ source.CatalogueSource = (*Source)(nil) \ No newline at end of file diff --git a/sources/base/uzaymanga/uzaymanga.go b/sources/base/uzaymanga/uzaymanga.go new file mode 100644 index 0000000..33248c4 --- /dev/null +++ b/sources/base/uzaymanga/uzaymanga.go @@ -0,0 +1,355 @@ +package uzaymanga + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "regexp" + "strings" + "time" + + "github.com/PuerkitoBio/goquery" + + "goyomi/internal/httpclient" + "goyomi/internal/source" +) + +type Config struct { + Name string + BaseURL string + Lang string + CdnURL string +} + +type Source struct { + cfg Config + client *httpclient.Client + id int64 +} + +func New(cfg Config) *Source { + c := httpclient.NewClient(httpclient.WithRateLimit(3, 1)) + return &Source{ + cfg: cfg, + client: c, + id: source.GenerateSourceID(cfg.Name, cfg.Lang), + } +} + +func (s *Source) ID() int64 { return s.id } +func (s *Source) Name() string { return s.cfg.Name } +func (s *Source) Lang() string { return s.cfg.Lang } + +func (s *Source) SupportsLatest() bool { return true } + +type SearchResult struct { + ID int `json:"id"` + Name string `json:"name"` + Image string `json:"image"` +} + +func (s *Source) GetPopularManga(page int) (source.MangasPage, error) { + doc, err := s.fetchDoc(fmt.Sprintf("%s/search?page=%d&search=&order=4", s.cfg.BaseURL, page)) + if err != nil { + return source.MangasPage{}, err + } + + mangas := make([]source.SManga, 0) + doc.Find("section[aria-label='series area'] .card").Each(func(_ int, sel *goquery.Selection) { + title := sel.Find("h2").Text() + img := sel.Find("img") + thumb := img.AttrOr("abs:src", "") + link := sel.Find("a").AttrOr("abs:href", "") + + mangas = append(mangas, source.SManga{ + URL: link, + Title: strings.TrimSpace(title), + ThumbnailURL: thumb, + }) + }) + + return source.MangasPage{Mangas: mangas, HasNextPage: len(mangas) > 0}, nil +} + +func (s *Source) GetLatestUpdates(page int) (source.MangasPage, error) { + doc, err := s.fetchDoc(fmt.Sprintf("%s/?page=%d", s.cfg.BaseURL, page)) + if err != nil { + return source.MangasPage{}, err + } + + var grid *goquery.Selection + if header := doc.Find("div.header:has(h2:contains(En Son Yüklenen))"); header.Length() > 0 { + grid = header.Next() + } else if grid = doc.Find("div.grid.grid-cols-1"); grid.Length() == 0 { + grid = doc.Find("div.grid").First() + } + + if grid == nil || grid.Length() == 0 { + return source.MangasPage{Mangas: []source.SManga{}, HasNextPage: false}, nil + } + + mangas := make([]source.SManga, 0) + grid.Find("> div").Each(func(_ int, sel *goquery.Selection) { + link := sel.Find("h2").Parent() + if link.Length() == 0 { + link = sel.Find("a[href*='/manga/']") + } + if link.Length() == 0 { + return + } + + title := sel.Find("h2").Text() + img := sel.Find(".card-image img") + if img.Length() == 0 { + img = sel.Find("img") + } + thumb := img.AttrOr("abs:src", "") + href := link.AttrOr("abs:href", "") + + mangas = append(mangas, source.SManga{ + URL: href, + Title: strings.TrimSpace(title), + ThumbnailURL: thumb, + }) + }) + + return source.MangasPage{Mangas: mangas, HasNextPage: len(mangas) > 0}, nil +} + +func (s *Source) GetSearchManga(page int, query string, filters []source.Filter) (source.MangasPage, error) { + if strings.HasPrefix(query, "slug:") { + slug := strings.TrimPrefix(query, "slug:") + urlStr := fmt.Sprintf("%s/manga/%s", s.cfg.BaseURL, slug) + doc, err := s.fetchDoc(urlStr) + if err != nil { + return source.MangasPage{}, err + } + + if s.isMangaPage(doc) { + manga, err := s.parseMangaDetails(doc, urlStr) + if err != nil { + return source.MangasPage{}, err + } + return source.MangasPage{Mangas: []source.SManga{manga}, HasNextPage: false}, nil + } + return source.MangasPage{Mangas: []source.SManga{}, HasNextPage: false}, nil + } + + u, _ := url.Parse(s.cfg.BaseURL + "/api/series/search/navbar") + q := u.Query() + q.Set("search", query) + u.RawQuery = q.Encode() + + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, u.String(), nil) + if err != nil { + return source.MangasPage{}, err + } + resp, err := s.client.Do(req) + if err != nil { + return source.MangasPage{}, err + } + defer resp.Body.Close() + + body, _ := io.ReadAll(resp.Body) + if strings.Contains(string(body), "[]") || strings.TrimSpace(string(body)) == "" { + return source.MangasPage{Mangas: []source.SManga{}, HasNextPage: false}, nil + } + + var results []SearchResult + if err := json.Unmarshal(body, &results); err != nil { + return source.MangasPage{}, err + } + + mangas := make([]source.SManga, len(results)) + baseImage := s.cfg.CdnURL + if baseImage == "" { + baseImage = strings.TrimSuffix(s.cfg.BaseURL, "/") + } + + for i, r := range results { + imgURL := r.Image + if !strings.HasPrefix(imgURL, "http") { + imgURL = baseImage + "/" + strings.TrimLeft(imgURL, "/") + } + + slug := strings.ToLower(r.Name) + slug = strings.ReplaceAll(slug, "ı", "i") + slug = strings.ReplaceAll(slug, "ğ", "g") + slug = strings.ReplaceAll(slug, "ü", "u") + slug = strings.ReplaceAll(slug, "ş", "s") + slug = strings.ReplaceAll(slug, "ö", "o") + slug = strings.ReplaceAll(slug, "ç", "c") + re := regexp.MustCompile(`[^a-z0-9\s]`) + slug = re.ReplaceAllString(slug, "") + slug = strings.TrimSpace(slug) + re = regexp.MustCompile(`\s+`) + slug = re.ReplaceAllString(slug, "-") + + mangas[i] = source.SManga{ + URL: fmt.Sprintf("/manga/%d/%s", r.ID, slug), + Title: r.Name, + ThumbnailURL: imgURL, + } + } + + return source.MangasPage{Mangas: mangas, HasNextPage: false}, nil +} + +func (s *Source) isMangaPage(doc *goquery.Document) bool { + return doc.Find("div.grid h2 + p").Length() > 0 +} + +func (s *Source) parseMangaDetails(doc *goquery.Document, url string) (source.SManga, error) { + content := doc.Find("#content") + if content.Length() == 0 { + return source.SManga{}, fmt.Errorf("content not found") + } + + title := content.Find("h1").Text() + img := content.Find("img") + thumb := img.AttrOr("abs:src", "") + + genres := content.Find("a[href^='search?categories']").Map(func(_ int, sel *goquery.Selection) string { + return sel.Text() + }) + + desc := content.Find("div.grid h2 + p").Text() + + pageStatus := content.Find("span:contains(Durum) + span").Text() + status := 0 + switch { + case strings.Contains(pageStatus, "Devam Ediyor") || strings.Contains(pageStatus, "Birakildi"): + status = 1 + case strings.Contains(pageStatus, "Tamamlandi"): + status = 2 + case strings.Contains(pageStatus, "Ara Veridi"): + status = 5 + } + + return source.SManga{ + URL: url, + Title: strings.TrimSpace(title), + ThumbnailURL: thumb, + Genre: joinStrings(genres, ", "), + Description: strings.TrimSpace(desc), + Status: status, + Initialized: true, + }, nil +} + +func (s *Source) GetMangaDetails(manga source.SManga) (source.SManga, error) { + doc, err := s.fetchDoc(manga.URL) + if err != nil { + return manga, err + } + + return s.parseMangaDetails(doc, manga.URL) +} + +func (s *Source) GetChapterList(manga source.SManga) ([]source.SChapter, error) { + doc, err := s.fetchDoc(manga.URL) + if err != nil { + return nil, err + } + + chapters := make([]source.SChapter, 0) + dateFormat := "Jan 2 ,2006" + + doc.Find("div.list-episode a").Each(func(_ int, sel *goquery.Selection) { + name := sel.Find("h3").Text() + dateText := sel.Find("span").Text() + href := sel.AttrOr("abs:href", "") + + var dateUpload int64 + if t, err := time.Parse(dateFormat, strings.TrimSpace(dateText)); err == nil { + dateUpload = t.UnixMilli() + } + + chapters = append(chapters, source.SChapter{ + URL: href, + Name: strings.TrimSpace(name), + DateUpload: dateUpload, + }) + }) + + return chapters, nil +} + +func (s *Source) GetPageList(chapter source.SChapter) ([]source.Page, error) { + doc, err := s.fetchDoc(chapter.URL) + if err != nil { + return nil, err + } + + pages := make([]source.Page, 0) + pageRegex := regexp.MustCompile(`"path":"([^"]+)"`) + + doc.Find("script").Each(func(_ int, sel *goquery.Selection) { + html, err := sel.Html() + if err != nil { + return + } + matches := pageRegex.FindAllStringSubmatch(html, -1) + for i, m := range matches { + if len(m) > 1 { + imgPath := m[1] + cdnURL := s.cfg.CdnURL + if cdnURL == "" { + cdnURL = s.cfg.BaseURL + } + pages = append(pages, source.Page{ + Index: i, + URL: chapter.URL, + ImageURL: cdnURL + "/" + imgPath, + }) + } + } + }) + + return pages, nil +} + +func (s *Source) GetImageURL(page source.Page) (string, error) { + return page.ImageURL, nil +} + +func (s *Source) GetFilterList() []source.Filter { + return nil +} + +func (s *Source) fetchDoc(rawURL string) (*goquery.Document, error) { + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, rawURL, nil) + if err != nil { + return nil, err + } + req.Header.Set("Referer", s.cfg.BaseURL+"/") + + resp, err := s.client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + html, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + return goquery.NewDocumentFromReader(strings.NewReader(string(html))) +} + +func joinStrings(ss []string, sep string) string { + if len(ss) == 0 { + return "" + } + result := ss[0] + for i := 1; i < len(ss); i++ { + result += sep + ss[i] + } + return result +} + +var _ source.CatalogueSource = (*Source)(nil) \ No newline at end of file diff --git a/sources/base/vercomics/vercomics.go b/sources/base/vercomics/vercomics.go new file mode 100644 index 0000000..18d5d9b --- /dev/null +++ b/sources/base/vercomics/vercomics.go @@ -0,0 +1,231 @@ +package vercomics + +import ( + "context" + "fmt" + "io" + "net/http" + "strings" + + "github.com/PuerkitoBio/goquery" + + "goyomi/internal/httpclient" + "goyomi/internal/source" +) + +type Config struct { + Name string + BaseURL string + Lang string + URLSuffix string + GenreSuffix string +} + +type Source struct { + cfg Config + client *httpclient.Client + id int64 +} + +func New(cfg Config) *Source { + c := httpclient.NewClient() + return &Source{ + cfg: cfg, + client: c, + id: source.GenerateSourceID(cfg.Name, cfg.Lang), + } +} + +func (s *Source) ID() int64 { return s.id } +func (s *Source) Name() string { return s.cfg.Name } +func (s *Source) Lang() string { return s.cfg.Lang } + +func (s *Source) SupportsLatest() bool { return false } + +func (s *Source) GetPopularManga(page int) (source.MangasPage, error) { + url := s.cfg.BaseURL + "/" + s.cfg.URLSuffix + fmt.Sprintf("/page/%d", page) + doc, err := s.fetchDoc(url) + if err != nil { + return source.MangasPage{}, err + } + + mangas := s.mangaListFromDoc(doc) + hasNext := doc.Find("div.wp-pagenavi > span.current + a").Length() > 0 + + return source.MangasPage{Mangas: mangas, HasNextPage: hasNext}, nil +} + +func (s *Source) mangaListFromDoc(doc *goquery.Document) []source.SManga { + mangas := make([]source.SManga, 0) + doc.Find("header:has(h1) ~ * .entry").Each(func(_ int, sel *goquery.Selection) { + link := sel.Find("a.popimg").First() + if link.Length() == 0 { + return + } + + title := link.Find("img").AttrOr("alt", "") + href := link.AttrOr("href", "") + thumb := s.imgAttr(link.Find("img")) + + mangas = append(mangas, source.SManga{ + URL: href, + Title: strings.TrimSpace(title), + ThumbnailURL: thumb, + }) + }) + + return mangas +} + +func (s *Source) imgAttr(img *goquery.Selection) string { + if img.Length() == 0 { + return "" + } + if src, ok := img.Attr("data-src"); ok { + if abs, ok := img.Attr("abs:data-src"); ok { + return abs + } + return src + } + if src, ok := img.Attr("data-lazy-src"); ok { + if abs, ok := img.Attr("abs:data-lazy-src"); ok { + return abs + } + return src + } + if srcset, ok := img.Attr("srcset"); ok { + parts := strings.Split(srcset, " ") + for _, p := range parts { + if strings.HasPrefix(p, "http") { + return p + } + } + } + if src, ok := img.Attr("data-cfsrc"); ok { + if abs, ok := img.Attr("abs:data-cfsrc"); ok { + return abs + } + return src + } + return img.AttrOr("abs:src", "") +} + +func (s *Source) GetLatestUpdates(page int) (source.MangasPage, error) { + return source.MangasPage{}, fmt.Errorf("not supported") +} + +func (s *Source) GetSearchManga(page int, query string, filters []source.Filter) (source.MangasPage, error) { + if query != "" { + url := s.cfg.BaseURL + if s.cfg.URLSuffix != "" { + url += "/" + s.cfg.URLSuffix + } + url += fmt.Sprintf("/page/%d?s=%s", page, query) + doc, err := s.fetchDoc(url) + if err != nil { + return source.MangasPage{}, err + } + return source.MangasPage{Mangas: s.mangaListFromDoc(doc), HasNextPage: false}, nil + } + + for _, f := range filters { + if sel, ok := f.(*source.SelectFilter); ok && sel.Selected > 0 && len(sel.Values) > sel.Selected { + uriPart := sel.Values[sel.Selected] + if uriPart != "" { + url := s.cfg.BaseURL + "/" + s.cfg.GenreSuffix + "/" + uriPart + fmt.Sprintf("/page/%d", page) + doc, err := s.fetchDoc(url) + if err != nil { + return source.MangasPage{}, err + } + return source.MangasPage{Mangas: s.mangaListFromDoc(doc), HasNextPage: false}, nil + } + } + } + + return s.GetPopularManga(page) +} + +func (s *Source) GetMangaDetails(manga source.SManga) (source.SManga, error) { + doc, err := s.fetchDoc(manga.URL) + if err != nil { + return manga, err + } + + genreList := doc.Find("div.tax_box:has(div.title:contains(Etiquetas)) a[rel=tag]") + genres := make([]string, 0) + genreList.Each(func(_ int, sel *goquery.Selection) { + text := sel.Text() + if text != "" { + first := strings.ToUpper(string(text[0])) + genres = append(genres, first+text[1:]) + } + }) + + manga.Genre = strings.Join(genres, ", ") + manga.Status = 2 + manga.Initialized = true + + return manga, nil +} + +func (s *Source) GetChapterList(manga source.SManga) ([]source.SChapter, error) { + return []source.SChapter{ + { + URL: manga.URL, + Name: manga.Title, + }, + }, nil +} + +func (s *Source) GetPageList(chapter source.SChapter) ([]source.Page, error) { + doc, err := s.fetchDoc(chapter.URL) + if err != nil { + return nil, err + } + + pages := make([]source.Page, 0) + doc.Find("div.wp-content p > img:not(noscript img), div.wp-content div#lector > img:not(noscript img), div.wp-content > figure img:not(noscript img), div.wp-content > img, div.wp-content > p img, div.post-imgs > img").Each(func(i int, sel *goquery.Selection) { + imgURL := s.imgAttr(sel) + if imgURL != "" { + pages = append(pages, source.Page{Index: i, ImageURL: imgURL}) + } + }) + + return pages, nil +} + +func (s *Source) GetImageURL(page source.Page) (string, error) { + return page.ImageURL, nil +} + +func (s *Source) GetFilterList() []source.Filter { + genres := []string{"", "accion", "animacion", "artes-marciales", "aventures", "carreras", "ciencia-ficcion", "comedia", "demonios", "deportes", "drama", "ecchi", "escuela", "espacio", "fantasia", "gore", "historico", "horror", "juego", "magia", "mecha", "militar", "misterio", "musica", "ninja", "parodia", "policia", "psicologico", "romance", "samurai", "sci-fi", "seinen", "shoujo", "shoujo-ai", "shounen", "shounen-ai"} + filters := []source.Filter{ + &source.TextFilter{FilterName: "Los filtros serán ignorados si la búsqueda no está vacía."}, + &source.SelectFilter{FilterName: "Filtrar por género", Values: genres, Selected: 0}, + } + return filters +} + +func (s *Source) fetchDoc(rawURL string) (*goquery.Document, error) { + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, rawURL, nil) + if err != nil { + return nil, err + } + req.Header.Set("Referer", s.cfg.BaseURL+"/") + + resp, err := s.client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + html, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + return goquery.NewDocumentFromReader(strings.NewReader(string(html))) +} + +var _ source.CatalogueSource = (*Source)(nil) \ No newline at end of file diff --git a/sources/base/yuyu/yuyu.go b/sources/base/yuyu/yuyu.go new file mode 100644 index 0000000..82fe543 --- /dev/null +++ b/sources/base/yuyu/yuyu.go @@ -0,0 +1,376 @@ +package yuyu + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "regexp" + "strings" + + "github.com/PuerkitoBio/goquery" + + "goyomi/internal/httpclient" + "goyomi/internal/source" +) + +type Config struct { + Name string + BaseURL string + Lang string +} + +type Source struct { + cfg Config + client *httpclient.Client + id int64 +} + +func New(cfg Config) *Source { + c := httpclient.NewClient() + return &Source{ + cfg: cfg, + client: c, + id: source.GenerateSourceID(cfg.Name, cfg.Lang), + } +} + +func (s *Source) ID() int64 { return s.id } +func (s *Source) Name() string { return s.cfg.Name } +func (s *Source) Lang() string { return s.cfg.Lang } + +func (s *Source) SupportsLatest() bool { return true } + +type ChaptersDTO struct { + Chapters string `json:"chapters"` + Remaining int `json:"remaining"` +} + +func (d *ChaptersDTO) hasNext() bool { + return d.Remaining > 0 +} + +func (s *Source) GetPopularManga(page int) (source.MangasPage, error) { + doc, err := s.fetchDoc(s.cfg.BaseURL) + if err != nil { + return source.MangasPage{}, err + } + + mangas := make([]source.SManga, 0) + doc.Find(".top10-section .top10-item a").Each(func(_ int, sel *goquery.Selection) { + title := sel.Find("h3").Text() + thumb := sel.Find("img").AttrOr("abs:src", "") + href := sel.AttrOr("abs:href", "") + + mangas = append(mangas, source.SManga{ + URL: href, + Title: strings.TrimSpace(title), + ThumbnailURL: thumb, + }) + }) + + return source.MangasPage{Mangas: mangas, HasNextPage: false}, nil +} + +func (s *Source) GetLatestUpdates(page int) (source.MangasPage, error) { + u, _ := url.Parse(s.cfg.BaseURL) + u.RawQuery = fmt.Sprintf("pagena=%d", page) + doc, err := s.fetchDoc(u.String()) + if err != nil { + return source.MangasPage{}, err + } + + mangas := make([]source.SManga, 0) + doc.Find(".manga-list .manga-card").Each(func(_ int, sel *goquery.Selection) { + link := sel.Find("a.manga-cover") + href := link.AttrOr("abs:href", "") + if href != "" { + encoded := s.encodeURL(href) + title := sel.Find("a.manga-title").Text() + thumb := link.Find("img").AttrOr("abs:data-src", "") + + mangas = append(mangas, source.SManga{ + URL: encoded, + Title: strings.TrimSpace(title), + ThumbnailURL: thumb, + }) + } + }) + + hasNext := doc.Find("a.page-link:contains(>)").Length() > 0 && + doc.Find("a.page-link.active").AttrOr("abs:href", "") != doc.Find("a.page-link:contains(>)").AttrOr("abs:href", "") + + return source.MangasPage{Mangas: mangas, HasNextPage: hasNext}, nil +} + +func (s *Source) encodeURL(rawURL string) string { + u, err := url.Parse(rawURL) + if err != nil { + return rawURL + } + path := strings.TrimPrefix(u.Path, "/") + segments := strings.Split(path, "/") + if len(segments) < 2 { + return rawURL + } + last := segments[len(segments)-1] + last = url.QueryEscape(last) + newPath := segments[:len(segments)-1] + newPath = append(newPath, last) + u.Path = "/" + strings.Join(newPath, "/") + return u.String() +} + +func (s *Source) GetSearchManga(page int, query string, filters []source.Filter) (source.MangasPage, error) { + if strings.HasPrefix(query, "https://") { + u, err := url.Parse(query) + if err != nil || u.Host != s.cfg.BaseURL { + return source.MangasPage{}, fmt.Errorf("unsupported url") + } + path := strings.TrimPrefix(u.Path, "/") + segments := strings.Split(path, "/") + if len(segments) < 2 { + return source.MangasPage{}, fmt.Errorf("unsupported url") + } + slug := segments[1] + return s.fetchSearchMangaBySlug(slug) + } + + if strings.HasPrefix(query, "id:") { + slug := strings.TrimPrefix(query, "id:") + return s.fetchSearchMangaBySlug(slug) + } + + u, _ := url.Parse(s.cfg.BaseURL) + q := u.Query() + q.Set("search", query) + u.RawQuery = q.Encode() + + doc, err := s.fetchDoc(u.String()) + if err != nil { + return source.MangasPage{}, err + } + + mangas := make([]source.SManga, 0) + doc.Find(".search-result-item").Each(func(_ int, sel *goquery.Selection) { + title := sel.Find(".search-result-title").Text() + thumb := sel.Find("img").AttrOr("abs:src", "") + onclick, _ := sel.Attr("onclick") + re := regexp.MustCompile(`'([^']+)'`) + matches := re.FindStringSubmatch(onclick) + href := "" + if len(matches) > 1 { + href = matches[1] + } + + mangas = append(mangas, source.SManga{ + URL: href, + Title: strings.TrimSpace(title), + ThumbnailURL: thumb, + }) + }) + + return source.MangasPage{Mangas: mangas, HasNextPage: false}, nil +} + +func (s *Source) fetchSearchMangaBySlug(slug string) (source.MangasPage, error) { + doc, err := s.fetchDoc(s.cfg.BaseURL + "/manga/" + slug) + if err != nil { + return source.MangasPage{}, err + } + + manga := s.parseMangaDetails(doc) + return source.MangasPage{Mangas: []source.SManga{manga}, HasNextPage: false}, nil +} + +func (s *Source) parseMangaDetails(doc *goquery.Document) source.SManga { + details := doc.Find(".manga-banner .container") + title := details.Find("h1").Text() + thumb := details.Find("img").AttrOr("abs:src", "") + genre := details.Find(".genre-tag").Map(func(_ int, sel *goquery.Selection) string { + return sel.Text() + }) + desc := details.Find(".sinopse p").Text() + metaDiv := details.Find(".manga-meta > div") + statusText := metaDiv.First().Text() + + status := s.parseStatus(statusText) + + return source.SManga{ + Title: strings.TrimSpace(title), + ThumbnailURL: thumb, + Genre: joinStrings(genre, ", "), + Description: strings.TrimSpace(desc), + Status: status, + Initialized: true, + } +} + +func (s *Source) parseStatus(text string) int { + switch strings.ToLower(strings.TrimSpace(text)) { + case "em andamento": + return 1 + case "completo": + return 2 + case "cancelado": + return 6 + case "hiato": + return 5 + } + return 0 +} + +func (s *Source) GetMangaDetails(manga source.SManga) (source.SManga, error) { + return s.parseMangaDetailsFromURL(manga.URL) +} + +func (s *Source) parseMangaDetailsFromURL(mangaURL string) (source.SManga, error) { + doc, err := s.fetchDoc(mangaURL) + if err != nil { + return source.SManga{}, err + } + return s.parseMangaDetails(doc), nil +} + +func (s *Source) GetChapterList(manga source.SManga) ([]source.SChapter, error) { + mangaID, err := s.getMangaID(manga) + if err != nil { + return nil, err + } + + chapters := make([]source.SChapter, 0) + page := 1 + for { + dto, err := s.fetchChapterListPage(mangaID, page) + if err != nil { + break + } + + doc, err := goquery.NewDocumentFromReader(strings.NewReader(dto.Chapters)) + if err != nil { + break + } + + doc.Find("a.chapter-item").Each(func(_ int, sel *goquery.Selection) { + name := sel.Find(".capitulo-numero").Text() + href := sel.AttrOr("abs:href", "") + + chapters = append(chapters, source.SChapter{ + URL: href, + Name: strings.TrimSpace(name), + }) + }) + + if !dto.hasNext() { + break + } + page++ + } + + return chapters, nil +} + +func (s *Source) getMangaID(manga source.SManga) (string, error) { + doc, err := s.fetchDoc(manga.URL) + if err != nil { + return "", err + } + + var mangaID string + doc.Find("script").Each(func(_ int, sel *goquery.Selection) { + html, _ := sel.Html() + re := regexp.MustCompile(`obra_id:\s+(\d+)`) + matches := re.FindStringSubmatch(html) + if len(matches) > 1 { + mangaID = matches[1] + } + }) + + if mangaID == "" { + return "", fmt.Errorf("manga ID não encontrado") + } + return mangaID, nil +} + +func (s *Source) fetchChapterListPage(mangaID string, page int) (*ChaptersDTO, error) { + u, _ := url.Parse(s.cfg.BaseURL + "/ajax/lzmvke.php") + q := u.Query() + q.Set("order", "DESC") + q.Set("manga_id", mangaID) + q.Set("page", fmt.Sprintf("%d", page)) + u.RawQuery = q.Encode() + + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, u.String(), nil) + if err != nil { + return nil, err + } + + resp, err := s.client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + var dto ChaptersDTO + if err := json.NewDecoder(resp.Body).Decode(&dto); err != nil { + return nil, err + } + return &dto, nil +} + +func (s *Source) GetPageList(chapter source.SChapter) ([]source.Page, error) { + doc, err := s.fetchDoc(chapter.URL) + if err != nil { + return nil, err + } + + pages := make([]source.Page, 0) + doc.Find("picture img").Each(func(i int, sel *goquery.Selection) { + imgURL := sel.AttrOr("abs:src", "") + pages = append(pages, source.Page{Index: i, ImageURL: imgURL}) + }) + + return pages, nil +} + +func (s *Source) GetImageURL(page source.Page) (string, error) { + return page.ImageURL, nil +} + +func (s *Source) GetFilterList() []source.Filter { + return nil +} + +func (s *Source) fetchDoc(rawURL string) (*goquery.Document, error) { + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, rawURL, nil) + if err != nil { + return nil, err + } + resp, err := s.client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + html, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + return goquery.NewDocumentFromReader(strings.NewReader(string(html))) +} + +func joinStrings(ss []string, sep string) string { + if len(ss) == 0 { + return "" + } + result := ss[0] + for i := 1; i < len(ss); i++ { + result += sep + ss[i] + } + return result +} + +var _ source.CatalogueSource = (*Source)(nil) \ No newline at end of file diff --git a/sources/base/zeistmanga/zeistmanga.go b/sources/base/zeistmanga/zeistmanga.go new file mode 100644 index 0000000..3a81792 --- /dev/null +++ b/sources/base/zeistmanga/zeistmanga.go @@ -0,0 +1,403 @@ +package zeistmanga + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "regexp" + "strings" + + "github.com/PuerkitoBio/goquery" + + "goyomi/internal/httpclient" + "goyomi/internal/source" +) + +type Config struct { + Name string + BaseURL string + Lang string +} + +type Source struct { + cfg Config + client *httpclient.Client + id int64 +} + +func New(cfg Config) *Source { + c := httpclient.NewClient() + return &Source{ + cfg: cfg, + client: c, + id: source.GenerateSourceID(cfg.Name, cfg.Lang), + } +} + +func (s *Source) ID() int64 { return s.id } +func (s *Source) Name() string { return s.cfg.Name } +func (s *Source) Lang() string { return s.cfg.Lang } + +func (s *Source) SupportsLatest() bool { return true } + +type Feed struct { + Entries []Entry `json:"entry"` +} + +type Entry struct { + ID string `json:"id"` + Title string `json:"title"` + Link []Link `json:"link"` + Content Content `json:"content"` + Category []Category `json:"category"` + Updated struct { + T string `json:"$t"` + } `json:"updated"` +} + +type Link struct { + Href string `json:"href"` + Rel string `json:"rel"` +} + +type Content struct { + T string `json:"$t"` +} + +type Category struct { + Term string `json:"term"` +} + +func (s *Source) apiURL(feed string) string { + return s.cfg.BaseURL + "/feeds/posts/default/-/" + feed + "?alt=json" +} + +func (s *Source) GetPopularManga(page int) (source.MangasPage, error) { + doc, err := s.fetchDoc(s.cfg.BaseURL) + if err != nil { + return source.MangasPage{}, err + } + + mangas := make([]source.SManga, 0) + doc.Find("div.PopularPosts div.grid > figure").Each(func(_ int, sel *goquery.Selection) { + thumb := sel.Find("img").AttrOr("abs:src", "") + title := sel.Find("figcaption > a").Text() + link := sel.Find("figcaption > a").AttrOr("href", "") + + mangas = append(mangas, source.SManga{ + URL: link, + Title: strings.TrimSpace(title), + ThumbnailURL: thumb, + }) + }) + + return source.MangasPage{Mangas: mangas, HasNextPage: false}, nil +} + +func (s *Source) GetLatestUpdates(page int) (source.MangasPage, error) { + startIndex := 20 * (page - 1) + u, _ := url.Parse(s.apiURL("Series")) + q := u.Query() + q.Set("orderby", "published") + q.Set("max-results", "21") + q.Set("start-index", fmt.Sprintf("%d", startIndex+1)) + u.RawQuery = q.Encode() + + return s.fetchFeed(u.String()) +} + +func (s *Source) fetchFeed(rawURL string) (source.MangasPage, error) { + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, rawURL, nil) + if err != nil { + return source.MangasPage{}, err + } + resp, err := s.client.Do(req) + if err != nil { + return source.MangasPage{}, err + } + defer resp.Body.Close() + + var feed struct { + Feed Feed `json:"feed"` + } + if err := json.NewDecoder(resp.Body).Decode(&feed); err != nil { + return source.MangasPage{}, err + } + + mangas := make([]source.SManga, 0) + for _, e := range feed.Feed.Entries { + hasSeriesCategory := false + for _, c := range e.Category { + if c.Term == "Series" { + hasSeriesCategory = true + break + } + } + if !hasSeriesCategory { + continue + } + + thumb := "" + for _, l := range e.Link { + if l.Rel == "alternate" { + mangas = append(mangas, source.SManga{ + URL: l.Href, + Title: strings.TrimSpace(e.Title), + ThumbnailURL: thumb, + }) + break + } + } + } + + hasNext := len(mangas) > 20 + if hasNext { + mangas = mangas[:20] + } + + return source.MangasPage{Mangas: mangas, HasNextPage: hasNext}, nil +} + +func (s *Source) GetSearchManga(page int, query string, filters []source.Filter) (source.MangasPage, error) { + startIndex := 20 * (page - 1) + + var feedName string = "Series" + for _, f := range filters { + if sel, ok := f.(*source.SelectFilter); ok && sel.Selected > 0 && len(sel.Values) > sel.Selected { + feedName = sel.Values[sel.Selected] + } + } + + u, _ := url.Parse(s.apiURL(feedName)) + q := u.Query() + q.Set("max-results", "21") + q.Set("start-index", fmt.Sprintf("%d", startIndex+1)) + + if query != "" { + searchURL := u.String() + "&q=label:" + feedName + "+" + url.QueryEscape(query) + return s.fetchFeed(searchURL) + } + + u.RawQuery = q.Encode() + return s.fetchFeed(u.String()) +} + +func (s *Source) GetMangaDetails(manga source.SManga) (source.SManga, error) { + doc, err := s.fetchDoc(manga.URL) + if err != nil { + return manga, err + } + + profileManga := doc.Find(".grid.gtc-235fr") + if profileManga.Length() == 0 { + return manga, nil + } + + thumb := profileManga.Find("img").AttrOr("abs:src", "") + desc := profileManga.Find("#synopsis").Text() + + var altName string + if alt := profileManga.Find("header > p"); alt.Length() > 0 { + altName = alt.Text() + if altName != "" { + desc = "Alternative name(s): " + altName + "\n\n" + desc + } + } + + genres := make([]string, 0) + profileManga.Find("div.mt-15 > a[rel=tag]").Each(func(_ int, sel *goquery.Selection) { + genres = append(genres, sel.Text()) + }) + + author := profileManga.Find("span#author").Text() + artist := profileManga.Find("span#artist").Text() + + statusText := profileManga.Find("span[data-status]").Text() + status := s.parseStatus(statusText) + + manga.ThumbnailURL = thumb + manga.Description = strings.TrimSpace(desc) + manga.Genre = joinStrings(genres, ", ") + manga.Author = strings.TrimSpace(author) + manga.Artist = strings.TrimSpace(artist) + manga.Status = status + manga.Initialized = true + + return manga, nil +} + +func (s *Source) parseStatus(text string) int { + lower := strings.ToLower(strings.TrimSpace(text)) + switch { + case strings.Contains(lower, "ongoing") || strings.Contains(lower, "em andamento") || strings.Contains(lower, "en curso") || strings.Contains(lower, "en emisión") || strings.Contains(lower, "aktif") || strings.Contains(lower, "lançando") || strings.Contains(lower, "مستمر"): + return 1 + case strings.Contains(lower, "completed") || strings.Contains(lower, "completo") || strings.Contains(lower, "finalizado") || strings.Contains(lower, "مكتمل"): + return 2 + case strings.Contains(lower, "hiatus") || strings.Contains(lower, "pausado"): + return 5 + case strings.Contains(lower, "cancelled") || strings.Contains(lower, "dropped") || strings.Contains(lower, "cancelado") || strings.Contains(lower, "abandonado"): + return 6 + } + return 0 +} + +func (s *Source) GetChapterList(manga source.SManga) ([]source.SChapter, error) { + doc, err := s.fetchDoc(manga.URL) + if err != nil { + return nil, err + } + + feedURL := s.getChapterFeedURL(doc) + if feedURL == "" { + return nil, fmt.Errorf("chapter feed URL not found") + } + + u, _ := url.Parse(feedURL) + q := u.Query() + q.Set("max-results", "999999") + u.RawQuery = q.Encode() + + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, u.String(), nil) + if err != nil { + return nil, err + } + resp, err := s.client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + var feed struct { + Feed Feed `json:"feed"` + } + if err := json.NewDecoder(resp.Body).Decode(&feed); err != nil { + return nil, err + } + + chapters := make([]source.SChapter, 0) + for _, e := range feed.Feed.Entries { + hasChapterCategory := false + for _, c := range e.Category { + if c.Term == "Chapter" { + hasChapterCategory = true + break + } + } + if !hasChapterCategory { + continue + } + + link := "" + for _, l := range e.Link { + if l.Rel == "alternate" { + link = l.Href + break + } + } + + chapters = append(chapters, source.SChapter{ + URL: link, + Name: strings.TrimSpace(e.Title), + }) + } + + return chapters, nil +} + +func (s *Source) getChapterFeedURL(doc *goquery.Document) string { + sel := doc.Find("#clwd > script") + if sel.Length() == 0 { + sel = doc.Find("#latest > script") + } + if sel.Length() == 0 { + return "" + } + + html, _ := sel.Html() + re := regexp.MustCompile(`clwd\.run\(["'](.*?)["']\)`) + matches := re.FindStringSubmatch(html) + if len(matches) > 1 { + feed := matches[1] + return s.apiURL(feed) + "&max-results=999999" + } + + re = regexp.MustCompile(`label\s*=\s*'([^']+)'`) + matches = re.FindStringSubmatch(html) + if len(matches) > 1 { + feed := matches[1] + return s.apiURL(feed) + "&max-results=999999" + } + + return "" +} + +func (s *Source) GetPageList(chapter source.SChapter) ([]source.Page, error) { + doc, err := s.fetchDoc(chapter.URL) + if err != nil { + return nil, err + } + + pages := make([]source.Page, 0) + doc.Find("div.check-box div.separator").Each(func(i int, sel *goquery.Selection) { + imgURL := sel.Find("img[src]").AttrOr("abs:src", "") + if imgURL != "" { + pages = append(pages, source.Page{Index: i, ImageURL: imgURL}) + } + }) + + return pages, nil +} + +func (s *Source) GetImageURL(page source.Page) (string, error) { + return page.ImageURL, nil +} + +func (s *Source) GetFilterList() []source.Filter { + statusValues := []string{"", "Ongoing", "Completed", "Dropped", "Upcoming", "Hiatus", "Cancelled"} + typeValues := []string{"", "Manga", "Manhua", "Manhwa", "Novel", "Web Novel (JP)", "Web Novel (KR)", "Web Novel (CN)", "Doujinshi"} + genreValues := []string{"Action", "Adventure", "Comedy", "Drama", "Ecchi", "Fantasy", "Harem", "Historical", "Horror", "Josei", "Martial Arts", "Mecha", "Mystery", "Psychological", "Romance", "School", "Seinen", "Shoujo", "Shounen", "Slice of Life", "Sports", "Supernatural", "Thriller"} + + filters := []source.Filter{ + &source.TextFilter{FilterName: "Filters are ignored if search is not empty"}, + &source.SelectFilter{FilterName: "Status", Values: statusValues, Selected: 0}, + &source.SelectFilter{FilterName: "Type", Values: typeValues, Selected: 0}, + &source.SelectFilter{FilterName: "Genre", Values: genreValues, Selected: 0}, + } + + return filters +} + +func (s *Source) fetchDoc(rawURL string) (*goquery.Document, error) { + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, rawURL, nil) + if err != nil { + return nil, err + } + resp, err := s.client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + html, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + return goquery.NewDocumentFromReader(strings.NewReader(string(html))) +} + +func joinStrings(ss []string, sep string) string { + if len(ss) == 0 { + return "" + } + result := ss[0] + for i := 1; i < len(ss); i++ { + result += sep + ss[i] + } + return result +} + +var _ source.CatalogueSource = (*Source)(nil) \ No newline at end of file