achmad
e17de903b2
- Batch 5: mangawork, mangotheme, mccms, multichan → flare (Kotlin uses cloudflareClient) - Batch 6: natsuid → flare (Kotlin uses cloudflareClient) - Batch 7: gravureblogger → flare (Kotlin uses cloudflareClient) All verified against Kotlin multisrc reference.
337 lines
8.8 KiB
Go
Executable File
337 lines
8.8 KiB
Go
Executable File
// Package natsuid implements the Natsuid manga base.
|
|
// WP-based site; uses nonce-authenticated multipart POST for search/browse; WP JSON API for details.
|
|
package natsuid
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"mime/multipart"
|
|
"net/http"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/PuerkitoBio/goquery"
|
|
|
|
"goyomi/internal/httpclient/flare"
|
|
"goyomi/internal/source"
|
|
"goyomi/sources/base/util"
|
|
)
|
|
|
|
type Config struct {
|
|
Name string
|
|
BaseURL string
|
|
Lang string
|
|
PostType string // WP post type slug, e.g. "manga"
|
|
}
|
|
|
|
type Source struct {
|
|
cfg Config
|
|
client *flare.Client
|
|
id int64
|
|
}
|
|
|
|
func New(cfg Config) *Source {
|
|
if cfg.PostType == "" {
|
|
cfg.PostType = "manga"
|
|
}
|
|
c := flare.NewClient(flare.WithRateLimit(1, 2))
|
|
return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)}
|
|
}
|
|
|
|
func (s *Source) ID() int64 { return s.id }
|
|
func (s *Source) Name() string { return s.cfg.Name }
|
|
func (s *Source) Lang() string { return s.cfg.Lang }
|
|
func (s *Source) SupportsLatest() bool { return true }
|
|
|
|
func (s *Source) base() string { return strings.TrimRight(s.cfg.BaseURL, "/") }
|
|
|
|
type mangaURL struct {
|
|
ID int `json:"id"`
|
|
Slug string `json:"slug"`
|
|
}
|
|
|
|
func encodeMangaURL(id int, slug string) string {
|
|
b, _ := json.Marshal(mangaURL{ID: id, Slug: slug})
|
|
return string(b)
|
|
}
|
|
|
|
func decodeMangaURL(u string) (mangaURL, error) {
|
|
var m mangaURL
|
|
return m, json.Unmarshal([]byte(u), &m)
|
|
}
|
|
|
|
func (s *Source) getNonce(ctx context.Context) (string, error) {
|
|
u := s.base() + "/wp-admin/admin-ajax.php?type=search_form&action=get_nonce"
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
resp, err := s.client.Do(req)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
defer resp.Body.Close()
|
|
doc, err := goquery.NewDocumentFromReader(resp.Body)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
nonce := doc.Find("input[name=search_nonce]").AttrOr("value", "")
|
|
return nonce, nil
|
|
}
|
|
|
|
func (s *Source) postSearch(ctx context.Context, page int, sort string, query string) (*goquery.Document, error) {
|
|
nonce, err := s.getNonce(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
var buf bytes.Buffer
|
|
mw := multipart.NewWriter(&buf)
|
|
_ = mw.WriteField("nonce", nonce)
|
|
_ = mw.WriteField("page", fmt.Sprintf("%d", page))
|
|
_ = mw.WriteField("sort", sort)
|
|
_ = mw.WriteField("genre", "[]")
|
|
_ = mw.WriteField("genre_exclude", "[]")
|
|
_ = mw.WriteField("author", "[]")
|
|
_ = mw.WriteField("status", "[]")
|
|
if query != "" {
|
|
_ = mw.WriteField("search", query)
|
|
}
|
|
mw.Close()
|
|
|
|
u := s.base() + "/wp-admin/admin-ajax.php?action=advanced_search"
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodPost, u, &buf)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
req.Header.Set("Content-Type", mw.FormDataContentType())
|
|
resp, err := s.client.Do(req)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer resp.Body.Close()
|
|
if resp.StatusCode != http.StatusOK {
|
|
return nil, fmt.Errorf("natsuid: HTTP %d", resp.StatusCode)
|
|
}
|
|
return goquery.NewDocumentFromReader(resp.Body)
|
|
}
|
|
|
|
func (s *Source) parseMangaList(doc *goquery.Document) source.MangasPage {
|
|
var mangas []source.SManga
|
|
doc.Find("article.manga-card, div.manga-card, div.card-manga").Each(func(_ int, el *goquery.Selection) {
|
|
a := el.Find("a").First()
|
|
if a.Length() == 0 {
|
|
return
|
|
}
|
|
href := a.AttrOr("href", "")
|
|
if href == "" {
|
|
return
|
|
}
|
|
slug := util.SlugFromURL(strings.TrimRight(href, "/"))
|
|
m := source.SManga{
|
|
Title: strings.TrimSpace(el.Find("h2, h3, .manga-title, .title").First().Text()),
|
|
}
|
|
if m.Title == "" {
|
|
m.Title = strings.TrimSpace(a.AttrOr("title", ""))
|
|
}
|
|
if img := el.Find("img").First(); img.Length() > 0 {
|
|
m.ThumbnailURL = util.AbsURL(s.cfg.BaseURL, img.AttrOr("src", img.AttrOr("data-src", "")))
|
|
}
|
|
// We don't have WP post ID yet; store slug-only temporarily
|
|
m.URL = encodeMangaURL(0, slug)
|
|
if m.URL != "" && m.Title != "" {
|
|
mangas = append(mangas, m)
|
|
}
|
|
})
|
|
hasNext := doc.Find(".next, a[rel=next], .pagination .next").Length() > 0
|
|
return source.MangasPage{Mangas: mangas, HasNextPage: hasNext}
|
|
}
|
|
|
|
func (s *Source) GetPopularManga(page int) (source.MangasPage, error) {
|
|
doc, err := s.postSearch(context.Background(), page, "popular", "")
|
|
if err != nil {
|
|
return source.MangasPage{}, err
|
|
}
|
|
return s.parseMangaList(doc), nil
|
|
}
|
|
|
|
func (s *Source) GetLatestUpdates(page int) (source.MangasPage, error) {
|
|
doc, err := s.postSearch(context.Background(), page, "updated", "")
|
|
if err != nil {
|
|
return source.MangasPage{}, err
|
|
}
|
|
return s.parseMangaList(doc), nil
|
|
}
|
|
|
|
func (s *Source) GetSearchManga(page int, query string, filters []source.Filter) (source.MangasPage, error) {
|
|
doc, err := s.postSearch(context.Background(), page, "popular", query)
|
|
if err != nil {
|
|
return source.MangasPage{}, err
|
|
}
|
|
return s.parseMangaList(doc), nil
|
|
}
|
|
|
|
type wpPost struct {
|
|
ID int `json:"id"`
|
|
Slug string `json:"slug"`
|
|
Title struct {
|
|
Rendered string `json:"rendered"`
|
|
} `json:"title"`
|
|
Content struct {
|
|
Rendered string `json:"rendered"`
|
|
} `json:"content"`
|
|
Excerpt struct {
|
|
Rendered string `json:"rendered"`
|
|
} `json:"excerpt"`
|
|
Embedded struct {
|
|
FeaturedMedia [][]struct {
|
|
SourceURL string `json:"source_url"`
|
|
} `json:"wp:featuredmedia"`
|
|
} `json:"_embedded"`
|
|
}
|
|
|
|
func (s *Source) GetMangaDetails(manga source.SManga) (source.SManga, error) {
|
|
mu, err := decodeMangaURL(manga.URL)
|
|
if err != nil {
|
|
return manga, err
|
|
}
|
|
|
|
var u string
|
|
if mu.ID > 0 {
|
|
u = fmt.Sprintf("%s/wp-json/wp/v2/%s/%d?_embed", s.base(), s.cfg.PostType, mu.ID)
|
|
} else {
|
|
u = fmt.Sprintf("%s/wp-json/wp/v2/%s?slug=%s&_embed", s.base(), s.cfg.PostType, mu.Slug)
|
|
}
|
|
|
|
req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, u, nil)
|
|
if err != nil {
|
|
return manga, err
|
|
}
|
|
resp, err := s.client.Do(req)
|
|
if err != nil {
|
|
return manga, err
|
|
}
|
|
defer resp.Body.Close()
|
|
body, _ := io.ReadAll(resp.Body)
|
|
|
|
if mu.ID > 0 {
|
|
var post wpPost
|
|
if err := json.Unmarshal(body, &post); err != nil {
|
|
return manga, err
|
|
}
|
|
return s.postToManga(manga.URL, post), nil
|
|
}
|
|
|
|
var posts []wpPost
|
|
if err := json.Unmarshal(body, &posts); err != nil || len(posts) == 0 {
|
|
return manga, err
|
|
}
|
|
return s.postToManga(manga.URL, posts[0]), nil
|
|
}
|
|
|
|
func (s *Source) postToManga(originalURL string, post wpPost) source.SManga {
|
|
m := source.SManga{
|
|
URL: originalURL,
|
|
Title: util.CleanText(post.Title.Rendered),
|
|
}
|
|
if len(post.Embedded.FeaturedMedia) > 0 && len(post.Embedded.FeaturedMedia[0]) > 0 {
|
|
m.ThumbnailURL = post.Embedded.FeaturedMedia[0][0].SourceURL
|
|
}
|
|
desc := util.CleanText(post.Excerpt.Rendered)
|
|
if desc == "" {
|
|
desc = util.CleanText(post.Content.Rendered)
|
|
}
|
|
m.Description = desc
|
|
return m
|
|
}
|
|
|
|
func (s *Source) GetChapterList(manga source.SManga) ([]source.SChapter, error) {
|
|
mu, err := decodeMangaURL(manga.URL)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
pageURL := fmt.Sprintf("%s/%s/%s/", s.base(), s.cfg.PostType, mu.Slug)
|
|
req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, pageURL, nil)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
resp, err := s.client.Do(req)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer resp.Body.Close()
|
|
doc, err := goquery.NewDocumentFromReader(resp.Body)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
var chapters []source.SChapter
|
|
doc.Find("div a:has(time)").Each(func(_ int, el *goquery.Selection) {
|
|
href := el.AttrOr("href", "")
|
|
if href == "" {
|
|
return
|
|
}
|
|
name := strings.TrimSpace(el.Find("span, div").Not("time").First().Text())
|
|
if name == "" {
|
|
name = strings.TrimSpace(el.Text())
|
|
}
|
|
var ts int64
|
|
if t := el.Find("time").First(); t.Length() > 0 {
|
|
dt := t.AttrOr("datetime", t.Text())
|
|
ts = parseDate(strings.TrimSpace(dt))
|
|
}
|
|
chapters = append(chapters, source.SChapter{
|
|
URL: href,
|
|
Name: name,
|
|
DateUpload: ts,
|
|
})
|
|
})
|
|
return chapters, nil
|
|
}
|
|
|
|
func parseDate(s string) int64 {
|
|
formats := []string{
|
|
time.RFC3339,
|
|
"2006-01-02T15:04:05",
|
|
"2006-01-02",
|
|
"January 2, 2006",
|
|
}
|
|
for _, f := range formats {
|
|
if t, err := time.Parse(f, s); err == nil {
|
|
return t.UnixMilli()
|
|
}
|
|
}
|
|
return util.ParseRelativeDate(s)
|
|
}
|
|
|
|
func (s *Source) GetPageList(chapter source.SChapter) ([]source.Page, error) {
|
|
req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, util.AbsURL(s.cfg.BaseURL, chapter.URL), nil)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
resp, err := s.client.Do(req)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer resp.Body.Close()
|
|
doc, err := goquery.NewDocumentFromReader(resp.Body)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
var pages []source.Page
|
|
doc.Find("main .relative section > img").Each(func(i int, img *goquery.Selection) {
|
|
u := img.AttrOr("src", img.AttrOr("data-src", ""))
|
|
if u != "" {
|
|
pages = append(pages, source.Page{Index: i, ImageURL: util.AbsURL(s.cfg.BaseURL, u)})
|
|
}
|
|
})
|
|
return pages, nil
|
|
}
|
|
|
|
func (s *Source) GetImageURL(page source.Page) (string, error) { return page.ImageURL, nil }
|
|
func (s *Source) GetFilterList() []source.Filter { return nil }
|