From 50ac3f180a862dbbef6388db6693d2f37f67a49d Mon Sep 17 00:00:00 2001 From: achmad Date: Mon, 11 May 2026 06:59:53 +0700 Subject: [PATCH] phase3: add gravureblogger, greenshit, hotcomics, initmanga, keyoapp bases (45/68) --- docs/phase3-bases.md | 10 +- go.mod | 5 +- go.sum | 7 + sources/base/gravureblogger/gravureblogger.go | 213 +++++++ sources/base/greenshit/greenshit.go | 408 ++++++++++++++ sources/base/hotcomics/hotcomics.go | 228 ++++++++ sources/base/initmanga/initmanga.go | 522 ++++++++++++++++++ sources/base/keyoapp/keyoapp.go | 325 +++++++++++ 8 files changed, 1711 insertions(+), 7 deletions(-) create mode 100644 sources/base/gravureblogger/gravureblogger.go create mode 100644 sources/base/greenshit/greenshit.go create mode 100644 sources/base/hotcomics/hotcomics.go create mode 100644 sources/base/initmanga/initmanga.go create mode 100644 sources/base/keyoapp/keyoapp.go diff --git a/docs/phase3-bases.md b/docs/phase3-bases.md index 04220f0..049de1d 100644 --- a/docs/phase3-bases.md +++ b/docs/phase3-bases.md @@ -25,17 +25,17 @@ Detailed implementation notes for complex bases are in the **Notes** section at - [x] `base/gigaviewer` ⚠️ see notes - [x] `base/gmanga` ⚠️ see notes - [x] `base/goda` -- [ ] `base/gravureblogger` -- [ ] `base/greenshit` +- [x] `base/gravureblogger` +- [x] `base/greenshit` - [x] `base/grouple` ⚠️ see notes - [x] `base/guya` ⚠️ see notes - [x] `base/heancms` ⚠️ see notes - [x] `base/hentaihand` ⚠️ see notes -- [ ] `base/hotcomics` +- [x] `base/hotcomics` - [x] `base/iken` ⚠️ see notes -- [ ] `base/initmanga` +- [x] `base/initmanga` - [x] `base/kemono` ⚠️ see notes -- [ ] `base/keyoapp` +- [x] `base/keyoapp` - [x] `base/lectormoe` ⚠️ see notes - [x] `base/libgroup` ⚠️ see notes - [x] `base/liliana` ⚠️ see notes diff --git a/go.mod b/go.mod index 88a17b8..4964e79 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,8 @@ require ( github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect - golang.org/x/net v0.52.0 // indirect + golang.org/x/crypto v0.51.0 // indirect + golang.org/x/net v0.53.0 // indirect golang.org/x/sync v0.20.0 // indirect - golang.org/x/text v0.35.0 // indirect + golang.org/x/text v0.37.0 // indirect ) diff --git a/go.sum b/go.sum index 03251e1..f828446 100644 --- a/go.sum +++ b/go.sum @@ -83,6 +83,8 @@ golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliY golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI= +golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= @@ -99,6 +101,8 @@ golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0= golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw= +golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA= +golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -121,6 +125,7 @@ golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -141,6 +146,8 @@ golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8= golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA= +golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= +golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/sources/base/gravureblogger/gravureblogger.go b/sources/base/gravureblogger/gravureblogger.go new file mode 100644 index 0000000..5f3475f --- /dev/null +++ b/sources/base/gravureblogger/gravureblogger.go @@ -0,0 +1,213 @@ +// Package gravureblogger implements the GravureBlogger base. +// Uses Google Blogger JSON feed API: GET {base}/feeds/posts/default?alt=json&... +package gravureblogger + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" + + "github.com/PuerkitoBio/goquery" + + "goyomi/internal/httpclient" + "goyomi/internal/source" + "goyomi/sources/base/util" +) + +const maxResults = 30 + +type Config struct { + Name string + BaseURL string + Lang string +} + +type Source struct { + cfg Config + client *httpclient.Client + id int64 +} + +func New(cfg Config) *Source { + c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) + return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} +} + +func (s *Source) ID() int64 { return s.id } +func (s *Source) Name() string { return s.cfg.Name } +func (s *Source) Lang() string { return s.cfg.Lang } +func (s *Source) SupportsLatest() bool { return false } + +func (s *Source) base() string { return strings.TrimRight(s.cfg.BaseURL, "/") } + +// Blogger JSON DTOs +type bloggerDTO struct { + Feed bloggerFeedDTO `json:"feed"` +} +type bloggerFeedDTO struct { + Category []bloggerCategoryDTO `json:"category"` + Entry []bloggerEntryDTO `json:"entry"` +} +type bloggerCategoryDTO struct { + Term string `json:"term"` +} +type bloggerEntryDTO struct { + Title bloggerTextDTO `json:"title"` + Published bloggerTextDTO `json:"published"` + Content bloggerTextDTO `json:"content"` + Link []bloggerLinkDTO `json:"link"` + Category []bloggerCategoryDTO `json:"category"` +} +type bloggerLinkDTO struct { + Rel string `json:"rel"` + Href string `json:"href"` +} +type bloggerTextDTO struct { + T string `json:"$t"` +} + +func (s *Source) fetchFeed(ctx context.Context, page int, query string) (source.MangasPage, error) { + startIndex := maxResults*(page-1) + 1 + u := fmt.Sprintf("%s/feeds/posts/default?alt=json&max-results=%d&start-index=%d", + s.base(), maxResults, startIndex) + if query != "" { + u += "&q=" + query + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil) + if err != nil { + return source.MangasPage{}, err + } + req.Header.Set("Referer", s.cfg.BaseURL+"/") + resp, err := s.client.Do(req) + if err != nil { + return source.MangasPage{}, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return source.MangasPage{}, fmt.Errorf("gravureblogger: HTTP %d", resp.StatusCode) + } + body, _ := io.ReadAll(resp.Body) + var data bloggerDTO + if err := json.Unmarshal(body, &data); err != nil { + return source.MangasPage{}, err + } + + var mangas []source.SManga + for _, entry := range data.Feed.Entry { + altLink := "" + for _, l := range entry.Link { + if l.Rel == "alternate" { + altLink = l.Href + break + } + } + if altLink == "" { + continue + } + // Store relative path + "#published" for date retrieval + relURL := util.AbsURL(s.cfg.BaseURL, altLink) + mangaURL := relURL + "#" + entry.Published.T + + // Extract thumbnail from HTML content + thumbnail := "" + if doc, err := goquery.NewDocumentFromReader(strings.NewReader("" + entry.Content.T + "")); err == nil { + doc.Find("img").First().Each(func(_ int, img *goquery.Selection) { + thumbnail, _ = img.Attr("src") + }) + } + + var genres []string + for _, cat := range entry.Category { + if cat.Term != "" { + genres = append(genres, cat.Term) + } + } + + mangas = append(mangas, source.SManga{ + URL: mangaURL, + Title: entry.Title.T, + ThumbnailURL: thumbnail, + Genre: strings.Join(genres, ", "), + Status: source.StatusCompleted, + }) + } + return source.MangasPage{Mangas: mangas, HasNextPage: len(data.Feed.Entry) == maxResults}, nil +} + +func (s *Source) GetPopularManga(page int) (source.MangasPage, error) { + return s.fetchFeed(context.Background(), page, "") +} + +func (s *Source) GetLatestUpdates(page int) (source.MangasPage, error) { + return s.GetPopularManga(page) +} + +func (s *Source) GetSearchManga(page int, query string, filters []source.Filter) (source.MangasPage, error) { + return s.fetchFeed(context.Background(), page, query) +} + +func (s *Source) GetMangaDetails(manga source.SManga) (source.SManga, error) { + // All data already in manga from list; just return as-is. + return manga, nil +} + +func (s *Source) GetChapterList(manga source.SManga) ([]source.SChapter, error) { + // URL is "...path#published_date" + hash := "" + chURL := manga.URL + if idx := strings.LastIndex(manga.URL, "#"); idx >= 0 { + hash = manga.URL[idx+1:] + chURL = manga.URL[:idx] + } + + var dateMs int64 + if hash != "" { + // Parse RFC3339 / ISO8601 date + if t, err := time.Parse(time.RFC3339Nano, hash); err == nil { + dateMs = t.UnixMilli() + } else if t, err := time.Parse("2006-01-02", hash[:10]); err == nil { + dateMs = t.UnixMilli() + } + } + + return []source.SChapter{{ + URL: chURL, + Name: "Gallery", + DateUpload: dateMs, + }}, nil +} + +func (s *Source) GetPageList(chapter source.SChapter) ([]source.Page, error) { + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, + util.AbsURL(s.cfg.BaseURL, chapter.URL), nil) + if err != nil { + return nil, err + } + req.Header.Set("Referer", s.cfg.BaseURL+"/") + resp, err := s.client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("gravureblogger: HTTP %d", resp.StatusCode) + } + doc, err := goquery.NewDocumentFromReader(resp.Body) + if err != nil { + return nil, err + } + var pages []source.Page + doc.Find("div.post-body a:has(> img)").Each(func(i int, a *goquery.Selection) { + if u, ok := a.Attr("href"); ok && u != "" { + pages = append(pages, source.Page{Index: i, ImageURL: util.AbsURL(s.cfg.BaseURL, u)}) + } + }) + return pages, nil +} + +func (s *Source) GetImageURL(page source.Page) (string, error) { return page.ImageURL, nil } +func (s *Source) GetFilterList() []source.Filter { return nil } diff --git a/sources/base/greenshit/greenshit.go b/sources/base/greenshit/greenshit.go new file mode 100644 index 0000000..054ee1b --- /dev/null +++ b/sources/base/greenshit/greenshit.go @@ -0,0 +1,408 @@ +// Package greenshit implements the GreenShit manga base. +// JSON REST API with Bearer token auth: POST {api}/auth/login → token → scan-id header. +package greenshit + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "regexp" + "strings" + "sync" + "time" + + "github.com/PuerkitoBio/goquery" + + "goyomi/internal/httpclient" + "goyomi/internal/source" +) + +type Config struct { + Name string + BaseURL string + APIURL string + CDNApiURL string // CDN base for thumbnails (with width) + CDNUrl string // CDN base for page images + Lang string + ScanID string + DefaultGenreID string // default: "1" + LimitPerPage string // default: "26" + Email string + Password string +} + +type Source struct { + cfg Config + client *httpclient.Client + id int64 + + mu sync.Mutex + cachedToken string + tokenExpiry int64 +} + +func New(cfg Config) *Source { + if cfg.DefaultGenreID == "" { + cfg.DefaultGenreID = "1" + } + if cfg.LimitPerPage == "" { + cfg.LimitPerPage = "26" + } + c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) + return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} +} + +func (s *Source) ID() int64 { return s.id } +func (s *Source) Name() string { return s.cfg.Name } +func (s *Source) Lang() string { return s.cfg.Lang } +func (s *Source) SupportsLatest() bool { return true } + +func (s *Source) api() string { return strings.TrimRight(s.cfg.APIURL, "/") } + +type loginRequestDTO struct { + Login string `json:"login"` + Senha string `json:"senha"` + TipoUsuario string `json:"tipo_usuario"` +} +type loginResponseDTO struct { + AccessToken string `json:"access_token"` + Token string `json:"token"` + ExpiresIn int64 `json:"expires_in"` +} + +func (s *Source) getToken(ctx context.Context) string { + s.mu.Lock() + defer s.mu.Unlock() + now := time.Now().UnixMilli() + if s.cachedToken != "" && now < s.tokenExpiry { + return s.cachedToken + } + if s.cfg.Email == "" || s.cfg.Password == "" { + return "" + } + body, _ := json.Marshal(loginRequestDTO{ + Login: s.cfg.Email, + Senha: s.cfg.Password, + TipoUsuario: "email", + }) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, s.api()+"/auth/login", bytes.NewReader(body)) + if err != nil { + return "" + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json") + req.Header.Set("scan-id", s.cfg.ScanID) + resp, err := s.client.Do(req) + if err != nil { + return "" + } + defer resp.Body.Close() + raw, _ := io.ReadAll(resp.Body) + var lr loginResponseDTO + if json.Unmarshal(raw, &lr) != nil { + return "" + } + tok := lr.AccessToken + if tok == "" { + tok = lr.Token + } + s.cachedToken = tok + exp := lr.ExpiresIn + if exp == 0 { + exp = 3600 + } + s.tokenExpiry = time.Now().UnixMilli() + exp*1000 + return tok +} + +func (s *Source) doGet(ctx context.Context, url string, out any) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return err + } + req.Header.Set("Accept", "application/json") + req.Header.Set("scan-id", s.cfg.ScanID) + if tok := s.getToken(ctx); tok != "" { + req.Header.Set("Authorization", "Bearer "+tok) + } + resp, err := s.client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("greenshit: HTTP %d for %s", resp.StatusCode, url) + } + body, _ := io.ReadAll(resp.Body) + return json.Unmarshal(body, out) +} + +// JSON DTOs + +type listDTO struct { + Obras []mangaDTO `json:"obras"` + CurrentPage int `json:"pagina_atual"` + TotalPages int `json:"paginas"` +} + +func (l listDTO) hasNext() bool { return l.TotalPages > l.CurrentPage } + +type mangaDTO struct { + ID int `json:"obr_id"` + Name string `json:"obr_nome"` + Description *string `json:"obr_descricao"` + Image *string `json:"obr_imagem"` + Tags []tagDTO `json:"tags"` + Status *statusDTO `json:"status"` + ScanID int `json:"scan_id"` + Chapters []chapterSimpleDTO `json:"capitulos"` +} + +type tagDTO struct { + Name string `json:"tag_nome"` +} + +type statusDTO struct { + Name string `json:"stt_nome"` +} + +type chapterSimpleDTO struct { + ID int `json:"cap_id"` + Name string `json:"cap_nome"` + Number float32 `json:"cap_numero"` + CreatedAt string `json:"cap_criado_em"` +} + +type chapterDetailDTO struct { + ID int `json:"cap_id"` + Name string `json:"cap_nome"` + Number float32 `json:"cap_numero"` + Pages []pageSrcDTO `json:"cap_paginas"` + Manga *mangaDTO `json:"obra"` +} + +type pageSrcDTO struct { + Src string `json:"src"` + Mime *string `json:"mime"` +} + +var ( + normalizeSlashRe = regexp.MustCompile(`(?:[^:])/{2,}`) +) + +func normalizeSlashes(u string) string { + // Replace consecutive slashes (not after colon) with single slash. + return normalizeSlashRe.ReplaceAllStringFunc(u, func(m string) string { + return string(m[0]) + "/" + }) +} + +func buildImageURL(path, src string, base string) string { + if src == "" { + return "" + } + if strings.HasPrefix(src, "http") { + return src + } + isWpLike := strings.HasPrefix(src, "uploads/") || strings.HasPrefix(src, "wp-content/") || + strings.HasPrefix(src, "manga_") || strings.HasPrefix(src, "WP-manga") + if isWpLike { + switch { + case strings.HasPrefix(src, "manga_"): + return normalizeSlashes(base + "/wp-content/uploads/WP-manga/data/" + src) + case strings.HasPrefix(src, "WP-manga"): + return normalizeSlashes(base + "/wp-content/uploads/" + src) + case strings.HasPrefix(src, "uploads/"): + return normalizeSlashes(base + "/wp-content/" + src) + case strings.HasPrefix(src, "wp-content/"): + return normalizeSlashes(base + "/" + src) + default: + return normalizeSlashes(base + "/wp-content/uploads/WP-manga/data/" + strings.TrimLeft(src, "/")) + } + } + safePath := strings.Trim(strings.ReplaceAll(path, "//", "/"), "/") + safeSrc := strings.Trim(strings.ReplaceAll(src, "//", "/"), "/") + return normalizeSlashes(base + "/" + safePath + "/" + safeSrc) +} + +func (s *Source) toSManga(m mangaDTO) source.SManga { + var tags []string + for _, t := range m.Tags { + if t.Name != "" { + tags = append(tags, t.Name) + } + } + thumbnail := "" + if m.Image != nil { + thumbnail = buildImageURL( + fmt.Sprintf("/scans/%d/obras/%d/", m.ScanID, m.ID), + *m.Image, + s.cfg.CDNApiURL, + ) + } + sm := source.SManga{ + URL: fmt.Sprintf("/obra/%d", m.ID), + Title: m.Name, + ThumbnailURL: thumbnail, + Genre: strings.Join(tags, ", "), + } + if m.Description != nil { + desc := *m.Description + if doc, err := goquery.NewDocumentFromReader(strings.NewReader(desc)); err == nil { + sm.Description = doc.Text() + } else { + sm.Description = desc + } + } + if m.Status != nil { + switch strings.ToLower(m.Status.Name) { + case "em andamento", "ativo": + sm.Status = source.StatusOngoing + case "concluído", "concluido": + sm.Status = source.StatusCompleted + case "hiato": + sm.Status = source.StatusHiatus + case "cancelado": + sm.Status = source.StatusCancelled + } + } + return sm +} + +func (s *Source) extractID(mangaURL string) string { + // URL format: /obra/{id} + parts := strings.Split(strings.Trim(mangaURL, "/"), "/") + if len(parts) >= 2 { + return parts[len(parts)-1] + } + return mangaURL +} + +func (s *Source) GetPopularManga(page int) (source.MangasPage, error) { + u := fmt.Sprintf("%s/obras/ranking?tipo=visualizacoes_geral&limite=%s&pagina=%d&gen_id=%s", + s.api(), s.cfg.LimitPerPage, page, s.cfg.DefaultGenreID) + var dto listDTO + if err := s.doGet(context.Background(), u, &dto); err != nil { + return source.MangasPage{}, err + } + mangas := make([]source.SManga, len(dto.Obras)) + for i, m := range dto.Obras { + mangas[i] = s.toSManga(m) + } + return source.MangasPage{Mangas: mangas, HasNextPage: dto.hasNext()}, nil +} + +func (s *Source) GetLatestUpdates(page int) (source.MangasPage, error) { + u := fmt.Sprintf("%s/obras/atualizacoes?pagina=%d&limite=%s&gen_id=%s", + s.api(), page, s.cfg.LimitPerPage, s.cfg.DefaultGenreID) + var dto listDTO + if err := s.doGet(context.Background(), u, &dto); err != nil { + return source.MangasPage{}, err + } + mangas := make([]source.SManga, len(dto.Obras)) + for i, m := range dto.Obras { + mangas[i] = s.toSManga(m) + } + return source.MangasPage{Mangas: mangas, HasNextPage: dto.hasNext()}, nil +} + +func (s *Source) GetSearchManga(page int, query string, filters []source.Filter) (source.MangasPage, error) { + u := fmt.Sprintf("%s/obras/search?limite=%s&pagina=%d&obr_nome=%s", + s.api(), s.cfg.LimitPerPage, page, query) + var dto listDTO + if err := s.doGet(context.Background(), u, &dto); err != nil { + return source.MangasPage{}, err + } + mangas := make([]source.SManga, len(dto.Obras)) + for i, m := range dto.Obras { + mangas[i] = s.toSManga(m) + } + return source.MangasPage{Mangas: mangas, HasNextPage: dto.hasNext()}, nil +} + +func (s *Source) GetMangaDetails(manga source.SManga) (source.SManga, error) { + id := s.extractID(manga.URL) + var dto mangaDTO + if err := s.doGet(context.Background(), fmt.Sprintf("%s/obras/%s", s.api(), id), &dto); err != nil { + return manga, err + } + result := s.toSManga(dto) + result.URL = manga.URL + return result, nil +} + +func (s *Source) GetChapterList(manga source.SManga) ([]source.SChapter, error) { + id := s.extractID(manga.URL) + var dto mangaDTO + if err := s.doGet(context.Background(), fmt.Sprintf("%s/obras/%s", s.api(), id), &dto); err != nil { + return nil, err + } + seen := map[string]bool{} + var chapters []source.SChapter + for _, ch := range dto.Chapters { + u := fmt.Sprintf("/capitulo/%d", ch.ID) + if seen[u] { + continue + } + seen[u] = true + chapters = append(chapters, source.SChapter{ + URL: u, + Name: ch.Name, + DateUpload: parseDate(ch.CreatedAt), + }) + } + // sort descending by chapter number (already from API) + return chapters, nil +} + +func (s *Source) GetPageList(chapter source.SChapter) ([]source.Page, error) { + // URL format: /capitulo/{id} + chID := strings.TrimPrefix(chapter.URL, "/capitulo/") + var dto chapterDetailDTO + if err := s.doGet(context.Background(), fmt.Sprintf("%s/capitulos/%s", s.api(), chID), &dto); err != nil { + return nil, err + } + obraID := 0 + scanID := 0 + chapterNum := "0" + if dto.Manga != nil { + obraID = dto.Manga.ID + scanID = dto.Manga.ScanID + if dto.Number != 0 { + numStr := fmt.Sprintf("%g", dto.Number) + chapterNum = numStr + } + } + pages := make([]source.Page, len(dto.Pages)) + for i, p := range dto.Pages { + path := fmt.Sprintf("/scans/%d/obras/%d/capitulos/%s/", scanID, obraID, chapterNum) + pages[i] = source.Page{ + Index: i, + ImageURL: buildImageURL(path, p.Src, s.cfg.CDNUrl), + } + } + return pages, nil +} + +func parseDate(s string) int64 { + if s == "" { + return 0 + } + formats := []string{ + "2006-01-02T15:04:05.000Z", + "2006-01-02T15:04:05Z", + "2006-01-02", + } + for _, f := range formats { + if t, err := time.Parse(f, s); err == nil { + return t.UnixMilli() + } + } + return 0 +} + +func (s *Source) GetImageURL(page source.Page) (string, error) { return page.ImageURL, nil } +func (s *Source) GetFilterList() []source.Filter { return nil } diff --git a/sources/base/hotcomics/hotcomics.go b/sources/base/hotcomics/hotcomics.go new file mode 100644 index 0000000..9bae447 --- /dev/null +++ b/sources/base/hotcomics/hotcomics.go @@ -0,0 +1,228 @@ +// Package hotcomics implements the HotComics manga base. +// HTML scraping; popular: GET {base}/en; requires cookie hc_vfs=Y. +package hotcomics + +import ( + "context" + "fmt" + "net/http" + "strings" + "time" + + "github.com/PuerkitoBio/goquery" + + "goyomi/internal/httpclient" + "goyomi/internal/source" + "goyomi/sources/base/util" +) + +type Config struct { + Name string + BaseURL string + Lang string +} + +type Source struct { + cfg Config + client *httpclient.Client + id int64 +} + +func New(cfg Config) *Source { + c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) + return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} +} + +func (s *Source) ID() int64 { return s.id } +func (s *Source) Name() string { return s.cfg.Name } +func (s *Source) Lang() string { return s.cfg.Lang } +func (s *Source) SupportsLatest() bool { return true } + +func (s *Source) base() string { return strings.TrimRight(s.cfg.BaseURL, "/") } + +func (s *Source) get(ctx context.Context, rawURL string) (*goquery.Document, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) + if err != nil { + return nil, err + } + req.Header.Set("Referer", s.cfg.BaseURL+"/") + // Inject required cookie + req.AddCookie(&http.Cookie{Name: "hc_vfs", Value: "Y"}) + resp, err := s.client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("hotcomics: HTTP %d", resp.StatusCode) + } + return goquery.NewDocumentFromReader(resp.Body) +} + +func imgAttr(img *goquery.Selection) string { + if v, ok := img.Attr("data-src"); ok && v != "" { + return v + } + v, _ := img.Attr("src") + return v +} + +func (s *Source) parseMangaList(doc *goquery.Document) source.MangasPage { + var mangas []source.SManga + seen := map[string]bool{} + doc.Find("li[itemtype*=ComicSeries]:not(.no-comic) > a").Each(func(_ int, a *goquery.Selection) { + m := source.SManga{} + m.URL, _ = a.Attr("href") + if m.URL == "" || seen[m.URL] { + return + } + seen[m.URL] = true + a.Find("div.visual img").First().Each(func(_ int, img *goquery.Selection) { + m.ThumbnailURL = imgAttr(img) + }) + m.Title = strings.TrimSpace(a.Find("div.main-text > h4.title").Text()) + if m.Title != "" { + mangas = append(mangas, m) + } + }) + hasNext := doc.Find("div.pagination a.vnext:not(.disabled)").Length() > 0 + return source.MangasPage{Mangas: mangas, HasNextPage: hasNext} +} + +func (s *Source) GetPopularManga(page int) (source.MangasPage, error) { + doc, err := s.get(context.Background(), s.base()+"/en") + if err != nil { + return source.MangasPage{}, err + } + return s.parseMangaList(doc), nil +} + +func (s *Source) GetLatestUpdates(page int) (source.MangasPage, error) { + doc, err := s.get(context.Background(), s.base()+"/en/new") + if err != nil { + return source.MangasPage{}, err + } + return s.parseMangaList(doc), nil +} + +func (s *Source) GetSearchManga(page int, query string, filters []source.Filter) (source.MangasPage, error) { + var u string + if query != "" { + u = fmt.Sprintf("%s/en/search?keyword=%s", s.base(), query) + } else { + u = fmt.Sprintf("%s/en?page=%d", s.base(), page) + } + doc, err := s.get(context.Background(), u) + if err != nil { + return source.MangasPage{}, err + } + return s.parseMangaList(doc), nil +} + +func (s *Source) GetMangaDetails(manga source.SManga) (source.SManga, error) { + doc, err := s.get(context.Background(), util.AbsURL(s.cfg.BaseURL, manga.URL)) + if err != nil { + return manga, err + } + result := source.SManga{URL: manga.URL} + result.Title = strings.TrimSpace(doc.Find("h2.episode-title").Text()) + if result.Title == "" { + result.Title = manga.Title + } + typeBox := doc.Find("p.type_box") + result.Author = strings.TrimSpace( + strings.TrimPrefix(typeBox.Find("span.writer").Text(), "ⓒ")) + var genres []string + typeBox.Find("span.type").First().Each(func(_ int, el *goquery.Selection) { + for _, g := range strings.Split(el.Text(), "/") { + if g = strings.TrimSpace(g); g != "" { + genres = append(genres, g) + } + } + }) + result.Genre = strings.Join(genres, ", ") + switch typeBox.Find("span.date").Text() { + case "End", "Ende": + result.Status = source.StatusCompleted + case "": + result.Status = source.StatusUnknown + default: + result.Status = source.StatusOngoing + } + var descParts []string + doc.Find("div.episode-contents header").First().Each(func(_ int, el *goquery.Selection) { + if t := strings.TrimSpace(el.Text()); t != "" { + descParts = append(descParts, t) + } + }) + doc.Find("div.title_content > h2:not(.episode-title)").First().Each(func(_ int, el *goquery.Selection) { + if t := strings.TrimSpace(el.Text()); t != "" { + descParts = append(descParts, t) + } + }) + result.Description = strings.Join(descParts, "\n\n") + return result, nil +} + +func (s *Source) GetChapterList(manga source.SManga) ([]source.SChapter, error) { + doc, err := s.get(context.Background(), util.AbsURL(s.cfg.BaseURL, manga.URL)) + if err != nil { + return nil, err + } + var chapters []source.SChapter + doc.Find("#tab-chapter a").Each(func(_ int, a *goquery.Selection) { + onclick := a.AttrOr("onclick", "") + // onclick="popupLogin('/chapter/url')" + u := strings.TrimSuffix(strings.TrimPrefix( + strings.TrimPrefix(onclick, "popupLogin('"), "popupLogin(\""), "'") + u = strings.TrimSuffix(u, "\")") + u = strings.TrimSuffix(u, "')") + if u == "" || u == onclick { + return + } + name := strings.TrimSpace(a.Find(".cell-num").Text()) + dateStr := strings.TrimSpace(a.Find(".cell-time").Text()) + chapters = append(chapters, source.SChapter{ + URL: u, + Name: name, + DateUpload: parseDate(dateStr), + }) + }) + // reverse: oldest first + for i, j := 0, len(chapters)-1; i < j; i, j = i+1, j-1 { + chapters[i], chapters[j] = chapters[j], chapters[i] + } + return chapters, nil +} + +func parseDate(s string) int64 { + if s == "" { + return 0 + } + t, err := time.Parse("Jan 02, 2006", s) + if err != nil { + return 0 + } + return t.UnixMilli() +} + +func (s *Source) GetPageList(chapter source.SChapter) ([]source.Page, error) { + doc, err := s.get(context.Background(), util.AbsURL(s.cfg.BaseURL, chapter.URL)) + if err != nil { + return nil, err + } + var pages []source.Page + doc.Find("#viewer-img img").Each(func(i int, img *goquery.Selection) { + u := imgAttr(img) + if u != "" { + if !strings.HasPrefix(u, "http") { + u = util.AbsURL(s.cfg.BaseURL, u) + } + pages = append(pages, source.Page{Index: i, ImageURL: u}) + } + }) + return pages, nil +} + +func (s *Source) GetImageURL(page source.Page) (string, error) { return page.ImageURL, nil } +func (s *Source) GetFilterList() []source.Filter { return nil } diff --git a/sources/base/initmanga/initmanga.go b/sources/base/initmanga/initmanga.go new file mode 100644 index 0000000..cac68e2 --- /dev/null +++ b/sources/base/initmanga/initmanga.go @@ -0,0 +1,522 @@ +// Package initmanga implements the InitManga manga base. +// HTML scraping (UIkit-based); pages use AES-256-CBC with PBKDF2-SHA512 key derivation. +package initmanga + +import ( + "context" + "crypto/aes" + "crypto/cipher" + "crypto/sha512" + "encoding/base64" + "encoding/hex" + "encoding/json" + "fmt" + "net/http" + "regexp" + "strings" + + "github.com/PuerkitoBio/goquery" + "golang.org/x/crypto/pbkdf2" + + "goyomi/internal/httpclient" + "goyomi/internal/source" + "goyomi/sources/base/util" +) + +type Config struct { + Name string + BaseURL string + Lang string + PopularUrlSlug string // e.g. "manga" or "seri" + LatestUrlSlug string // e.g. "manga" +} + +type Source struct { + cfg Config + client *httpclient.Client + id int64 +} + +func New(cfg Config) *Source { + if cfg.PopularUrlSlug == "" { + cfg.PopularUrlSlug = "seri" + } + if cfg.LatestUrlSlug == "" { + cfg.LatestUrlSlug = cfg.PopularUrlSlug + } + c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) + return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} +} + +func (s *Source) ID() int64 { return s.id } +func (s *Source) Name() string { return s.cfg.Name } +func (s *Source) Lang() string { return s.cfg.Lang } +func (s *Source) SupportsLatest() bool { return true } + +func (s *Source) base() string { return strings.TrimRight(s.cfg.BaseURL, "/") } + +func (s *Source) get(ctx context.Context, rawURL string) (*goquery.Document, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) + if err != nil { + return nil, err + } + req.Header.Set("Referer", s.cfg.BaseURL+"/") + resp, err := s.client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("initmanga: HTTP %d", resp.StatusCode) + } + return goquery.NewDocumentFromReader(resp.Body) +} + +const popularMangaSelector = "div.manga-item-grid > div.uk-panel.uk-position-relative, " + + "div.manga-item-grid > div.uk-panel:not(.manga-item-ranking):not(.user-item-info), " + + "div.uk-panel.uk-position-relative, " + + "div.uk-panel:not(.manga-item-ranking):not(.user-item-info)" + +const nextPageSelector = "head link[rel=next], link[rel=next], " + + "ul.uk-pagination li:not(.uk-disabled) a[aria-label=\"Sonraki sayfa\"], " + + "a:contains(Sonraki sayfa), a:contains(Next page), a.next" + +func mangaFromElement(el *goquery.Selection, baseURL string) source.SManga { + m := source.SManga{} + link := el.Find("h3 a, div.uk-overflow-hidden a").First() + if link.Length() == 0 { + link = el.Find("a").First() + } + m.URL, _ = link.Attr("href") + m.Title = strings.TrimSpace(el.Find("h3").Text()) + if m.Title == "" { + clone := el.Find("a").Clone() + clone.Find("span, small").Remove() + m.Title = strings.TrimSpace(clone.Text()) + } + imgEl := el.Find("img") + if src := imgEl.AttrOr("data-src", ""); src != "" { + m.ThumbnailURL = util.AbsURL(baseURL, src) + } else if src := imgEl.AttrOr("src", ""); src != "" { + m.ThumbnailURL = util.AbsURL(baseURL, src) + } + return m +} + +func (s *Source) parsePage(doc *goquery.Document) source.MangasPage { + var mangas []source.SManga + doc.Find(popularMangaSelector).Each(func(_ int, el *goquery.Selection) { + m := mangaFromElement(el, s.cfg.BaseURL) + if m.URL != "" && m.Title != "" { + mangas = append(mangas, m) + } + }) + hasNext := doc.Find(nextPageSelector).Length() > 0 + return source.MangasPage{Mangas: mangas, HasNextPage: hasNext} +} + +func (s *Source) fetchListing(slug string, page int) (source.MangasPage, error) { + var u string + if page == 1 { + u = fmt.Sprintf("%s/%s/", s.base(), slug) + } else { + u = fmt.Sprintf("%s/%s/page/%d/", s.base(), slug, page) + } + doc, err := s.get(context.Background(), u) + if err != nil { + return source.MangasPage{}, err + } + return s.parsePage(doc), nil +} + +func (s *Source) GetPopularManga(page int) (source.MangasPage, error) { + return s.fetchListing(s.cfg.PopularUrlSlug, page) +} + +func (s *Source) GetLatestUpdates(page int) (source.MangasPage, error) { + return s.fetchListing(s.cfg.LatestUrlSlug, page) +} + +type searchDTO struct { + Title *string `json:"title"` + URL *string `json:"url"` + Thumb *string `json:"thumb"` +} + +func (s *Source) GetSearchManga(page int, query string, filters []source.Filter) (source.MangasPage, error) { + u := fmt.Sprintf("%s/wp-json/initlise/v1/search?term=%s&page=%d", s.base(), query, page) + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, u, nil) + if err != nil { + return source.MangasPage{}, err + } + req.Header.Set("Referer", s.cfg.BaseURL+"/") + resp, err := s.client.Do(req) + if err != nil { + return source.MangasPage{}, err + } + defer resp.Body.Close() + + body := make([]byte, 0, 4096) + buf := make([]byte, 4096) + for { + n, readErr := resp.Body.Read(buf) + body = append(body, buf[:n]...) + if readErr != nil { + break + } + } + + bodyStr := strings.TrimSpace(string(body)) + if strings.HasPrefix(bodyStr, "<") { + // HTML response — parse as list page + doc, err := goquery.NewDocumentFromReader(strings.NewReader(bodyStr)) + if err != nil { + return source.MangasPage{}, err + } + return s.parsePage(doc), nil + } + + var dtos []searchDTO + if err := json.Unmarshal(body, &dtos); err != nil { + return source.MangasPage{}, err + } + var mangas []source.SManga + for _, dto := range dtos { + m := source.SManga{} + if dto.Title != nil { + m.Title = strings.TrimSpace(*dto.Title) + } + if dto.URL != nil { + m.URL = *dto.URL + } + if dto.Thumb != nil { + m.ThumbnailURL = *dto.Thumb + } + if m.URL != "" && m.Title != "" { + mangas = append(mangas, m) + } + } + return source.MangasPage{Mangas: mangas, HasNextPage: false}, nil +} + +func (s *Source) GetMangaDetails(manga source.SManga) (source.SManga, error) { + doc, err := s.get(context.Background(), util.AbsURL(s.cfg.BaseURL, manga.URL)) + if err != nil { + return manga, err + } + result := source.SManga{URL: manga.URL} + + descClone := doc.Find("div#manga-description").Clone() + descClone.Find("a, span").Remove() + result.Description = strings.TrimSpace(descClone.Text()) + if altName := strings.TrimSpace(doc.Find("span#comic-othername").Text()); altName != "" { + result.Description += "\n\nAlternatif Başlık: " + altName + } + + var genres []string + doc.Find("span.uk-label-contest").Each(func(_ int, el *goquery.Selection) { + t := strings.TrimPrefix(strings.TrimSpace(el.Text()), "#") + if t != "" { + genres = append(genres, t) + } + }) + if len(genres) == 0 { + doc.Find("div#genre-tags a").Each(func(_ int, el *goquery.Selection) { + if t := strings.TrimSpace(el.Text()); t != "" { + genres = append(genres, t) + } + }) + } + result.Genre = strings.Join(genres, ", ") + + result.Author = strings.TrimSpace(doc.Find("div.manga-info-details a").FilterFunction(func(_ int, s *goquery.Selection) bool { + parent, _ := s.Parent().Html() + return strings.Contains(parent, "Yazar") + }).Text()) + if result.Author == "" { + t := doc.Find("div.manga-info-details").FilterFunction(func(_ int, el *goquery.Selection) bool { + return strings.Contains(el.Text(), "Yazar") + }).Text() + result.Author = strings.TrimSpace(strings.SplitN(strings.SplitN(t, "Yazar:", 2)[len(strings.SplitN(t, "Yazar:", 2))-1], "Çizer:", 2)[0]) + } + + statusText := strings.ToLower(strings.TrimSpace( + doc.Find("span#manga-status, div.manga-status-ribbons span.manga-status-ribbon__text").First().Text())) + if statusText == "" { + t := doc.Find("div.manga-info-details").FilterFunction(func(_ int, el *goquery.Selection) bool { + return strings.Contains(el.Text(), "Durum") + }).Text() + statusText = strings.ToLower(strings.TrimSpace(strings.SplitN(t, "Durum:", 2)[len(strings.SplitN(t, "Durum:", 2))-1])) + } + switch { + case strings.Contains(statusText, "güncel") || strings.Contains(statusText, "devam") || strings.Contains(statusText, "ongoing"): + result.Status = source.StatusOngoing + case strings.Contains(statusText, "tamamland") || strings.Contains(statusText, "bitti") || strings.Contains(statusText, "completed"): + result.Status = source.StatusCompleted + case strings.Contains(statusText, "ara ver") || strings.Contains(statusText, "sezon") || strings.Contains(statusText, "hiatus"): + result.Status = source.StatusHiatus + case strings.Contains(statusText, "bırakıldı") || strings.Contains(statusText, "iptal") || strings.Contains(statusText, "dropped"): + result.Status = source.StatusCancelled + default: + result.Status = source.StatusUnknown + } + + for _, sel := range []string{"div.story-cover-wrap img", "div.single-thumb img", "a.story-cover img"} { + if img := doc.Find(sel).First(); img.Length() > 0 { + if src := img.AttrOr("abs:src", img.AttrOr("src", "")); src != "" { + result.ThumbnailURL = util.AbsURL(s.cfg.BaseURL, src) + break + } + } + } + + title := strings.TrimSpace(doc.Find("h1").First().Text()) + if title == "" { + title = strings.TrimSpace(doc.Find("h2.uk-h3").First().Text()) + } + result.Title = title + if result.Title == "" { + result.Title = manga.Title + } + return result, nil +} + +func (s *Source) GetChapterList(manga source.SManga) ([]source.SChapter, error) { + mangaURL := util.AbsURL(s.cfg.BaseURL, manga.URL) + doc, err := s.get(context.Background(), mangaURL) + if err != nil { + return nil, err + } + var chapters []source.SChapter + chapterFromEl := func(el *goquery.Selection) source.SChapter { + ch := source.SChapter{} + el.Find("a").First().Each(func(_ int, a *goquery.Selection) { + ch.URL, _ = a.Attr("href") + }) + rawName := strings.TrimSpace(el.Find("h3").Text()) + ch.Name = strings.TrimSpace(lastPart(rawName, "–", "-")) + if ch.Name == "" { + ch.Name = rawName + } + dateStr := el.Find("time").AttrOr("datetime", "") + ch.DateUpload = util.ParseAbsoluteDate(dateStr, "2006-01-02T15:04:05") + return ch + } + + doc.Find("div.chapter-item").Each(func(_ int, el *goquery.Selection) { + if ch := chapterFromEl(el); ch.URL != "" { + chapters = append(chapters, ch) + } + }) + + // paginate + page := 2 + for { + paginURL := strings.TrimRight(mangaURL, "/") + "/bolum/page/" + fmt.Sprintf("%d", page) + "/" + nextDoc, err := s.get(context.Background(), paginURL) + if err != nil { + break + } + items := nextDoc.Find("div.chapter-item") + if items.Length() == 0 { + break + } + items.Each(func(_ int, el *goquery.Selection) { + if ch := chapterFromEl(el); ch.URL != "" { + chapters = append(chapters, ch) + } + }) + if nextDoc.Find("ul.uk-pagination a[href^=http]").Length() == 0 { + break + } + page++ + } + return chapters, nil +} + +func lastPart(s string, seps ...string) string { + for _, sep := range seps { + if idx := strings.LastIndex(s, sep); idx >= 0 { + return s[idx+len(sep):] + } + } + return s +} + +// Regexes for page decryption +var ( + reEncryptedData = regexp.MustCompile(`(?s)var\s+InitMangaEncryptedChapter\s*=\s*(\{.*?\});`) + reInitMangaChunk = regexp.MustCompile(`(?s)InitMangaEncryptedChapter\s*=\s*(\{.*?\})`) + reDecryptKey = regexp.MustCompile(`["']?decryption_key["']?\s*[:=]\s*["']([^"']+)["']`) + reSmartKey = regexp.MustCompile(`(?s)InitMangaData[\s\S]*?decryption_key["']?\s*[:=]\s*["']([^"']+)["']`) +) + +type encryptedChapter struct { + Ciphertext string `json:"ciphertext"` + IV string `json:"iv"` + Salt string `json:"salt"` +} + +func decryptWithPassphrase(ciphertextB64, passphrase, saltHex, ivHex string) (string, error) { + salt, err := hex.DecodeString(saltHex) + if err != nil { + return "", err + } + iv, err := hex.DecodeString(ivHex) + if err != nil { + return "", err + } + ct, err := base64.StdEncoding.DecodeString(ciphertextB64) + if err != nil { + return "", err + } + key := pbkdf2.Key([]byte(passphrase), salt, 999, 32, sha512.New) + block, err := aes.NewCipher(key) + if err != nil { + return "", err + } + if len(ct)%aes.BlockSize != 0 { + return "", fmt.Errorf("initmanga: ciphertext not block-aligned") + } + mode := cipher.NewCBCDecrypter(block, iv) + mode.CryptBlocks(ct, ct) + // PKCS5 unpad + if len(ct) == 0 { + return "", fmt.Errorf("initmanga: empty decrypted block") + } + padLen := int(ct[len(ct)-1]) + if padLen == 0 || padLen > aes.BlockSize || padLen > len(ct) { + return "", fmt.Errorf("initmanga: invalid padding %d", padLen) + } + return string(ct[:len(ct)-padLen]), nil +} + +func (s *Source) GetPageList(chapter source.SChapter) ([]source.Page, error) { + doc, err := s.get(context.Background(), util.AbsURL(s.cfg.BaseURL, chapter.URL)) + if err != nil { + return nil, err + } + html, _ := doc.Html() + + // --- Path 1: script[src*=dmFyIElua] base64-encoded data --- + var encObj encryptedChapter + doc.Find("script[src]").Each(func(_ int, el *goquery.Selection) { + if encObj.Ciphertext != "" { + return + } + src := el.AttrOr("src", "") + if !strings.Contains(src, "dmFyIElua") { + return + } + b64 := strings.TrimSuffix( + strings.SplitN(src, "base64,", 2)[len(strings.SplitN(src, "base64,", 2))-1], "\"") + raw, err := base64.StdEncoding.DecodeString(b64) + if err != nil { + return + } + decoded := string(raw) + // extract InitMangaEncryptedChapter JSON + jsonStr := "" + if m := reInitMangaChunk.FindStringSubmatch(decoded); len(m) > 1 { + jsonStr = m[1] + } else if idx := strings.Index(decoded, "InitMangaEncryptedChapter="); idx >= 0 { + jsonStr = strings.SplitN(decoded[idx+len("InitMangaEncryptedChapter="):], ";", 2)[0] + } + if jsonStr == "" { + return + } + json.Unmarshal([]byte(jsonStr), &encObj) //nolint:errcheck + }) + + // --- Path 2: inline JS var InitMangaEncryptedChapter --- + if encObj.Ciphertext == "" { + if m := reEncryptedData.FindStringSubmatch(html); len(m) > 1 { + json.Unmarshal([]byte(m[1]), &encObj) //nolint:errcheck + } + } + + if encObj.Ciphertext != "" { + // Find decryption key + rawKey := "" + // Try script#init-main-js-extra src (base64-encoded) + doc.Find("script#init-main-js-extra[src]").First().Each(func(_ int, el *goquery.Selection) { + src := el.AttrOr("src", "") + if !strings.Contains(src, "base64,") { + return + } + b64 := strings.SplitN(src, "base64,", 2)[1] + b64 = strings.TrimSuffix(b64, "\"") + if raw, err := base64.StdEncoding.DecodeString(b64); err == nil { + if m := reDecryptKey.FindStringSubmatch(string(raw)); len(m) > 1 { + rawKey = m[1] + } + } + }) + if rawKey == "" { + if m := reSmartKey.FindStringSubmatch(html); len(m) > 1 { + rawKey = m[1] + } + } + if rawKey != "" { + passBytes, err := base64.StdEncoding.DecodeString(rawKey) + if err == nil { + passphrase := string(passBytes) + if content, err := decryptWithPassphrase(encObj.Ciphertext, passphrase, encObj.Salt, encObj.IV); err == nil { + return parseDecryptedPages(content, s.cfg.BaseURL), nil + } + } + } + } + + // Fallback: plain img tags + return fallbackPages(doc, s.cfg.BaseURL), nil +} + +func parseDecryptedPages(content, baseURL string) []source.Page { + trimmed := strings.TrimSpace(content) + if strings.HasPrefix(trimmed, "<") { + doc, err := goquery.NewDocumentFromReader(strings.NewReader(trimmed)) + if err != nil { + return nil + } + var pages []source.Page + doc.Find("img").Each(func(i int, img *goquery.Selection) { + src := img.AttrOr("data-src", img.AttrOr("src", "")) + if src == "" || strings.HasPrefix(src, "data:") { + return + } + pages = append(pages, source.Page{Index: i, ImageURL: util.AbsURL(baseURL, src)}) + }) + return pages + } + // Try JSON array of URLs + var srcs []string + if err := json.Unmarshal([]byte(trimmed), &srcs); err == nil { + pages := make([]source.Page, 0, len(srcs)) + for i, src := range srcs { + switch { + case strings.HasPrefix(src, "//"): + src = "https:" + src + case strings.HasPrefix(src, "/"): + src = strings.TrimRight(baseURL, "/") + src + } + pages = append(pages, source.Page{Index: i, ImageURL: src}) + } + return pages + } + return nil +} + +func fallbackPages(doc *goquery.Document, baseURL string) []source.Page { + var pages []source.Page + doc.Find("div#chapter-content img[src]").Each(func(i int, img *goquery.Selection) { + src := img.AttrOr("src", img.AttrOr("data-src", "")) + if src != "" { + pages = append(pages, source.Page{Index: i, ImageURL: util.AbsURL(baseURL, src)}) + } + }) + return pages +} + +func (s *Source) GetImageURL(page source.Page) (string, error) { return page.ImageURL, nil } +func (s *Source) GetFilterList() []source.Filter { return nil } diff --git a/sources/base/keyoapp/keyoapp.go b/sources/base/keyoapp/keyoapp.go new file mode 100644 index 0000000..8c31525 --- /dev/null +++ b/sources/base/keyoapp/keyoapp.go @@ -0,0 +1,325 @@ +// Package keyoapp implements the Keyoapp manga base. +// HTML scraping; popular from homepage; pages via CDN URL extracted from inline JS. +package keyoapp + +import ( + "context" + "fmt" + "net/http" + "net/url" + "regexp" + "strings" + + "github.com/PuerkitoBio/goquery" + + "goyomi/internal/httpclient" + "goyomi/internal/source" + "goyomi/sources/base/util" +) + +type Config struct { + Name string + BaseURL string + Lang string +} + +type Source struct { + cfg Config + client *httpclient.Client + id int64 +} + +func New(cfg Config) *Source { + c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) + return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} +} + +func (s *Source) ID() int64 { return s.id } +func (s *Source) Name() string { return s.cfg.Name } +func (s *Source) Lang() string { return s.cfg.Lang } +func (s *Source) SupportsLatest() bool { return true } + +func (s *Source) base() string { return strings.TrimRight(s.cfg.BaseURL, "/") } + +func (s *Source) get(ctx context.Context, rawURL string) (*goquery.Document, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) + if err != nil { + return nil, err + } + req.Header.Set("Referer", s.cfg.BaseURL+"/") + resp, err := s.client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("keyoapp: HTTP %d", resp.StatusCode) + } + return goquery.NewDocumentFromReader(resp.Body) +} + +var imgURLRe = regexp.MustCompile(`url\(['"]?([^('")\s]+)`) + +func getImageURL(el *goquery.Selection, baseURL string) string { + // Find any descendant with background-image style + var u string + el.Find("*[style]").Each(func(_ int, s *goquery.Selection) { + if u != "" { + return + } + style := s.AttrOr("style", "") + if !strings.Contains(style, "background-image") { + return + } + if m := imgURLRe.FindStringSubmatch(style); len(m) > 1 { + raw := m[1] + // strip w= query that keyoapp uses for thumbnail sizing + if parsed, err := url.Parse(raw); err == nil { + q := parsed.Query() + q.Del("w") + parsed.RawQuery = q.Encode() + u = parsed.String() + } else { + u = raw + } + } + }) + if u != "" { + return util.AbsURL(baseURL, u) + } + return "" +} + +func relURL(raw, baseURL string) string { + u, err := url.Parse(raw) + if err != nil { + return raw + } + base, err := url.Parse(baseURL) + if err != nil { + return raw + } + if u.Host == base.Host { + return u.Path + } + return raw +} + +func (s *Source) mangaFromElement(el *goquery.Selection) source.SManga { + m := source.SManga{} + m.ThumbnailURL = getImageURL(el, s.cfg.BaseURL) + el.Find("a[href]").First().Each(func(_ int, a *goquery.Selection) { + m.Title = a.AttrOr("title", "") + href := a.AttrOr("href", "") + m.URL = relURL(href, s.cfg.BaseURL) + }) + return m +} + +var popularSelectors = []string{"Popular", "Popularie", "Trending"} + +func popularSelector() string { + var parts []string + for _, s := range popularSelectors { + parts = append(parts, fmt.Sprintf("div:contains(%s) + div .group.overflow-hidden.grid", s)) + } + return strings.Join(parts, ", ") +} + +func (s *Source) GetPopularManga(page int) (source.MangasPage, error) { + doc, err := s.get(context.Background(), s.base()) + if err != nil { + return source.MangasPage{}, err + } + var mangas []source.SManga + doc.Find(popularSelector()).Each(func(_ int, el *goquery.Selection) { + m := s.mangaFromElement(el) + if m.URL != "" && m.Title != "" { + mangas = append(mangas, m) + } + }) + return source.MangasPage{Mangas: mangas, HasNextPage: false}, nil +} + +func (s *Source) GetLatestUpdates(page int) (source.MangasPage, error) { + doc, err := s.get(context.Background(), s.base()+"/latest/") + if err != nil { + return source.MangasPage{}, err + } + var mangas []source.SManga + doc.Find("div.grid > div.group").Each(func(_ int, el *goquery.Selection) { + m := s.mangaFromElement(el) + if m.URL != "" && m.Title != "" { + mangas = append(mangas, m) + } + }) + return source.MangasPage{Mangas: mangas, HasNextPage: false}, nil +} + +func (s *Source) GetSearchManga(page int, query string, filters []source.Filter) (source.MangasPage, error) { + u := fmt.Sprintf("%s/series/?q=%s", s.base(), query) + doc, err := s.get(context.Background(), u) + if err != nil { + return source.MangasPage{}, err + } + var mangas []source.SManga + // Filter client-side by title + doc.Find("#searched_series_page > button").Each(func(_ int, el *goquery.Selection) { + title := el.AttrOr("title", "") + if query != "" && !strings.Contains(strings.ToLower(title), strings.ToLower(query)) { + return + } + m := s.mangaFromElement(el) + if m.URL == "" { + el.Find("a[href]").First().Each(func(_ int, a *goquery.Selection) { + m.URL = relURL(a.AttrOr("href", ""), s.cfg.BaseURL) + m.Title = a.AttrOr("title", strings.TrimSpace(a.Text())) + }) + } + if m.Title == "" { + m.Title = title + } + if m.URL != "" { + mangas = append(mangas, m) + } + }) + return source.MangasPage{Mangas: mangas, HasNextPage: false}, nil +} + +func parseStatus(s *goquery.Selection) int { + if s == nil || s.Length() == 0 { + return source.StatusUnknown + } + switch strings.ToLower(strings.TrimSpace(s.Text())) { + case "ongoing": + return source.StatusOngoing + case "dropped": + return source.StatusCancelled + case "paused": + return source.StatusHiatus + case "completed": + return source.StatusCompleted + default: + return source.StatusUnknown + } +} + +func (s *Source) GetMangaDetails(manga source.SManga) (source.SManga, error) { + doc, err := s.get(context.Background(), util.AbsURL(s.cfg.BaseURL, manga.URL)) + if err != nil { + return manga, err + } + result := source.SManga{URL: manga.URL} + + // Thumbnail from div[class*=photoURL] background-image style + result.ThumbnailURL = getImageURL(doc.Find("div[class*=photoURL]").First(), s.cfg.BaseURL) + result.Description = strings.TrimSpace(doc.Find("div:containsOwn(Synopsis) ~ div").First().Text()) + result.Status = parseStatus(doc.Find("div:has(span:containsOwn(Status)) ~ div").First()) + result.Author = strings.TrimSpace(doc.Find("div:has(span:containsOwn(Author)) ~ div").First().Text()) + result.Artist = strings.TrimSpace(doc.Find("div:has(span:containsOwn(Artist)) ~ div").First().Text()) + + // Title from h1 inside the series header + result.Title = strings.TrimSpace(doc.Find("h1").First().Text()) + if result.Title == "" { + result.Title = manga.Title + } + + // Genres from grid links + var genres []string + doc.Find("div.grid:has(>h1) > div > a:not([title='Status'])").Each(func(_ int, a *goquery.Selection) { + if t := strings.TrimSpace(a.Text()); t != "" { + genres = append(genres, t) + } + }) + result.Genre = strings.Join(genres, ", ") + return result, nil +} + +func (s *Source) GetChapterList(manga source.SManga) ([]source.SChapter, error) { + doc, err := s.get(context.Background(), util.AbsURL(s.cfg.BaseURL, manga.URL)) + if err != nil { + return nil, err + } + // Exclude upcoming and (optionally) paid chapters + sel := "#chapters > a:not(:has(.text-sm span))" + var chapters []source.SChapter + doc.Find(sel).Each(func(_ int, el *goquery.Selection) { + ch := source.SChapter{} + el.Find("a[href]").First().Each(func(_ int, a *goquery.Selection) { + href := a.AttrOr("href", "") + ch.URL = relURL(href, s.cfg.BaseURL) + }) + if ch.URL == "" { + href := el.AttrOr("href", "") + ch.URL = relURL(href, s.cfg.BaseURL) + } + ch.Name = strings.TrimSpace(el.Find(".text-sm").Text()) + if dateEl := el.Find(".text-xs").First(); dateEl.Length() > 0 { + ch.DateUpload = util.ParseRelativeDate(strings.TrimSpace(dateEl.Text())) + if ch.DateUpload == 0 { + ch.DateUpload = util.ParseAbsoluteDate(strings.TrimSpace(dateEl.Text()), "Jan 2, 2006") + } + } + if ch.URL != "" { + chapters = append(chapters, ch) + } + }) + return chapters, nil +} + +var ( + cdnHostRe = regexp.MustCompile("realUrl\\s*=\\s*`[^`]+//([^/`]+)") + cdnCleanRe = regexp.MustCompile(`\$\{[^}]*\}`) +) + +func getCdnURL(doc *goquery.Document) string { + var cdnURL string + doc.Find("script").Each(func(_ int, el *goquery.Selection) { + if cdnURL != "" { + return + } + html, _ := el.Html() + if m := cdnHostRe.FindStringSubmatch(html); len(m) > 1 { + host := cdnCleanRe.ReplaceAllString(m[1], "") + cdnURL = "https://" + host + "/uploads" + } + }) + return cdnURL +} + +var oldCdnRe = regexp.MustCompile(`^(https?:)?//cdn\d*\.keyoapp\.com`) + +func (s *Source) GetPageList(chapter source.SChapter) ([]source.Page, error) { + doc, err := s.get(context.Background(), util.AbsURL(s.cfg.BaseURL, chapter.URL)) + if err != nil { + return nil, err + } + cdnURL := getCdnURL(doc) + // Primary: #pages > img[uid] with CDN URL + var pages []source.Page + if cdnURL != "" { + doc.Find("#pages > img[uid]").Each(func(i int, img *goquery.Selection) { + uid := img.AttrOr("uid", "") + if uid != "" { + pages = append(pages, source.Page{Index: i, ImageURL: cdnURL + "/" + uid}) + } + }) + } + if len(pages) > 0 { + return pages, nil + } + // Fallback: old CDN direct src + doc.Find("#pages > img").Each(func(i int, img *goquery.Selection) { + src := img.AttrOr("src", "") + if src == "" { + src = img.AttrOr("data-src", "") + } + if oldCdnRe.MatchString(src) { + pages = append(pages, source.Page{Index: i, ImageURL: src}) + } + }) + return pages, nil +} + +func (s *Source) GetImageURL(page source.Page) (string, error) { return page.ImageURL, nil } +func (s *Source) GetFilterList() []source.Filter { return nil }