diff --git a/sources/base/colorlibanime/colorlibanime.go b/sources/base/colorlibanime/colorlibanime.go index 3210045..d4e24df 100755 --- a/sources/base/colorlibanime/colorlibanime.go +++ b/sources/base/colorlibanime/colorlibanime.go @@ -11,7 +11,7 @@ import ( "github.com/PuerkitoBio/goquery" - "goyomi/internal/httpclient" + "goyomi/internal/httpclient/flare" "goyomi/internal/source" "goyomi/sources/base/util" ) @@ -24,12 +24,12 @@ type Config struct { type Source struct { cfg Config - client *httpclient.Client + client *flare.Client id int64 } func New(cfg Config) *Source { - c := httpclient.NewClient(httpclient.WithRateLimit(3, 1)) + c := flare.NewClient(flare.WithRateLimit(3, 1)) return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} } @@ -41,12 +41,7 @@ func (s *Source) SupportsLatest() bool { return true } func (s *Source) base() string { return strings.TrimRight(s.cfg.BaseURL, "/") } func (s *Source) get(ctx context.Context, rawURL string) (*goquery.Document, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) - if err != nil { - return nil, err - } - req.Header.Set("Referer", s.cfg.BaseURL+"/") - resp, err := s.client.Do(req) + resp, err := s.client.Get(ctx, rawURL) if err != nil { return nil, err } diff --git a/sources/base/eromuse/eromuse.go b/sources/base/eromuse/eromuse.go index d233bd7..cae0e87 100755 --- a/sources/base/eromuse/eromuse.go +++ b/sources/base/eromuse/eromuse.go @@ -10,7 +10,7 @@ import ( "github.com/PuerkitoBio/goquery" - "goyomi/internal/httpclient" + "goyomi/internal/httpclient/flare" "goyomi/internal/source" "goyomi/sources/base/util" ) @@ -23,12 +23,12 @@ type Config struct { type Source struct { cfg Config - client *httpclient.Client + client *flare.Client id int64 } func New(cfg Config) *Source { - c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) + c := flare.NewClient(flare.WithRateLimit(1, 2)) return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} } @@ -40,12 +40,7 @@ func (s *Source) SupportsLatest() bool { return true } func (s *Source) base() string { return strings.TrimRight(s.cfg.BaseURL, "/") } func (s *Source) get(ctx context.Context, rawURL string) (*goquery.Document, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) - if err != nil { - return nil, err - } - req.Header.Set("Referer", s.cfg.BaseURL+"/") - resp, err := s.client.Do(req) + resp, err := s.client.Get(ctx, rawURL) if err != nil { return nil, err } @@ -56,6 +51,26 @@ func (s *Source) get(ctx context.Context, rawURL string) (*goquery.Document, err return goquery.NewDocumentFromReader(resp.Body) } +func (s *Source) getWithNextPage(ctx context.Context, rawURL string) (*goquery.Document, string, error) { + resp, err := s.client.Get(ctx, rawURL) + if err != nil { + return nil, "", err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, "", fmt.Errorf("eromuse: HTTP %d", resp.StatusCode) + } + doc, err := goquery.NewDocumentFromReader(resp.Body) + if err != nil { + return nil, "", err + } + var next string + doc.Find(".pagination span.current + span a").First().Each(func(_ int, a *goquery.Selection) { + next, _ = a.Attr("href") + }) + return doc, next, nil +} + func (s *Source) parseMangaList(doc *goquery.Document) ([]source.SManga, string) { var mangas []source.SManga doc.Find("a.c-tile:has(img)").Each(func(_ int, el *goquery.Selection) { diff --git a/sources/base/fansubscat/fansubscat.go b/sources/base/fansubscat/fansubscat.go index cc81f3e..92522ad 100755 --- a/sources/base/fansubscat/fansubscat.go +++ b/sources/base/fansubscat/fansubscat.go @@ -10,7 +10,7 @@ import ( "net/http" "strings" - "goyomi/internal/httpclient" + "goyomi/internal/httpclient/flare" "goyomi/internal/source" "goyomi/sources/base/util" ) @@ -25,7 +25,7 @@ type Config struct { type Source struct { cfg Config - client *httpclient.Client + client *flare.Client id int64 } @@ -33,7 +33,7 @@ func New(cfg Config) *Source { if cfg.APIURL == "" { cfg.APIURL = cfg.BaseURL } - c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) + c := flare.NewClient(flare.WithRateLimit(1, 2)) return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} } @@ -45,13 +45,7 @@ func (s *Source) SupportsLatest() bool { return true } func (s *Source) api() string { return strings.TrimRight(s.cfg.APIURL, "/") } func (s *Source) getJSON(ctx context.Context, rawURL string, out any) error { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) - if err != nil { - return err - } - req.Header.Set("Accept", "application/json") - req.Header.Set("Referer", s.cfg.BaseURL+"/") - resp, err := s.client.Do(req) + resp, err := s.client.Get(ctx, rawURL) if err != nil { return err } diff --git a/sources/base/fmreader/fmreader.go b/sources/base/fmreader/fmreader.go index cc62d9a..8ad6450 100755 --- a/sources/base/fmreader/fmreader.go +++ b/sources/base/fmreader/fmreader.go @@ -11,7 +11,7 @@ import ( "github.com/PuerkitoBio/goquery" - "goyomi/internal/httpclient" + "goyomi/internal/httpclient/flare" "goyomi/internal/source" "goyomi/sources/base/util" ) @@ -25,7 +25,7 @@ type Config struct { type Source struct { cfg Config - client *httpclient.Client + client *flare.Client id int64 } @@ -33,7 +33,7 @@ func New(cfg Config) *Source { if cfg.RequestPath == "" { cfg.RequestPath = "manga-list.html" } - c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) + c := flare.NewClient(flare.WithRateLimit(1, 2)) return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} } @@ -43,11 +43,7 @@ func (s *Source) Lang() string { return s.cfg.Lang } func (s *Source) SupportsLatest() bool { return true } func (s *Source) get(ctx context.Context, rawURL string) (*goquery.Document, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) - if err != nil { - return nil, err - } - resp, err := s.client.Do(req) + resp, err := s.client.Get(ctx, rawURL) if err != nil { return nil, err } diff --git a/sources/base/galleryadults/galleryadults.go b/sources/base/galleryadults/galleryadults.go index 4f4c3f1..4f1837d 100755 --- a/sources/base/galleryadults/galleryadults.go +++ b/sources/base/galleryadults/galleryadults.go @@ -11,7 +11,7 @@ import ( "github.com/PuerkitoBio/goquery" - "goyomi/internal/httpclient" + "goyomi/internal/httpclient/flare" "goyomi/internal/source" "goyomi/sources/base/util" ) @@ -25,12 +25,12 @@ type Config struct { type Source struct { cfg Config - client *httpclient.Client + client *flare.Client id int64 } func New(cfg Config) *Source { - c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) + c := flare.NewClient(flare.WithRateLimit(1, 2)) return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} } @@ -42,12 +42,7 @@ func (s *Source) SupportsLatest() bool { return true } func (s *Source) base() string { return strings.TrimRight(s.cfg.BaseURL, "/") } func (s *Source) get(ctx context.Context, rawURL string) (*goquery.Document, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) - if err != nil { - return nil, err - } - req.Header.Set("Referer", s.cfg.BaseURL+"/") - resp, err := s.client.Do(req) + resp, err := s.client.Get(ctx, rawURL) if err != nil { return nil, err } diff --git a/sources/base/greenshit/greenshit.go b/sources/base/greenshit/greenshit.go index 054ee1b..8b4849b 100755 --- a/sources/base/greenshit/greenshit.go +++ b/sources/base/greenshit/greenshit.go @@ -3,7 +3,6 @@ package greenshit import ( - "bytes" "context" "encoding/json" "fmt" @@ -16,7 +15,7 @@ import ( "github.com/PuerkitoBio/goquery" - "goyomi/internal/httpclient" + "goyomi/internal/httpclient/flare" "goyomi/internal/source" ) @@ -36,7 +35,7 @@ type Config struct { type Source struct { cfg Config - client *httpclient.Client + client *flare.Client id int64 mu sync.Mutex @@ -51,7 +50,7 @@ func New(cfg Config) *Source { if cfg.LimitPerPage == "" { cfg.LimitPerPage = "26" } - c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) + c := flare.NewClient(flare.WithRateLimit(1, 2)) return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} } @@ -88,14 +87,7 @@ func (s *Source) getToken(ctx context.Context) string { Senha: s.cfg.Password, TipoUsuario: "email", }) - req, err := http.NewRequestWithContext(ctx, http.MethodPost, s.api()+"/auth/login", bytes.NewReader(body)) - if err != nil { - return "" - } - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Accept", "application/json") - req.Header.Set("scan-id", s.cfg.ScanID) - resp, err := s.client.Do(req) + resp, err := s.client.Post(ctx, s.api()+"/auth/login", "application/json", strings.NewReader(string(body))) if err != nil { return "" } @@ -118,17 +110,8 @@ func (s *Source) getToken(ctx context.Context) string { return tok } -func (s *Source) doGet(ctx context.Context, url string, out any) error { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) - if err != nil { - return err - } - req.Header.Set("Accept", "application/json") - req.Header.Set("scan-id", s.cfg.ScanID) - if tok := s.getToken(ctx); tok != "" { - req.Header.Set("Authorization", "Bearer "+tok) - } - resp, err := s.client.Do(req) +func (s *Source) getJSON(ctx context.Context, url string, out any) error { + resp, err := s.client.Get(ctx, url) if err != nil { return err } @@ -285,7 +268,7 @@ func (s *Source) GetPopularManga(page int) (source.MangasPage, error) { u := fmt.Sprintf("%s/obras/ranking?tipo=visualizacoes_geral&limite=%s&pagina=%d&gen_id=%s", s.api(), s.cfg.LimitPerPage, page, s.cfg.DefaultGenreID) var dto listDTO - if err := s.doGet(context.Background(), u, &dto); err != nil { + if err := s.getJSON(context.Background(), u, &dto); err != nil { return source.MangasPage{}, err } mangas := make([]source.SManga, len(dto.Obras)) @@ -299,7 +282,7 @@ func (s *Source) GetLatestUpdates(page int) (source.MangasPage, error) { u := fmt.Sprintf("%s/obras/atualizacoes?pagina=%d&limite=%s&gen_id=%s", s.api(), page, s.cfg.LimitPerPage, s.cfg.DefaultGenreID) var dto listDTO - if err := s.doGet(context.Background(), u, &dto); err != nil { + if err := s.getJSON(context.Background(), u, &dto); err != nil { return source.MangasPage{}, err } mangas := make([]source.SManga, len(dto.Obras)) @@ -313,7 +296,7 @@ func (s *Source) GetSearchManga(page int, query string, filters []source.Filter) u := fmt.Sprintf("%s/obras/search?limite=%s&pagina=%d&obr_nome=%s", s.api(), s.cfg.LimitPerPage, page, query) var dto listDTO - if err := s.doGet(context.Background(), u, &dto); err != nil { + if err := s.getJSON(context.Background(), u, &dto); err != nil { return source.MangasPage{}, err } mangas := make([]source.SManga, len(dto.Obras)) @@ -326,7 +309,7 @@ func (s *Source) GetSearchManga(page int, query string, filters []source.Filter) func (s *Source) GetMangaDetails(manga source.SManga) (source.SManga, error) { id := s.extractID(manga.URL) var dto mangaDTO - if err := s.doGet(context.Background(), fmt.Sprintf("%s/obras/%s", s.api(), id), &dto); err != nil { + if err := s.getJSON(context.Background(), fmt.Sprintf("%s/obras/%s", s.api(), id), &dto); err != nil { return manga, err } result := s.toSManga(dto) @@ -337,7 +320,7 @@ func (s *Source) GetMangaDetails(manga source.SManga) (source.SManga, error) { func (s *Source) GetChapterList(manga source.SManga) ([]source.SChapter, error) { id := s.extractID(manga.URL) var dto mangaDTO - if err := s.doGet(context.Background(), fmt.Sprintf("%s/obras/%s", s.api(), id), &dto); err != nil { + if err := s.getJSON(context.Background(), fmt.Sprintf("%s/obras/%s", s.api(), id), &dto); err != nil { return nil, err } seen := map[string]bool{} @@ -362,7 +345,7 @@ func (s *Source) GetPageList(chapter source.SChapter) ([]source.Page, error) { // URL format: /capitulo/{id} chID := strings.TrimPrefix(chapter.URL, "/capitulo/") var dto chapterDetailDTO - if err := s.doGet(context.Background(), fmt.Sprintf("%s/capitulos/%s", s.api(), chID), &dto); err != nil { + if err := s.getJSON(context.Background(), fmt.Sprintf("%s/capitulos/%s", s.api(), chID), &dto); err != nil { return nil, err } obraID := 0 diff --git a/sources/base/grouple/grouple.go b/sources/base/grouple/grouple.go index 10e3ccd..6474807 100755 --- a/sources/base/grouple/grouple.go +++ b/sources/base/grouple/grouple.go @@ -10,7 +10,7 @@ import ( "github.com/PuerkitoBio/goquery" - "goyomi/internal/httpclient" + "goyomi/internal/httpclient/flare" "goyomi/internal/source" "goyomi/sources/base/util" ) @@ -23,12 +23,12 @@ type Config struct { type Source struct { cfg Config - client *httpclient.Client + client *flare.Client id int64 } func New(cfg Config) *Source { - c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) + c := flare.NewClient(flare.WithRateLimit(1, 2)) return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} } @@ -38,11 +38,7 @@ func (s *Source) Lang() string { return s.cfg.Lang } func (s *Source) SupportsLatest() bool { return true } func (s *Source) get(ctx context.Context, rawURL string) (*goquery.Document, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) - if err != nil { - return nil, err - } - resp, err := s.client.Do(req) + resp, err := s.client.Get(ctx, rawURL) if err != nil { return nil, err } diff --git a/sources/base/heancms/heancms.go b/sources/base/heancms/heancms.go index eec340c..5051c38 100755 --- a/sources/base/heancms/heancms.go +++ b/sources/base/heancms/heancms.go @@ -12,7 +12,7 @@ import ( "strings" "time" - "goyomi/internal/httpclient" + "goyomi/internal/httpclient/flare" "goyomi/internal/source" "goyomi/sources/base/util" ) @@ -30,7 +30,7 @@ type Config struct { type Source struct { cfg Config - client *httpclient.Client + client *flare.Client id int64 } @@ -50,7 +50,7 @@ func New(cfg Config) *Source { if cfg.MangaSubDirectory == "" { cfg.MangaSubDirectory = "series" } - c := httpclient.NewClient(httpclient.WithRateLimit(1, 2)) + c := flare.NewClient(flare.WithRateLimit(1, 2)) return &Source{ cfg: cfg, client: c, @@ -228,12 +228,7 @@ func (s *Source) toSChapter(dto chapterDTO, seriesSlug string) source.SChapter { // --- API calls --- func (s *Source) getJSON(ctx context.Context, rawURL string, out any) error { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) - if err != nil { - return err - } - req.Header.Set("Accept", "application/json") - resp, err := s.client.Do(req) + resp, err := s.client.Get(ctx, rawURL) if err != nil { return err } diff --git a/sources/base/lectormoe/lectormoe.go b/sources/base/lectormoe/lectormoe.go index b04baac..d0a6b66 100755 --- a/sources/base/lectormoe/lectormoe.go +++ b/sources/base/lectormoe/lectormoe.go @@ -10,7 +10,7 @@ import ( "net/http" "strings" - "goyomi/internal/httpclient" + "goyomi/internal/httpclient/flare" "goyomi/internal/source" "goyomi/sources/base/util" ) @@ -27,7 +27,7 @@ type Config struct { type Source struct { cfg Config - client *httpclient.Client + client *flare.Client id int64 } @@ -38,7 +38,7 @@ func New(cfg Config) *Source { if cfg.OrganizationDomain == "" { cfg.OrganizationDomain = util.SlugFromURL(cfg.BaseURL) } - c := httpclient.NewClient(httpclient.WithRateLimit(3, 1)) + c := flare.NewClient(flare.WithRateLimit(3, 1)) return &Source{cfg: cfg, client: c, id: source.GenerateSourceID(cfg.Name, cfg.Lang)} } @@ -50,14 +50,7 @@ func (s *Source) SupportsLatest() bool { return true } func (s *Source) api() string { return strings.TrimRight(s.cfg.APIBaseURL, "/") } func (s *Source) getJSON(ctx context.Context, rawURL string, out any) error { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) - if err != nil { - return err - } - req.Header.Set("Accept", "application/json") - req.Header.Set("Referer", s.cfg.BaseURL+"/") - req.Header.Set("x-organization", s.cfg.OrganizationDomain) - resp, err := s.client.Do(req) + resp, err := s.client.Get(ctx, rawURL) if err != nil { return err }