Optimise MAL search queries by ~11x (#2832)
Previously, the app made one request for the search, and then fired off 1 request per search result to obtain additional data, such as each title's synopsis, etc. However, MAL's search allows field selection during the initial query, which will return all the data in that first response, avoiding the massive bunch of requests (and alleviating some pressure on MAL from our userbase). By combining the selected fields into one constant, I was able to also get rid of the MALUserListSearch entirely because it was redundant. This allows for a unified MALManga->TrackSearch helper, further reducing complexity. I got to my "11x" improvement because on page of search results has 10 elements, and this change turns 11 (1+10 for results) requests into 1. (cherry picked from commit 9bf2d78a421213b1885456f5b54c3286edc539e1) # Conflicts: # CHANGELOG.md # app/src/main/java/eu/kanade/tachiyomi/data/track/myanimelist/MyAnimeListApi.kt
This commit is contained in:
MajorTanya
@@ -12,15 +12,12 @@ import eu.kanade.tachiyomi.data.track.myanimelist.dto.MALMangaMetadata
|
||||
import eu.kanade.tachiyomi.data.track.myanimelist.dto.MALOAuth
|
||||
import eu.kanade.tachiyomi.data.track.myanimelist.dto.MALSearchResult
|
||||
import eu.kanade.tachiyomi.data.track.myanimelist.dto.MALUser
|
||||
import eu.kanade.tachiyomi.data.track.myanimelist.dto.MALUserSearchResult
|
||||
import eu.kanade.tachiyomi.network.DELETE
|
||||
import eu.kanade.tachiyomi.network.GET
|
||||
import eu.kanade.tachiyomi.network.POST
|
||||
import eu.kanade.tachiyomi.network.awaitSuccess
|
||||
import eu.kanade.tachiyomi.network.parseAs
|
||||
import eu.kanade.tachiyomi.util.PkceUtil
|
||||
import kotlinx.coroutines.async
|
||||
import kotlinx.coroutines.awaitAll
|
||||
import kotlinx.serialization.json.Json
|
||||
import okhttp3.FormBody
|
||||
import okhttp3.Headers
|
||||
@@ -80,15 +77,15 @@ class MyAnimeListApi(
|
||||
// MAL API throws a 400 when the query is over 64 characters...
|
||||
.appendQueryParameter("q", query.take(64))
|
||||
.appendQueryParameter("nsfw", "true")
|
||||
.appendQueryParameter("fields", SEARCH_FIELDS)
|
||||
.build()
|
||||
with(json) {
|
||||
authClient.newCall(GET(url.toString()))
|
||||
.awaitSuccess()
|
||||
.parseAs<MALSearchResult>()
|
||||
.data
|
||||
.map { async { getMangaDetails(it.node.id) } }
|
||||
.awaitAll()
|
||||
.filter { !it.publishing_type.contains("novel") }
|
||||
.filter { !(it.node.mediaType.contains("novel")) }
|
||||
.map { parseSearchItem(it.node) }
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -97,29 +94,13 @@ class MyAnimeListApi(
|
||||
return withIOContext {
|
||||
val url = "$BASE_API_URL/manga".toUri().buildUpon()
|
||||
.appendPath(id.toString())
|
||||
.appendQueryParameter(
|
||||
"fields",
|
||||
"id,title,synopsis,num_chapters,mean,main_picture,status,media_type,start_date",
|
||||
)
|
||||
.appendQueryParameter("fields", SEARCH_FIELDS)
|
||||
.build()
|
||||
with(json) {
|
||||
authClient.newCall(GET(url.toString()))
|
||||
.awaitSuccess()
|
||||
.parseAs<MALManga>()
|
||||
.let {
|
||||
TrackSearch.create(trackId).apply {
|
||||
remote_id = it.id
|
||||
title = it.title
|
||||
summary = it.synopsis
|
||||
total_chapters = it.numChapters
|
||||
score = it.mean
|
||||
cover_url = it.covers?.large.orEmpty()
|
||||
tracking_url = "https://myanimelist.net/manga/$remote_id"
|
||||
publishing_status = it.status.replace("_", " ")
|
||||
publishing_type = it.mediaType.replace("_", " ")
|
||||
start_date = it.startDate ?: ""
|
||||
}
|
||||
}
|
||||
.let { parseSearchItem(it) }
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -183,8 +164,7 @@ class MyAnimeListApi(
|
||||
|
||||
val matches = myListSearchResult.data
|
||||
.filter { it.node.title.contains(query, ignoreCase = true) }
|
||||
.map { async { getMangaDetails(it.node.id) } }
|
||||
.awaitAll()
|
||||
.map { parseSearchItem(it.node) }
|
||||
|
||||
// Check next page if there's more
|
||||
if (!myListSearchResult.paging.next.isNullOrBlank()) {
|
||||
@@ -230,10 +210,10 @@ class MyAnimeListApi(
|
||||
}
|
||||
}
|
||||
|
||||
private suspend fun getListPage(offset: Int): MALUserSearchResult {
|
||||
private suspend fun getListPage(offset: Int): MALSearchResult {
|
||||
return withIOContext {
|
||||
val urlBuilder = "$BASE_API_URL/users/@me/mangalist".toUri().buildUpon()
|
||||
.appendQueryParameter("fields", "list_status{start_date,finish_date}")
|
||||
.appendQueryParameter("fields", SEARCH_FIELDS)
|
||||
.appendQueryParameter("limit", LIST_PAGINATION_AMOUNT.toString())
|
||||
if (offset > 0) {
|
||||
urlBuilder.appendQueryParameter("offset", offset.toString())
|
||||
@@ -262,6 +242,21 @@ class MyAnimeListApi(
|
||||
}
|
||||
}
|
||||
|
||||
private fun parseSearchItem(searchItem: MALManga): TrackSearch {
|
||||
return TrackSearch.create(trackId).apply {
|
||||
remote_id = searchItem.id
|
||||
title = searchItem.title
|
||||
summary = searchItem.synopsis
|
||||
total_chapters = searchItem.numChapters
|
||||
score = searchItem.mean
|
||||
cover_url = searchItem.covers?.large.orEmpty()
|
||||
tracking_url = "https://myanimelist.net/manga/$remote_id"
|
||||
publishing_status = searchItem.status.replace("_", " ")
|
||||
publishing_type = searchItem.mediaType.replace("_", " ")
|
||||
start_date = searchItem.startDate ?: ""
|
||||
}
|
||||
}
|
||||
|
||||
private fun parseDate(isoDate: String): Long {
|
||||
return SimpleDateFormat("yyyy-MM-dd", Locale.US).parse(isoDate)?.time ?: 0L
|
||||
}
|
||||
@@ -273,7 +268,7 @@ class MyAnimeListApi(
|
||||
return try {
|
||||
val outputDf = SimpleDateFormat("yyyy-MM-dd", Locale.US)
|
||||
outputDf.format(epochTime)
|
||||
} catch (e: Exception) {
|
||||
} catch (_: Exception) {
|
||||
null
|
||||
}
|
||||
}
|
||||
@@ -284,6 +279,9 @@ class MyAnimeListApi(
|
||||
private const val BASE_OAUTH_URL = "https://myanimelist.net/v1/oauth2"
|
||||
private const val BASE_API_URL = "https://api.myanimelist.net/v2"
|
||||
|
||||
private const val SEARCH_FIELDS =
|
||||
"id,title,synopsis,num_chapters,mean,main_picture,status,media_type,start_date"
|
||||
|
||||
private const val LIST_PAGINATION_AMOUNT = 250
|
||||
|
||||
private var codeVerifier: String = ""
|
||||
|
||||
@@ -5,14 +5,15 @@ import kotlinx.serialization.Serializable
|
||||
@Serializable
|
||||
data class MALSearchResult(
|
||||
val data: List<MALSearchResultNode>,
|
||||
val paging: MALSearchPaging,
|
||||
)
|
||||
|
||||
@Serializable
|
||||
data class MALSearchResultNode(
|
||||
val node: MALSearchResultItem,
|
||||
val node: MALManga,
|
||||
)
|
||||
|
||||
@Serializable
|
||||
data class MALSearchResultItem(
|
||||
val id: Int,
|
||||
data class MALSearchPaging(
|
||||
val next: String?,
|
||||
)
|
||||
|
||||
@@ -1,25 +0,0 @@
|
||||
package eu.kanade.tachiyomi.data.track.myanimelist.dto
|
||||
|
||||
import kotlinx.serialization.Serializable
|
||||
|
||||
@Serializable
|
||||
data class MALUserSearchResult(
|
||||
val data: List<MALUserSearchItem>,
|
||||
val paging: MALUserSearchPaging,
|
||||
)
|
||||
|
||||
@Serializable
|
||||
data class MALUserSearchItem(
|
||||
val node: MALUserSearchItemNode,
|
||||
)
|
||||
|
||||
@Serializable
|
||||
data class MALUserSearchPaging(
|
||||
val next: String?,
|
||||
)
|
||||
|
||||
@Serializable
|
||||
data class MALUserSearchItemNode(
|
||||
val id: Int,
|
||||
val title: String,
|
||||
)
|
||||
Reference in New Issue
Block a user