Feature/improve browsing source performance (#743)
* Improve browse source database operations * Reuse "insertOrGet" for "processEntries"
This commit is contained in:
schroda
@@ -8,18 +8,14 @@ package suwayomi.tachidesk.manga.impl
|
||||
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
|
||||
|
||||
import eu.kanade.tachiyomi.source.model.MangasPage
|
||||
import eu.kanade.tachiyomi.source.model.UpdateStrategy
|
||||
import org.jetbrains.exposed.sql.and
|
||||
import org.jetbrains.exposed.sql.insertAndGetId
|
||||
import org.jetbrains.exposed.sql.batchInsert
|
||||
import org.jetbrains.exposed.sql.select
|
||||
import org.jetbrains.exposed.sql.transactions.transaction
|
||||
import suwayomi.tachidesk.manga.impl.Manga.getMangaMetaMap
|
||||
import suwayomi.tachidesk.manga.impl.util.source.GetCatalogueSource.getCatalogueSourceOrStub
|
||||
import suwayomi.tachidesk.manga.model.dataclass.MangaDataClass
|
||||
import suwayomi.tachidesk.manga.model.dataclass.PagedMangaListDataClass
|
||||
import suwayomi.tachidesk.manga.model.dataclass.toGenreList
|
||||
import suwayomi.tachidesk.manga.model.table.MangaStatus
|
||||
import suwayomi.tachidesk.manga.model.table.MangaTable
|
||||
import suwayomi.tachidesk.manga.model.table.toDataClass
|
||||
|
||||
object MangaList {
|
||||
fun proxyThumbnailUrl(mangaId: Int): String {
|
||||
@@ -50,35 +46,38 @@ object MangaList {
|
||||
|
||||
fun MangasPage.insertOrGet(sourceId: Long): List<Int> {
|
||||
return transaction {
|
||||
val existingMangaUrlsToId =
|
||||
MangaTable.slice(MangaTable.url, MangaTable.id).select {
|
||||
(MangaTable.sourceReference eq sourceId) and (MangaTable.url inList mangas.map { it.url })
|
||||
}.associate { Pair(it[MangaTable.url], it[MangaTable.id].value) }
|
||||
val existingMangaUrls = existingMangaUrlsToId.map { it.key }
|
||||
|
||||
val mangasToInsert = mangas.filter { !existingMangaUrls.contains(it.url) }
|
||||
|
||||
val insertedMangaUrlsToId =
|
||||
MangaTable.batchInsert(mangasToInsert) {
|
||||
this[MangaTable.url] = it.url
|
||||
this[MangaTable.title] = it.title
|
||||
|
||||
this[MangaTable.artist] = it.artist
|
||||
this[MangaTable.author] = it.author
|
||||
this[MangaTable.description] = it.description
|
||||
this[MangaTable.genre] = it.genre
|
||||
this[MangaTable.status] = it.status
|
||||
this[MangaTable.thumbnail_url] = it.thumbnail_url
|
||||
this[MangaTable.updateStrategy] = it.update_strategy.name
|
||||
|
||||
this[MangaTable.sourceReference] = sourceId
|
||||
}.associate { Pair(it[MangaTable.url], it[MangaTable.id].value) }
|
||||
|
||||
// delete thumbnail in case cached data still exists
|
||||
insertedMangaUrlsToId.forEach { (_, id) -> Manga.clearThumbnail(id) }
|
||||
|
||||
val mangaUrlsToId = existingMangaUrlsToId + insertedMangaUrlsToId
|
||||
|
||||
mangas.map { manga ->
|
||||
val mangaEntry =
|
||||
MangaTable.select {
|
||||
(MangaTable.url eq manga.url) and (MangaTable.sourceReference eq sourceId)
|
||||
}.firstOrNull()
|
||||
if (mangaEntry == null) { // create manga entry
|
||||
val mangaId =
|
||||
MangaTable.insertAndGetId {
|
||||
it[url] = manga.url
|
||||
it[title] = manga.title
|
||||
|
||||
it[artist] = manga.artist
|
||||
it[author] = manga.author
|
||||
it[description] = manga.description
|
||||
it[genre] = manga.genre
|
||||
it[status] = manga.status
|
||||
it[thumbnail_url] = manga.thumbnail_url
|
||||
it[updateStrategy] = manga.update_strategy.name
|
||||
|
||||
it[sourceReference] = sourceId
|
||||
}.value
|
||||
|
||||
// delete thumbnail in case cached data still exists
|
||||
Manga.clearThumbnail(mangaId)
|
||||
|
||||
mangaId
|
||||
} else {
|
||||
mangaEntry[MangaTable.id].value
|
||||
}
|
||||
mangaUrlsToId[manga.url]
|
||||
?: throw Exception("MangaList::insertOrGet($sourceId): Something went wrong inserting browsed source mangas")
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -87,84 +86,8 @@ object MangaList {
|
||||
val mangasPage = this
|
||||
val mangaList =
|
||||
transaction {
|
||||
return@transaction mangasPage.mangas.map { manga ->
|
||||
var mangaEntry =
|
||||
MangaTable.select {
|
||||
(MangaTable.url eq manga.url) and (MangaTable.sourceReference eq sourceId)
|
||||
}.firstOrNull()
|
||||
if (mangaEntry == null) { // create manga entry
|
||||
val mangaId =
|
||||
MangaTable.insertAndGetId {
|
||||
it[url] = manga.url
|
||||
it[title] = manga.title
|
||||
|
||||
it[artist] = manga.artist
|
||||
it[author] = manga.author
|
||||
it[description] = manga.description
|
||||
it[genre] = manga.genre
|
||||
it[status] = manga.status
|
||||
it[thumbnail_url] = manga.thumbnail_url
|
||||
it[updateStrategy] = manga.update_strategy.name
|
||||
|
||||
it[sourceReference] = sourceId
|
||||
}.value
|
||||
|
||||
// delete thumbnail in case cached data still exists
|
||||
Manga.clearThumbnail(mangaId)
|
||||
|
||||
mangaEntry =
|
||||
MangaTable.select {
|
||||
(MangaTable.url eq manga.url) and (MangaTable.sourceReference eq sourceId)
|
||||
}.first()
|
||||
|
||||
MangaDataClass(
|
||||
id = mangaId,
|
||||
sourceId = sourceId.toString(),
|
||||
url = manga.url,
|
||||
title = manga.title,
|
||||
thumbnailUrl = proxyThumbnailUrl(mangaId),
|
||||
thumbnailUrlLastFetched = mangaEntry[MangaTable.thumbnailUrlLastFetched],
|
||||
initialized = manga.initialized,
|
||||
artist = manga.artist,
|
||||
author = manga.author,
|
||||
description = manga.description,
|
||||
genre = manga.genre.toGenreList(),
|
||||
status = MangaStatus.valueOf(manga.status).name,
|
||||
inLibrary = false, // It's a new manga entry
|
||||
inLibraryAt = 0,
|
||||
meta = getMangaMetaMap(mangaId),
|
||||
realUrl = mangaEntry[MangaTable.realUrl],
|
||||
lastFetchedAt = mangaEntry[MangaTable.lastFetchedAt],
|
||||
chaptersLastFetchedAt = mangaEntry[MangaTable.chaptersLastFetchedAt],
|
||||
updateStrategy = UpdateStrategy.valueOf(mangaEntry[MangaTable.updateStrategy]),
|
||||
freshData = true,
|
||||
)
|
||||
} else {
|
||||
val mangaId = mangaEntry[MangaTable.id].value
|
||||
MangaDataClass(
|
||||
id = mangaId,
|
||||
sourceId = sourceId.toString(),
|
||||
url = manga.url,
|
||||
title = manga.title,
|
||||
thumbnailUrl = proxyThumbnailUrl(mangaId),
|
||||
thumbnailUrlLastFetched = mangaEntry[MangaTable.thumbnailUrlLastFetched],
|
||||
initialized = true,
|
||||
artist = mangaEntry[MangaTable.artist],
|
||||
author = mangaEntry[MangaTable.author],
|
||||
description = mangaEntry[MangaTable.description],
|
||||
genre = mangaEntry[MangaTable.genre].toGenreList(),
|
||||
status = MangaStatus.valueOf(mangaEntry[MangaTable.status]).name,
|
||||
inLibrary = mangaEntry[MangaTable.inLibrary],
|
||||
inLibraryAt = mangaEntry[MangaTable.inLibraryAt],
|
||||
meta = getMangaMetaMap(mangaId),
|
||||
realUrl = mangaEntry[MangaTable.realUrl],
|
||||
lastFetchedAt = mangaEntry[MangaTable.lastFetchedAt],
|
||||
chaptersLastFetchedAt = mangaEntry[MangaTable.chaptersLastFetchedAt],
|
||||
updateStrategy = UpdateStrategy.valueOf(mangaEntry[MangaTable.updateStrategy]),
|
||||
freshData = false,
|
||||
)
|
||||
}
|
||||
}
|
||||
val mangaIds = insertOrGet(sourceId)
|
||||
return@transaction MangaTable.select { MangaTable.id inList mangaIds }.map { MangaTable.toDataClass(it) }
|
||||
}
|
||||
return PagedMangaListDataClass(
|
||||
mangaList,
|
||||
|
||||
Reference in New Issue
Block a user