From 6d33d726630c0ef69a0b85006b9e9a394c119026 Mon Sep 17 00:00:00 2001 From: Alexandre Journet Date: Tue, 31 Oct 2023 00:47:03 +0100 Subject: [PATCH] #733: Improve perfs on getChapterList with onlineFetch (Less databases calls) (#737) * improve(#733): less databases calls on getChapterList with onlineFetch * improve(#733): fixes (delete with ids), tried batch update but not successfull * improve(#733): fixes (batch update) * improve(#733): clean imports * improve(#733): fixes SChapter to ChapterDataClass, * improve(#733): re-added recognize chap number --------- Co-authored-by: Alexandre JOURNET --- .../suwayomi/tachidesk/manga/impl/Chapter.kt | 121 +++++++++++------- .../manga/model/dataclass/ChapterDataClass.kt | 33 ++++- 2 files changed, 109 insertions(+), 45 deletions(-) diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Chapter.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Chapter.kt index 952f35d0..ffc5cbc4 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Chapter.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Chapter.kt @@ -18,11 +18,13 @@ import org.jetbrains.exposed.sql.Op import org.jetbrains.exposed.sql.ResultRow import org.jetbrains.exposed.sql.SortOrder import org.jetbrains.exposed.sql.SortOrder.ASC -import org.jetbrains.exposed.sql.SqlExpressionBuilder.eq +import org.jetbrains.exposed.sql.SqlExpressionBuilder.inList import org.jetbrains.exposed.sql.and +import org.jetbrains.exposed.sql.batchInsert import org.jetbrains.exposed.sql.deleteWhere import org.jetbrains.exposed.sql.insert import org.jetbrains.exposed.sql.select +import org.jetbrains.exposed.sql.statements.BatchUpdateStatement import org.jetbrains.exposed.sql.transactions.transaction import org.jetbrains.exposed.sql.update import suwayomi.tachidesk.manga.impl.Manga.getManga @@ -127,46 +129,66 @@ object Chapter { chapter.chapter_number = chapterNumber.toFloat() } - var now = Instant.now().epochSecond + val now = Instant.now().epochSecond + val chaptersInDb = + transaction { + ChapterTable.select { ChapterTable.manga eq mangaId } + .map { ChapterTable.toDataClass(it) } + .toSet() + } + + val chaptersToInsert = mutableListOf() + val chaptersToUpdate = mutableListOf() + + chapterList.reversed().forEachIndexed { index, fetchedChapter -> + val chapterEntry = chaptersInDb.find { it.url == fetchedChapter.url } + + val chapterData = + ChapterDataClass.fromSChapter( + fetchedChapter, + chapterEntry?.id ?: 0, + index + 1, + now, + mangaId, + (source as? HttpSource)?.getChapterUrl(fetchedChapter), + ) + + if (chapterEntry == null) { + chaptersToInsert.add(chapterData) + } else { + chaptersToUpdate.add(chapterData) + } + } transaction { - chapterList.reversed().forEachIndexed { index, fetchedChapter -> - val chapterEntry = ChapterTable.select { ChapterTable.url eq fetchedChapter.url }.firstOrNull() - if (chapterEntry == null) { - ChapterTable.insert { - it[url] = fetchedChapter.url - it[name] = fetchedChapter.name - it[date_upload] = fetchedChapter.date_upload - it[chapter_number] = fetchedChapter.chapter_number - it[scanlator] = fetchedChapter.scanlator - - it[sourceOrder] = index + 1 - it[fetchedAt] = now++ - it[ChapterTable.manga] = mangaId - - it[realUrl] = - runCatching { - (source as? HttpSource)?.getChapterUrl(fetchedChapter) - }.getOrNull() - } - } else { - ChapterTable.update({ ChapterTable.url eq fetchedChapter.url }) { - it[name] = fetchedChapter.name - it[date_upload] = fetchedChapter.date_upload - it[chapter_number] = fetchedChapter.chapter_number - it[scanlator] = fetchedChapter.scanlator - - it[sourceOrder] = index + 1 - it[ChapterTable.manga] = mangaId - - it[realUrl] = - runCatching { - (source as? HttpSource)?.getChapterUrl(fetchedChapter) - }.getOrNull() - } + if (chaptersToInsert.isNotEmpty()) { + ChapterTable.batchInsert(chaptersToInsert) { + this[ChapterTable.url] = it.url + this[ChapterTable.name] = it.name + this[ChapterTable.date_upload] = it.uploadDate + this[ChapterTable.chapter_number] = it.chapterNumber + this[ChapterTable.scanlator] = it.scanlator + this[ChapterTable.sourceOrder] = it.index + this[ChapterTable.fetchedAt] = it.fetchedAt + this[ChapterTable.manga] = it.mangaId + this[ChapterTable.realUrl] = it.realUrl } } + BatchUpdateStatement(ChapterTable).apply { + chaptersToUpdate.forEach { + addBatch(EntityID(it.id, ChapterTable)) + this[ChapterTable.name] = it.name + this[ChapterTable.date_upload] = it.uploadDate + this[ChapterTable.chapter_number] = it.chapterNumber + this[ChapterTable.scanlator] = it.scanlator + this[ChapterTable.sourceOrder] = it.index + this[ChapterTable.fetchedAt] = it.fetchedAt + this[ChapterTable.realUrl] = it.realUrl + } + execute(this@transaction) + } + MangaTable.update({ MangaTable.id eq mangaId }) { it[MangaTable.chaptersLastFetchedAt] = Instant.now().epochSecond } @@ -186,18 +208,25 @@ object Chapter { ChapterTable.select { ChapterTable.manga eq mangaId } .orderBy(ChapterTable.url to ASC).toList() } + val chapterUrls = chapterList.map { it.url }.toSet() - dbChapterList.forEachIndexed { index, dbChapter -> - if ( - !chapterUrls.contains(dbChapter[ChapterTable.url]) || // is orphaned - (index < dbChapterList.lastIndex && dbChapter[ChapterTable.url] == dbChapterList[index + 1][ChapterTable.url]) // is duplicate - ) { - transaction { - PageTable.deleteWhere { PageTable.chapter eq dbChapter[ChapterTable.id] } - ChapterTable.deleteWhere { ChapterTable.id eq dbChapter[ChapterTable.id] } + val chaptersIdsToDelete = + dbChapterList.mapIndexedNotNull { index, dbChapter -> + val isOrphaned = !chapterUrls.contains(dbChapter[ChapterTable.url]) + val isDuplicate = + index < dbChapterList.lastIndex && dbChapter[ChapterTable.url] == dbChapterList[index + 1][ChapterTable.url] + val deleteChapter = isOrphaned || isDuplicate + if (deleteChapter) { + dbChapter[ChapterTable.id].value + } else { + null } } + + transaction { + PageTable.deleteWhere { PageTable.chapter inList chaptersIdsToDelete } + ChapterTable.deleteWhere { ChapterTable.id inList chaptersIdsToDelete } } } @@ -340,15 +369,19 @@ object Chapter { when { input.chapterIds != null -> Op.build { (ChapterTable.manga eq mangaId) and (ChapterTable.id inList input.chapterIds) } + input.chapterIndexes != null -> Op.build { (ChapterTable.manga eq mangaId) and (ChapterTable.sourceOrder inList input.chapterIndexes) } + else -> null } + else -> { // mangaId is null, only chapterIndexes is valid for this case when { input.chapterIds != null -> Op.build { (ChapterTable.id inList input.chapterIds) } + else -> null } } diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/model/dataclass/ChapterDataClass.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/model/dataclass/ChapterDataClass.kt index 4d4e156a..1023a679 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/model/dataclass/ChapterDataClass.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/model/dataclass/ChapterDataClass.kt @@ -1,5 +1,7 @@ package suwayomi.tachidesk.manga.model.dataclass +import eu.kanade.tachiyomi.source.model.SChapter + /* * Copyright (C) Contributors to the Suwayomi project * @@ -38,4 +40,33 @@ data class ChapterDataClass( val chapterCount: Int? = null, /** used to store client specific values */ val meta: Map = emptyMap(), -) +) { + companion object { + fun fromSChapter( + sChapter: SChapter, + id: Int, + index: Int, + fetchedAt: Long, + mangaId: Int, + realUrl: String?, + ): ChapterDataClass { + return ChapterDataClass( + id = id, + url = sChapter.url, + name = sChapter.name, + uploadDate = sChapter.date_upload, + chapterNumber = sChapter.chapter_number, + scanlator = sChapter.scanlator ?: "", + index = index, + fetchedAt = fetchedAt, + realUrl = realUrl, + mangaId = mangaId, + read = false, + bookmarked = false, + lastPageRead = 0, + lastReadAt = 0, + downloaded = false, + ) + } + } +}