Feature/improve browsing source performance (#743)

* Improve browse source database operations

* Reuse "insertOrGet" for "processEntries"

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/MangaList.kt

@@ -8,18 +8,14 @@ package suwayomi.tachidesk.manga.impl
  * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
 
 import eu.kanade.tachiyomi.source.model.MangasPage
-import eu.kanade.tachiyomi.source.model.UpdateStrategy
 import org.jetbrains.exposed.sql.and
-import org.jetbrains.exposed.sql.insertAndGetId
+import org.jetbrains.exposed.sql.batchInsert
 import org.jetbrains.exposed.sql.select
 import org.jetbrains.exposed.sql.transactions.transaction
-import suwayomi.tachidesk.manga.impl.Manga.getMangaMetaMap
 import suwayomi.tachidesk.manga.impl.util.source.GetCatalogueSource.getCatalogueSourceOrStub
-import suwayomi.tachidesk.manga.model.dataclass.MangaDataClass
 import suwayomi.tachidesk.manga.model.dataclass.PagedMangaListDataClass
-import suwayomi.tachidesk.manga.model.dataclass.toGenreList
-import suwayomi.tachidesk.manga.model.table.MangaStatus
 import suwayomi.tachidesk.manga.model.table.MangaTable
+import suwayomi.tachidesk.manga.model.table.toDataClass
 
 object MangaList {
     fun proxyThumbnailUrl(mangaId: Int): String {
@@ -50,35 +46,38 @@ object MangaList {
 
     fun MangasPage.insertOrGet(sourceId: Long): List<Int> {
         return transaction {
+            val existingMangaUrlsToId =
+                MangaTable.slice(MangaTable.url, MangaTable.id).select {
+                    (MangaTable.sourceReference eq sourceId) and (MangaTable.url inList mangas.map { it.url })
+                }.associate { Pair(it[MangaTable.url], it[MangaTable.id].value) }
+            val existingMangaUrls = existingMangaUrlsToId.map { it.key }
+
+            val mangasToInsert = mangas.filter { !existingMangaUrls.contains(it.url) }
+
+            val insertedMangaUrlsToId =
+                MangaTable.batchInsert(mangasToInsert) {
+                    this[MangaTable.url] = it.url
+                    this[MangaTable.title] = it.title
+
+                    this[MangaTable.artist] = it.artist
+                    this[MangaTable.author] = it.author
+                    this[MangaTable.description] = it.description
+                    this[MangaTable.genre] = it.genre
+                    this[MangaTable.status] = it.status
+                    this[MangaTable.thumbnail_url] = it.thumbnail_url
+                    this[MangaTable.updateStrategy] = it.update_strategy.name
+
+                    this[MangaTable.sourceReference] = sourceId
+                }.associate { Pair(it[MangaTable.url], it[MangaTable.id].value) }
+
+            // delete thumbnail in case cached data still exists
+            insertedMangaUrlsToId.forEach { (_, id) -> Manga.clearThumbnail(id) }
+
+            val mangaUrlsToId = existingMangaUrlsToId + insertedMangaUrlsToId
+
             mangas.map { manga ->
-                val mangaEntry =
+                mangaUrlsToId[manga.url]
-                    MangaTable.select {
+                    ?: throw Exception("MangaList::insertOrGet($sourceId): Something went wrong inserting browsed source mangas")
-                        (MangaTable.url eq manga.url) and (MangaTable.sourceReference eq sourceId)
-                    }.firstOrNull()
-                if (mangaEntry == null) { // create manga entry
-                    val mangaId =
-                        MangaTable.insertAndGetId {
-                            it[url] = manga.url
-                            it[title] = manga.title
-
-                            it[artist] = manga.artist
-                            it[author] = manga.author
-                            it[description] = manga.description
-                            it[genre] = manga.genre
-                            it[status] = manga.status
-                            it[thumbnail_url] = manga.thumbnail_url
-                            it[updateStrategy] = manga.update_strategy.name
-
-                            it[sourceReference] = sourceId
-                        }.value
-
-                    // delete thumbnail in case cached data still exists
-                    Manga.clearThumbnail(mangaId)
-
-                    mangaId
-                } else {
-                    mangaEntry[MangaTable.id].value
-                }
             }
         }
     }
@@ -87,84 +86,8 @@ object MangaList {
         val mangasPage = this
         val mangaList =
             transaction {
-                return@transaction mangasPage.mangas.map { manga ->
+                val mangaIds = insertOrGet(sourceId)
-                    var mangaEntry =
+                return@transaction MangaTable.select { MangaTable.id inList mangaIds }.map { MangaTable.toDataClass(it) }
-                        MangaTable.select {
-                            (MangaTable.url eq manga.url) and (MangaTable.sourceReference eq sourceId)
-                        }.firstOrNull()
-                    if (mangaEntry == null) { // create manga entry
-                        val mangaId =
-                            MangaTable.insertAndGetId {
-                                it[url] = manga.url
-                                it[title] = manga.title
-
-                                it[artist] = manga.artist
-                                it[author] = manga.author
-                                it[description] = manga.description
-                                it[genre] = manga.genre
-                                it[status] = manga.status
-                                it[thumbnail_url] = manga.thumbnail_url
-                                it[updateStrategy] = manga.update_strategy.name
-
-                                it[sourceReference] = sourceId
-                            }.value
-
-                        // delete thumbnail in case cached data still exists
-                        Manga.clearThumbnail(mangaId)
-
-                        mangaEntry =
-                            MangaTable.select {
-                                (MangaTable.url eq manga.url) and (MangaTable.sourceReference eq sourceId)
-                            }.first()
-
-                        MangaDataClass(
-                            id = mangaId,
-                            sourceId = sourceId.toString(),
-                            url = manga.url,
-                            title = manga.title,
-                            thumbnailUrl = proxyThumbnailUrl(mangaId),
-                            thumbnailUrlLastFetched = mangaEntry[MangaTable.thumbnailUrlLastFetched],
-                            initialized = manga.initialized,
-                            artist = manga.artist,
-                            author = manga.author,
-                            description = manga.description,
-                            genre = manga.genre.toGenreList(),
-                            status = MangaStatus.valueOf(manga.status).name,
-                            inLibrary = false, // It's a new manga entry
-                            inLibraryAt = 0,
-                            meta = getMangaMetaMap(mangaId),
-                            realUrl = mangaEntry[MangaTable.realUrl],
-                            lastFetchedAt = mangaEntry[MangaTable.lastFetchedAt],
-                            chaptersLastFetchedAt = mangaEntry[MangaTable.chaptersLastFetchedAt],
-                            updateStrategy = UpdateStrategy.valueOf(mangaEntry[MangaTable.updateStrategy]),
-                            freshData = true,
-                        )
-                    } else {
-                        val mangaId = mangaEntry[MangaTable.id].value
-                        MangaDataClass(
-                            id = mangaId,
-                            sourceId = sourceId.toString(),
-                            url = manga.url,
-                            title = manga.title,
-                            thumbnailUrl = proxyThumbnailUrl(mangaId),
-                            thumbnailUrlLastFetched = mangaEntry[MangaTable.thumbnailUrlLastFetched],
-                            initialized = true,
-                            artist = mangaEntry[MangaTable.artist],
-                            author = mangaEntry[MangaTable.author],
-                            description = mangaEntry[MangaTable.description],
-                            genre = mangaEntry[MangaTable.genre].toGenreList(),
-                            status = MangaStatus.valueOf(mangaEntry[MangaTable.status]).name,
-                            inLibrary = mangaEntry[MangaTable.inLibrary],
-                            inLibraryAt = mangaEntry[MangaTable.inLibraryAt],
-                            meta = getMangaMetaMap(mangaId),
-                            realUrl = mangaEntry[MangaTable.realUrl],
-                            lastFetchedAt = mangaEntry[MangaTable.lastFetchedAt],
-                            chaptersLastFetchedAt = mangaEntry[MangaTable.chaptersLastFetchedAt],
-                            updateStrategy = UpdateStrategy.valueOf(mangaEntry[MangaTable.updateStrategy]),
-                            freshData = false,
-                        )
-                    }
-                }
             }
         return PagedMangaListDataClass(
             mangaList,