From 69826596580768e4927c09c9d3f45d4135e9e56c Mon Sep 17 00:00:00 2001 From: schroda <50052685+schroda@users.noreply.github.com> Date: Sun, 28 Jul 2024 21:57:50 +0200 Subject: [PATCH] Fix/inserting duplicated chapters into database (#991) * Prevent adding duplicated chapters into the db it's possible that the source returns a list containing chapters with the same url once such duplicated chapters have been added, they aren't being removed anymore as long as there is a chapter with the same url in the fetched chapter list, even if the duplicated chapter itself does not exist anymore on the source * Drop duplicated chapters from database table * Add unique constraint to chapter table This is to completely prevent duplicated chapters from being added to the database. Since once a duplicated chapter has been added to the database, it does not get removed anymore as long as a chapter with the same url is included in the requested source chapter list --- .../suwayomi/tachidesk/manga/impl/Chapter.kt | 18 ++++++++++----- .../M0039_DeleteDuplicatedChapters.kt | 23 +++++++++++++++++++ ...M0040_AddUniqueConstraintToChapterTable.kt | 19 +++++++++++++++ 3 files changed, 54 insertions(+), 6 deletions(-) create mode 100644 server/src/main/kotlin/suwayomi/tachidesk/server/database/migration/M0039_DeleteDuplicatedChapters.kt create mode 100644 server/src/main/kotlin/suwayomi/tachidesk/server/database/migration/M0040_AddUniqueConstraintToChapterTable.kt diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Chapter.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Chapter.kt index 2e896062..bb03f1c8 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Chapter.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Chapter.kt @@ -145,14 +145,20 @@ object Chapter { val currentLatestChapterNumber = Manga.getLatestChapter(mangaId)?.chapterNumber ?: 0f val numberOfCurrentChapters = getCountOfMangaChapters(mangaId) - val chapterList = source.getChapterList(sManga) - if (chapterList.isEmpty()) { + val chapters = source.getChapterList(sManga) + // it's possible that the source returns a list containing chapters with the same url + // once such duplicated chapters have been added, they aren't being removed anymore as long as there is + // a chapter with the same url in the fetched chapter list, even if the duplicated chapter itself + // does not exist anymore on the source + val uniqueChapters = chapters.distinctBy { it.url } + + if (uniqueChapters.isEmpty()) { throw Exception("No chapters found") } // Recognize number for new chapters. - chapterList.forEach { chapter -> + uniqueChapters.forEach { chapter -> (source as? HttpSource)?.prepareNewChapter(chapter, sManga) val chapterNumber = ChapterRecognition.parseChapterNumber(manga.title, chapter.name, chapter.chapter_number.toDouble()) chapter.chapter_number = chapterNumber.toFloat() @@ -178,7 +184,7 @@ object Chapter { val chaptersToInsert = mutableListOf() // do not yet have an ID from the database val chaptersToUpdate = mutableListOf() - chapterList.reversed().forEachIndexed { index, fetchedChapter -> + uniqueChapters.reversed().forEachIndexed { index, fetchedChapter -> val chapterEntry = chaptersInDb.find { it.url == fetchedChapter.url } val chapterData = @@ -221,7 +227,7 @@ object Chapter { val deletedChapterNumberDateFetchMap = mutableMapOf() // clear any orphaned/duplicate chapters that are in the db but not in `chapterList` - val chapterUrls = chapterList.map { it.url }.toSet() + val chapterUrls = uniqueChapters.map { it.url }.toSet() val chaptersIdsToDelete = chaptersInDb.mapNotNull { dbChapter -> @@ -298,7 +304,7 @@ object Chapter { downloadNewChapters(mangaId, currentLatestChapterNumber, numberOfCurrentChapters, insertedChapters) } - chapterList + uniqueChapters } return chapterList diff --git a/server/src/main/kotlin/suwayomi/tachidesk/server/database/migration/M0039_DeleteDuplicatedChapters.kt b/server/src/main/kotlin/suwayomi/tachidesk/server/database/migration/M0039_DeleteDuplicatedChapters.kt new file mode 100644 index 00000000..b10d8b09 --- /dev/null +++ b/server/src/main/kotlin/suwayomi/tachidesk/server/database/migration/M0039_DeleteDuplicatedChapters.kt @@ -0,0 +1,23 @@ +package suwayomi.tachidesk.server.database.migration + +/* + * Copyright (C) Contributors to the Suwayomi project + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +import de.neonew.exposed.migrations.helpers.SQLMigration + +@Suppress("ClassName", "unused") +class M0039_DeleteDuplicatedChapters : SQLMigration() { + override val sql: String = + """ + DELETE FROM CHAPTER + WHERE id NOT IN ( + SELECT MIN(id) + FROM CHAPTER + GROUP BY URL, MANGA + ); + """.trimIndent() +} diff --git a/server/src/main/kotlin/suwayomi/tachidesk/server/database/migration/M0040_AddUniqueConstraintToChapterTable.kt b/server/src/main/kotlin/suwayomi/tachidesk/server/database/migration/M0040_AddUniqueConstraintToChapterTable.kt new file mode 100644 index 00000000..288c483a --- /dev/null +++ b/server/src/main/kotlin/suwayomi/tachidesk/server/database/migration/M0040_AddUniqueConstraintToChapterTable.kt @@ -0,0 +1,19 @@ +package suwayomi.tachidesk.server.database.migration + +/* + * Copyright (C) Contributors to the Suwayomi project + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +import de.neonew.exposed.migrations.helpers.SQLMigration + +@Suppress("ClassName", "unused") +class M0040_AddUniqueConstraintToChapterTable : SQLMigration() { + override val sql: String = + """ + ALTER TABLE CHAPTER + ADD CONSTRAINT UC_CHAPTER UNIQUE (URL, MANGA) + """.trimIndent() +}