From ef6be74ec25d0b76db9dc55c986605a90916af36 Mon Sep 17 00:00:00 2001 From: schroda <50052685+schroda@users.noreply.github.com> Date: Sun, 1 Sep 2024 00:54:06 +0200 Subject: [PATCH] Fix/chapter downloaded check (#1012) * Properly check for first page in cbz files The download check for cbz files only checked if the archive existed but didn't check for the first page * Streamline getImageImpl of ChapterDownloadProviders * Exclude comic info file from page list In case the download folder did not contain any page files, only the comic info file existed, which caused the download check to incorrectly detect the first page * Add logging to ChapterForDownload#asDownloadReady --- .../manga/impl/ChapterDownloadHelper.kt | 2 +- .../manga/impl/chapter/ChapterForDownload.kt | 61 ++++++++++--------- .../fileProvider/ChaptersFilesProvider.kt | 49 ++++++++++++++- .../fileProvider/impl/ArchiveProvider.kt | 17 +++--- .../fileProvider/impl/FolderProvider.kt | 23 ++++--- 5 files changed, 103 insertions(+), 49 deletions(-) diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/ChapterDownloadHelper.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/ChapterDownloadHelper.kt index 9a13adeb..a417f9f8 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/ChapterDownloadHelper.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/ChapterDownloadHelper.kt @@ -41,7 +41,7 @@ object ChapterDownloadHelper { private fun provider( mangaId: Int, chapterId: Int, - ): ChaptersFilesProvider { + ): ChaptersFilesProvider<*> { val chapterFolder = File(getChapterDownloadPath(mangaId, chapterId)) val cbzFile = File(getChapterCbzPath(mangaId, chapterId)) if (cbzFile.exists()) return ArchiveProvider(mangaId, chapterId) diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/chapter/ChapterForDownload.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/chapter/ChapterForDownload.kt index 4eba6d46..946843e4 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/chapter/ChapterForDownload.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/chapter/ChapterForDownload.kt @@ -9,6 +9,8 @@ package suwayomi.tachidesk.manga.impl.chapter import eu.kanade.tachiyomi.source.model.Page import eu.kanade.tachiyomi.source.model.SChapter +import mu.KLogger +import mu.KotlinLogging import org.jetbrains.exposed.sql.ResultRow import org.jetbrains.exposed.sql.SqlExpressionBuilder.eq import org.jetbrains.exposed.sql.and @@ -17,17 +19,13 @@ import org.jetbrains.exposed.sql.deleteWhere import org.jetbrains.exposed.sql.select import org.jetbrains.exposed.sql.transactions.transaction import org.jetbrains.exposed.sql.update -import suwayomi.tachidesk.manga.impl.Page.getPageName -import suwayomi.tachidesk.manga.impl.util.getChapterCbzPath -import suwayomi.tachidesk.manga.impl.util.getChapterDownloadPath +import suwayomi.tachidesk.manga.impl.ChapterDownloadHelper import suwayomi.tachidesk.manga.impl.util.source.GetCatalogueSource.getCatalogueSourceOrStub -import suwayomi.tachidesk.manga.impl.util.storage.ImageResponse import suwayomi.tachidesk.manga.model.dataclass.ChapterDataClass import suwayomi.tachidesk.manga.model.table.ChapterTable import suwayomi.tachidesk.manga.model.table.MangaTable import suwayomi.tachidesk.manga.model.table.PageTable import suwayomi.tachidesk.manga.model.table.toDataClass -import java.io.File suspend fun getChapterDownloadReady( chapterId: Int? = null, @@ -55,8 +53,25 @@ private class ChapterForDownload( optChapterIndex: Int? = null, optMangaId: Int? = null, ) { + var chapterEntry: ResultRow + val chapterId: Int + val chapterIndex: Int + val mangaId: Int + + val logger: KLogger + suspend fun asDownloadReady(): ChapterDataClass { - if (isNotCompletelyDownloaded()) { + val log = KotlinLogging.logger("${logger.name}::asDownloadReady") + + val isMarkedAsDownloaded = chapterEntry[ChapterTable.isDownloaded] + val doesFirstPageExist = firstPageExists() + val isDownloaded = isMarkedAsDownloaded && doesFirstPageExist + + log.debug { "isDownloaded= $isDownloaded (isMarkedAsDownloaded= $isMarkedAsDownloaded, doesFirstPageExist= $doesFirstPageExist)" } + + if (!isDownloaded) { + log.debug { "reset download status and fetch page list" } + markAsNotDownloaded() val pageList = fetchPageList() @@ -69,16 +84,16 @@ private class ChapterForDownload( private fun asDataClass() = ChapterTable.toDataClass(chapterEntry) - var chapterEntry: ResultRow - val chapterId: Int - val chapterIndex: Int - val mangaId: Int - init { chapterEntry = freshChapterEntry(optChapterId, optChapterIndex, optMangaId) chapterId = chapterEntry[ChapterTable.id].value chapterIndex = chapterEntry[ChapterTable.sourceOrder] mangaId = chapterEntry[ChapterTable.manga].value + + logger = + KotlinLogging.logger( + "${ChapterForDownload::class.java.name}(mangaId= $mangaId, chapterId= $chapterId, chapterIndex= $chapterIndex)", + ) } private fun freshChapterEntry( @@ -151,24 +166,12 @@ private class ChapterForDownload( } } - private fun isNotCompletelyDownloaded(): Boolean { - return !( - chapterEntry[ChapterTable.isDownloaded] && - (firstPageExists() || File(getChapterCbzPath(mangaId, chapterEntry[ChapterTable.id].value)).exists()) - ) - } - private fun firstPageExists(): Boolean { - val chapterId = chapterEntry[ChapterTable.id].value - - val chapterDir = getChapterDownloadPath(mangaId, chapterId) - - println(chapterDir) - println(getPageName(0)) - - return ImageResponse.findFileNameStartingWith( - chapterDir, - getPageName(0), - ) != null + return try { + ChapterDownloadHelper.getImage(mangaId, chapterId, 0).first.close() + true + } catch (e: Exception) { + false + } } } diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/fileProvider/ChaptersFilesProvider.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/fileProvider/ChaptersFilesProvider.kt index 20d48d35..e746c122 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/fileProvider/ChaptersFilesProvider.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/fileProvider/ChaptersFilesProvider.kt @@ -1,5 +1,6 @@ package suwayomi.tachidesk.manga.impl.download.fileProvider +import eu.kanade.tachiyomi.source.local.metadata.COMIC_INFO_FILE import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.FlowPreview import kotlinx.coroutines.Job @@ -7,6 +8,7 @@ import kotlinx.coroutines.flow.distinctUntilChanged import kotlinx.coroutines.flow.launchIn import kotlinx.coroutines.flow.onEach import kotlinx.coroutines.flow.sample +import org.apache.commons.compress.archivers.zip.ZipArchiveEntry import org.jetbrains.exposed.sql.select import org.jetbrains.exposed.sql.transactions.transaction import suwayomi.tachidesk.manga.impl.Page @@ -20,11 +22,54 @@ import suwayomi.tachidesk.manga.model.table.MangaTable import java.io.File import java.io.InputStream +sealed class FileType { + data class RegularFile(val file: File) : FileType() + + data class ZipFile(val entry: ZipArchiveEntry) : FileType() + + fun getName(): String { + return when (this) { + is FileType.RegularFile -> { + this.file.name + } + is FileType.ZipFile -> { + this.entry.name + } + } + } + + fun getExtension(): String { + return when (this) { + is FileType.RegularFile -> { + this.file.extension + } + is FileType.ZipFile -> { + this.entry.name.substringAfterLast(".") + } + } + } +} + /* * Base class for downloaded chapter files provider, example: Folder, Archive */ -abstract class ChaptersFilesProvider(val mangaId: Int, val chapterId: Int) : DownloadedFilesProvider { - abstract fun getImageImpl(index: Int): Pair +abstract class ChaptersFilesProvider(val mangaId: Int, val chapterId: Int) : DownloadedFilesProvider { + protected abstract fun getImageFiles(): List + + protected abstract fun getImageInputStream(image: Type): InputStream + + fun getImageImpl(index: Int): Pair { + val images = getImageFiles().filter { it.getName() != COMIC_INFO_FILE }.sortedBy { it.getName() } + + if (images.isEmpty()) { + throw Exception("no downloaded images found") + } + + val image = images[index] + val imageFileType = image.getExtension() + + return Pair(getImageInputStream(image).buffered(), "image/$imageFileType") + } override fun getImage(): RetrieveFile1Args { return RetrieveFile1Args(::getImageImpl) diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/fileProvider/impl/ArchiveProvider.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/fileProvider/impl/ArchiveProvider.kt index 4d7886ff..2ac29521 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/fileProvider/impl/ArchiveProvider.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/fileProvider/impl/ArchiveProvider.kt @@ -10,6 +10,7 @@ import org.kodein.di.DI import org.kodein.di.conf.global import org.kodein.di.instance import suwayomi.tachidesk.manga.impl.download.fileProvider.ChaptersFilesProvider +import suwayomi.tachidesk.manga.impl.download.fileProvider.FileType import suwayomi.tachidesk.manga.impl.util.getChapterCachePath import suwayomi.tachidesk.manga.impl.util.getChapterCbzPath import suwayomi.tachidesk.manga.impl.util.getMangaDownloadDir @@ -20,14 +21,14 @@ import java.io.InputStream private val applicationDirs by DI.global.instance() -class ArchiveProvider(mangaId: Int, chapterId: Int) : ChaptersFilesProvider(mangaId, chapterId) { - override fun getImageImpl(index: Int): Pair { - val cbzPath = getChapterCbzPath(mangaId, chapterId) - val zipFile = ZipFile(cbzPath) - val zipEntry = zipFile.entries.toList().sortedWith(compareBy({ it.name }, { it.name }))[index] - val inputStream = zipFile.getInputStream(zipEntry) - val fileType = zipEntry.name.substringAfterLast(".") - return Pair(inputStream.buffered(), "image/$fileType") +class ArchiveProvider(mangaId: Int, chapterId: Int) : ChaptersFilesProvider(mangaId, chapterId) { + override fun getImageFiles(): List { + val zipFile = ZipFile(getChapterCbzPath(mangaId, chapterId)) + return zipFile.entries.toList().map { FileType.ZipFile(it) } + } + + override fun getImageInputStream(image: FileType.ZipFile): InputStream { + return ZipFile(getChapterCbzPath(mangaId, chapterId)).getInputStream(image.entry) } override fun extractExistingDownload() { diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/fileProvider/impl/FolderProvider.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/fileProvider/impl/FolderProvider.kt index 3b5766dc..0386b7b8 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/fileProvider/impl/FolderProvider.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/fileProvider/impl/FolderProvider.kt @@ -4,27 +4,32 @@ import org.kodein.di.DI import org.kodein.di.conf.global import org.kodein.di.instance import suwayomi.tachidesk.manga.impl.download.fileProvider.ChaptersFilesProvider +import suwayomi.tachidesk.manga.impl.download.fileProvider.FileType.RegularFile import suwayomi.tachidesk.manga.impl.util.getChapterCachePath import suwayomi.tachidesk.manga.impl.util.getChapterDownloadPath import suwayomi.tachidesk.manga.impl.util.storage.FileDeletionHelper import suwayomi.tachidesk.server.ApplicationDirs import java.io.File import java.io.FileInputStream -import java.io.InputStream private val applicationDirs by DI.global.instance() /* * Provides downloaded files when pages were downloaded into folders * */ -class FolderProvider(mangaId: Int, chapterId: Int) : ChaptersFilesProvider(mangaId, chapterId) { - override fun getImageImpl(index: Int): Pair { - val chapterDir = getChapterDownloadPath(mangaId, chapterId) - val folder = File(chapterDir) - folder.mkdirs() - val file = folder.listFiles()?.sortedBy { it.name }?.get(index) - val fileType = file!!.name.substringAfterLast(".") - return Pair(FileInputStream(file).buffered(), "image/$fileType") +class FolderProvider(mangaId: Int, chapterId: Int) : ChaptersFilesProvider(mangaId, chapterId) { + override fun getImageFiles(): List { + val chapterFolder = File(getChapterDownloadPath(mangaId, chapterId)) + + if (!chapterFolder.exists()) { + throw Exception("download folder does not exist") + } + + return chapterFolder.listFiles().orEmpty().toList().map(::RegularFile) + } + + override fun getImageInputStream(image: RegularFile): FileInputStream { + return FileInputStream(image.file) } override fun extractExistingDownload() {