From 538bd3f126a5f89aa10a0775f8c894c443dae59f Mon Sep 17 00:00:00 2001 From: Mitchell Syer Date: Fri, 16 May 2025 15:57:53 -0400 Subject: [PATCH] Improve Downloads Handling (#1387) * Improve Downloads Handling * Update known pagecount for downloaded chapters * Get fresh data for downloadReady * Format * Assume downloaded if first page is found * Filter out ComicInfoFile --- .../manga/impl/ChapterDownloadHelper.kt | 5 + .../suwayomi/tachidesk/manga/impl/Page.kt | 7 +- .../manga/impl/chapter/ChapterForDownload.kt | 22 +++-- .../fileProvider/ChaptersFilesProvider.kt | 97 +++++++++++-------- .../suwayomi/tachidesk/manga/impl/PageTest.kt | 8 +- .../manga/impl/update/TestUpdater.kt | 6 ++ 6 files changed, 94 insertions(+), 51 deletions(-) diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/ChapterDownloadHelper.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/ChapterDownloadHelper.kt index 619a65fc..3d3463c4 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/ChapterDownloadHelper.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/ChapterDownloadHelper.kt @@ -22,6 +22,11 @@ object ChapterDownloadHelper { index: Int, ): Pair = provider(mangaId, chapterId).getImage().execute(index) + fun getImageCount( + mangaId: Int, + chapterId: Int, + ): Int = provider(mangaId, chapterId).getImageCount() + fun delete( mangaId: Int, chapterId: Int, diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Page.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Page.kt index 8c938ce1..19c4f9bc 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Page.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Page.kt @@ -110,7 +110,7 @@ object Page { } } - val fileName = getPageName(index) + val fileName = getPageName(index, chapterEntry[ChapterTable.pageCount]) val cacheSaveDir = getChapterCachePath(mangaId, chapterId) @@ -121,5 +121,8 @@ object Page { } /** converts 0 to "001" */ - fun getPageName(index: Int): String = String.format("%03d", index + 1) + fun getPageName( + index: Int, + pageCount: Int, + ): String = String.format("%0${pageCount.toString().length.coerceAtLeast(3)}d", index + 1) } diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/chapter/ChapterForDownload.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/chapter/ChapterForDownload.kt index 5222b1e0..f7c44e7a 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/chapter/ChapterForDownload.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/chapter/ChapterForDownload.kt @@ -70,7 +70,7 @@ private class ChapterForDownload( val isMarkedAsDownloaded = chapterEntry[ChapterTable.isDownloaded] val doesFirstPageExist = firstPageExists() - val isDownloaded = isMarkedAsDownloaded && doesFirstPageExist + val isDownloaded = isMarkedAsDownloaded || doesFirstPageExist log.debug { "isDownloaded= $isDownloaded (isMarkedAsDownloaded= $isMarkedAsDownloaded, doesFirstPageExist= $doesFirstPageExist)" } @@ -80,12 +80,22 @@ private class ChapterForDownload( markAsNotDownloaded() updatePageList() + } else { + updatePageCount(ChapterDownloadHelper.getImageCount(mangaId, chapterId)) } return asDataClass() } - private fun asDataClass() = ChapterTable.toDataClass(chapterEntry) + private fun asDataClass() = + ChapterTable.toDataClass( + transaction { + ChapterTable + .selectAll() + .where { ChapterTable.id eq chapterId } + .first() + }, + ) init { chapterEntry = freshChapterEntry(optChapterId, optChapterIndex, optMangaId) @@ -160,19 +170,15 @@ private class ChapterForDownload( } } - updatePageCount(pageList, chapterId) + updatePageCount(pageList.size) // chapter was updated chapterEntry = freshChapterEntry(chapterId, chapterIndex, mangaId) } - private fun updatePageCount( - pageList: List, - chapterId: Int, - ) { + private fun updatePageCount(pageCount: Int) { transaction { ChapterTable.update({ ChapterTable.id eq chapterId }) { - val pageCount = pageList.size it[ChapterTable.pageCount] = pageCount it[ChapterTable.lastPageRead] = chapterEntry[ChapterTable.lastPageRead].coerceAtMost(pageCount - 1).coerceAtLeast(0) } diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/fileProvider/ChaptersFilesProvider.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/fileProvider/ChaptersFilesProvider.kt index aa891f0d..e85c5645 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/fileProvider/ChaptersFilesProvider.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/fileProvider/ChaptersFilesProvider.kt @@ -9,8 +9,10 @@ import kotlinx.coroutines.flow.launchIn import kotlinx.coroutines.flow.onEach import kotlinx.coroutines.flow.sample import org.apache.commons.compress.archivers.zip.ZipArchiveEntry +import org.jetbrains.exposed.sql.SqlExpressionBuilder.eq import org.jetbrains.exposed.sql.selectAll import org.jetbrains.exposed.sql.transactions.transaction +import org.jetbrains.exposed.sql.update import suwayomi.tachidesk.manga.impl.Page import suwayomi.tachidesk.manga.impl.download.model.DownloadChapter import suwayomi.tachidesk.manga.impl.util.createComicInfoFile @@ -76,6 +78,8 @@ abstract class ChaptersFilesProvider( return Pair(getImageInputStream(image).buffered(), "image/$imageFileType") } + fun getImageCount(): Int = getImageFiles().filter { it.getName() != COMIC_INFO_FILE }.size + override fun getImage(): RetrieveFile1Args = RetrieveFile1Args(::getImageImpl) /** @@ -100,45 +104,56 @@ abstract class ChaptersFilesProvider( downloadCacheFolder.mkdirs() val pageCount = download.chapter.pageCount - for (pageNum in 0 until pageCount) { - var pageProgressJob: Job? = null - val fileName = Page.getPageName(pageNum) // might have to change this to index stored in database - - val pageExistsInFinalDownloadFolder = ImageResponse.findFileNameStartingWith(finalDownloadFolder, fileName) != null - val pageExistsInCacheDownloadFolder = ImageResponse.findFileNameStartingWith(cacheChapterDir, fileName) != null - - val doesPageAlreadyExist = pageExistsInFinalDownloadFolder || pageExistsInCacheDownloadFolder - if (doesPageAlreadyExist) { - continue - } - - try { - Page - .getPageImage( - mangaId = download.mangaId, - chapterIndex = download.chapterIndex, - index = pageNum, - ) { flow -> - pageProgressJob = - flow - .sample(100) - .distinctUntilChanged() - .onEach { - download.progress = (pageNum.toFloat() + (it.toFloat() * 0.01f)) / pageCount - step( - null, - false, - ) // don't throw on canceled download here since we can't do anything - }.launchIn(scope) - }.first - .close() - } finally { - // always cancel the page progress job even if it throws an exception to avoid memory leaks - pageProgressJob?.cancel() - } - // TODO: retry on error with 2,4,8 seconds of wait - download.progress = ((pageNum + 1).toFloat()) / pageCount + if ( + downloadCacheFolder + .listFiles() + .orEmpty() + .filter { it.name != COMIC_INFO_FILE } + .size >= pageCount + ) { + download.progress = 1f step(download, false) + } else { + for (pageNum in 0 until pageCount) { + var pageProgressJob: Job? = null + val fileName = Page.getPageName(pageNum, pageCount) // might have to change this to index stored in database + + val pageExistsInFinalDownloadFolder = ImageResponse.findFileNameStartingWith(finalDownloadFolder, fileName) != null + val pageExistsInCacheDownloadFolder = ImageResponse.findFileNameStartingWith(cacheChapterDir, fileName) != null + + val doesPageAlreadyExist = pageExistsInFinalDownloadFolder || pageExistsInCacheDownloadFolder + if (doesPageAlreadyExist) { + continue + } + + try { + Page + .getPageImage( + mangaId = download.mangaId, + chapterIndex = download.chapterIndex, + index = pageNum, + ) { flow -> + pageProgressJob = + flow + .sample(100) + .distinctUntilChanged() + .onEach { + download.progress = (pageNum.toFloat() + (it.toFloat() * 0.01f)) / pageCount + step( + null, + false, + ) // don't throw on canceled download here since we can't do anything + }.launchIn(scope) + }.first + .close() + } finally { + // always cancel the page progress job even if it throws an exception to avoid memory leaks + pageProgressJob?.cancel() + } + // TODO: retry on error with 2,4,8 seconds of wait + download.progress = ((pageNum + 1).toFloat()) / pageCount + step(download, false) + } } createComicInfoFile( @@ -153,6 +168,12 @@ abstract class ChaptersFilesProvider( handleSuccessfulDownload() + transaction { + ChapterTable.update({ ChapterTable.id eq chapterId }) { + it[ChapterTable.pageCount] = getImageCount() + } + } + File(cacheChapterDir).deleteRecursively() return true diff --git a/server/src/test/kotlin/suwayomi/tachidesk/manga/impl/PageTest.kt b/server/src/test/kotlin/suwayomi/tachidesk/manga/impl/PageTest.kt index 18a07ca7..3b4d5c5a 100644 --- a/server/src/test/kotlin/suwayomi/tachidesk/manga/impl/PageTest.kt +++ b/server/src/test/kotlin/suwayomi/tachidesk/manga/impl/PageTest.kt @@ -15,16 +15,18 @@ import kotlin.test.assertEquals class PageTest : ApplicationTest() { @Test fun testGetPageName() { - val tests = listOf(0, 1, 2, 100) + val tests = listOf(0 to 5, 1 to 5, 2 to 5, 100 to 100, 998 to 1000, 1400 to 1500) val testResults = - tests.map { - getPageName(it) + tests.map { (page, count) -> + getPageName(page, count) } assertEquals(testResults[0], "001") assertEquals(testResults[1], "002") assertEquals(testResults[2], "003") assertEquals(testResults[3], "101") + assertEquals(testResults[4], "0999") + assertEquals(testResults[5], "1401") } } diff --git a/server/src/test/kotlin/suwayomi/tachidesk/manga/impl/update/TestUpdater.kt b/server/src/test/kotlin/suwayomi/tachidesk/manga/impl/update/TestUpdater.kt index 8a8570f5..3b319a50 100644 --- a/server/src/test/kotlin/suwayomi/tachidesk/manga/impl/update/TestUpdater.kt +++ b/server/src/test/kotlin/suwayomi/tachidesk/manga/impl/update/TestUpdater.kt @@ -45,10 +45,16 @@ class TestUpdater : IUpdater { override val status: Flow get() = TODO("Not yet implemented") + override val updates: Flow + get() = TODO("Not yet implemented") override val statusDeprecated: StateFlow get() = TODO("Not yet implemented") override fun reset() { TODO("Not yet implemented") } + + override fun getStatus(): UpdateUpdates { + TODO("Not yet implemented") + } }