admin/Suwayomi-Server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve Downloads Handling (#1387)

Browse Source 
* Improve Downloads Handling

* Update known pagecount for downloaded chapters

* Get fresh data for downloadReady

* Format

* Assume downloaded if first page is found

* Filter out ComicInfoFile
This commit is contained in:
Mitchell Syer
2025-05-16 15:57:53 -04:00
committed by GitHub
parent 336f985894
commit 538bd3f126
6 changed files with 94 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/src/main/kotlin/suwayomi/tachidesk/manga/impl/ChapterDownloadHelper.kt
+5
View File
@@ -22,6 +22,11 @@ object ChapterDownloadHelper {
index: Int,
): Pair<InputStream, String> = provider(mangaId, chapterId).getImage().execute(index)
fun getImageCount(
mangaId: Int,
chapterId: Int,
): Int = provider(mangaId, chapterId).getImageCount()
fun delete(
mangaId: Int,
chapterId: Int,
server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Page.kt
+5 -2
View File
@@ -110,7 +110,7 @@ object Page {
}
}
val fileName = getPageName(index)
val fileName = getPageName(index, chapterEntry[ChapterTable.pageCount])
val cacheSaveDir = getChapterCachePath(mangaId, chapterId)
@@ -121,5 +121,8 @@ object Page {
}
/** converts 0 to "001" */
fun getPageName(index: Int): String = String.format("%03d", index + 1)
fun getPageName(
index: Int,
pageCount: Int,
): String = String.format("%0${pageCount.toString().length.coerceAtLeast(3)}d", index + 1)
}
server/src/main/kotlin/suwayomi/tachidesk/manga/impl/chapter/ChapterForDownload.kt
+14 -8
View File
@@ -70,7 +70,7 @@ private class ChapterForDownload(
val isMarkedAsDownloaded = chapterEntry[ChapterTable.isDownloaded]
val doesFirstPageExist = firstPageExists()
val isDownloaded = isMarkedAsDownloaded && doesFirstPageExist
val isDownloaded = isMarkedAsDownloaded || doesFirstPageExist
log.debug { "isDownloaded= $isDownloaded (isMarkedAsDownloaded= $isMarkedAsDownloaded, doesFirstPageExist= $doesFirstPageExist)" }
@@ -80,12 +80,22 @@ private class ChapterForDownload(
markAsNotDownloaded()
updatePageList()
} else {
updatePageCount(ChapterDownloadHelper.getImageCount(mangaId, chapterId))
}
return asDataClass()
}
private fun asDataClass() = ChapterTable.toDataClass(chapterEntry)
private fun asDataClass() =
ChapterTable.toDataClass(
transaction {
ChapterTable
.selectAll()
.where { ChapterTable.id eq chapterId }
.first()
},
)
init {
chapterEntry = freshChapterEntry(optChapterId, optChapterIndex, optMangaId)
@@ -160,19 +170,15 @@ private class ChapterForDownload(
}
}
updatePageCount(pageList, chapterId)
updatePageCount(pageList.size)
// chapter was updated
chapterEntry = freshChapterEntry(chapterId, chapterIndex, mangaId)
}
private fun updatePageCount(
pageList: List<Page>,
chapterId: Int,
) {
private fun updatePageCount(pageCount: Int) {
transaction {
ChapterTable.update({ ChapterTable.id eq chapterId }) {
val pageCount = pageList.size
it[ChapterTable.pageCount] = pageCount
it[ChapterTable.lastPageRead] = chapterEntry[ChapterTable.lastPageRead].coerceAtMost(pageCount - 1).coerceAtLeast(0)
}
server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/fileProvider/ChaptersFilesProvider.kt
+59 -38
View File
@@ -9,8 +9,10 @@ import kotlinx.coroutines.flow.launchIn
import kotlinx.coroutines.flow.onEach
import kotlinx.coroutines.flow.sample
import org.apache.commons.compress.archivers.zip.ZipArchiveEntry
import org.jetbrains.exposed.sql.SqlExpressionBuilder.eq
import org.jetbrains.exposed.sql.selectAll
import org.jetbrains.exposed.sql.transactions.transaction
import org.jetbrains.exposed.sql.update
import suwayomi.tachidesk.manga.impl.Page
import suwayomi.tachidesk.manga.impl.download.model.DownloadChapter
import suwayomi.tachidesk.manga.impl.util.createComicInfoFile
@@ -76,6 +78,8 @@ abstract class ChaptersFilesProvider<Type : FileType>(
return Pair(getImageInputStream(image).buffered(), "image/$imageFileType")
}
fun getImageCount(): Int = getImageFiles().filter { it.getName() != COMIC_INFO_FILE }.size
override fun getImage(): RetrieveFile1Args<Int> = RetrieveFile1Args(::getImageImpl)
/**
@@ -100,45 +104,56 @@ abstract class ChaptersFilesProvider<Type : FileType>(
downloadCacheFolder.mkdirs()
val pageCount = download.chapter.pageCount
for (pageNum in 0 until pageCount) {
var pageProgressJob: Job? = null
val fileName = Page.getPageName(pageNum) // might have to change this to index stored in database
val pageExistsInFinalDownloadFolder = ImageResponse.findFileNameStartingWith(finalDownloadFolder, fileName) != null
val pageExistsInCacheDownloadFolder = ImageResponse.findFileNameStartingWith(cacheChapterDir, fileName) != null
val doesPageAlreadyExist = pageExistsInFinalDownloadFolder || pageExistsInCacheDownloadFolder
if (doesPageAlreadyExist) {
continue
}
try {
Page
.getPageImage(
mangaId = download.mangaId,
chapterIndex = download.chapterIndex,
index = pageNum,
) { flow ->
pageProgressJob =
flow
.sample(100)
.distinctUntilChanged()
.onEach {
download.progress = (pageNum.toFloat() + (it.toFloat() * 0.01f)) / pageCount
step(
null,
false,
) // don't throw on canceled download here since we can't do anything
}.launchIn(scope)
}.first
.close()
} finally {
// always cancel the page progress job even if it throws an exception to avoid memory leaks
pageProgressJob?.cancel()
}
// TODO: retry on error with 2,4,8 seconds of wait
download.progress = ((pageNum + 1).toFloat()) / pageCount
if (
downloadCacheFolder
.listFiles()
.orEmpty()
.filter { it.name != COMIC_INFO_FILE }
.size >= pageCount
) {
download.progress = 1f
step(download, false)
} else {
for (pageNum in 0 until pageCount) {
var pageProgressJob: Job? = null
val fileName = Page.getPageName(pageNum, pageCount) // might have to change this to index stored in database
val pageExistsInFinalDownloadFolder = ImageResponse.findFileNameStartingWith(finalDownloadFolder, fileName) != null
val pageExistsInCacheDownloadFolder = ImageResponse.findFileNameStartingWith(cacheChapterDir, fileName) != null
val doesPageAlreadyExist = pageExistsInFinalDownloadFolder || pageExistsInCacheDownloadFolder
if (doesPageAlreadyExist) {
continue
}
try {
Page
.getPageImage(
mangaId = download.mangaId,
chapterIndex = download.chapterIndex,
index = pageNum,
) { flow ->
pageProgressJob =
flow
.sample(100)
.distinctUntilChanged()
.onEach {
download.progress = (pageNum.toFloat() + (it.toFloat() * 0.01f)) / pageCount
step(
null,
false,
) // don't throw on canceled download here since we can't do anything
}.launchIn(scope)
}.first
.close()
} finally {
// always cancel the page progress job even if it throws an exception to avoid memory leaks
pageProgressJob?.cancel()
}
// TODO: retry on error with 2,4,8 seconds of wait
download.progress = ((pageNum + 1).toFloat()) / pageCount
step(download, false)
}
}
createComicInfoFile(
@@ -153,6 +168,12 @@ abstract class ChaptersFilesProvider<Type : FileType>(
handleSuccessfulDownload()
transaction {
ChapterTable.update({ ChapterTable.id eq chapterId }) {
it[ChapterTable.pageCount] = getImageCount()
}
}
File(cacheChapterDir).deleteRecursively()
return true
server/src/test/kotlin/suwayomi/tachidesk/manga/impl/PageTest.kt
+5 -3
View File
@@ -15,16 +15,18 @@ import kotlin.test.assertEquals
class PageTest : ApplicationTest() {
@Test
fun testGetPageName() {
val tests = listOf(0, 1, 2, 100)
val tests = listOf(0 to 5, 1 to 5, 2 to 5, 100 to 100, 998 to 1000, 1400 to 1500)
val testResults =
tests.map {
getPageName(it)
tests.map { (page, count) ->
getPageName(page, count)
}
assertEquals(testResults[0], "001")
assertEquals(testResults[1], "002")
assertEquals(testResults[2], "003")
assertEquals(testResults[3], "101")
assertEquals(testResults[4], "0999")
assertEquals(testResults[5], "1401")
}
}
server/src/test/kotlin/suwayomi/tachidesk/manga/impl/update/TestUpdater.kt
+6
View File
@@ -45,10 +45,16 @@ class TestUpdater : IUpdater {
override val status: Flow<UpdateStatus>
get() = TODO("Not yet implemented")
override val updates: Flow<UpdateUpdates>
get() = TODO("Not yet implemented")
override val statusDeprecated: StateFlow<UpdateStatus>
get() = TODO("Not yet implemented")
override fun reset() {
TODO("Not yet implemented")
}
override fun getStatus(): UpdateUpdates {
TODO("Not yet implemented")
}
}