add batch download api (#436)

* Add POST /downloads endpoint for creating multiple * Fix review notes * Add chapter id to API endpoints * Rewrite batch chapter download to use chapter id instead of mangaId+chapterIndex combination * Change EnqueueInput format to be more futureproof * Change endpoint path * Change endpoint path
2022-11-06 22:32:18 +01:00
parent 8b20e2b48f
commit 2ac5c1362c
6 changed files with 118 additions and 22 deletions
@@ -110,6 +110,7 @@ object MangaAPI {
        path("download") {
            get("{mangaId}/chapter/{chapterIndex}", DownloadController.queueChapter)
            delete("{mangaId}/chapter/{chapterIndex}", DownloadController.unqueueChapter)
+            post("batch", DownloadController.queueChapters)
        }

        path("update") {
@@ -9,13 +9,21 @@ package suwayomi.tachidesk.manga.controller

 import io.javalin.http.HttpCode
 import io.javalin.websocket.WsConfig
+import kotlinx.serialization.decodeFromString
+import kotlinx.serialization.json.Json
+import org.kodein.di.DI
+import org.kodein.di.conf.global
+import org.kodein.di.instance
 import suwayomi.tachidesk.manga.impl.download.DownloadManager
+import suwayomi.tachidesk.manga.impl.download.DownloadManager.EnqueueInput
 import suwayomi.tachidesk.server.JavalinSetup.future
 import suwayomi.tachidesk.server.util.handler
 import suwayomi.tachidesk.server.util.pathParam
 import suwayomi.tachidesk.server.util.withOperation

 object DownloadController {
+    private val json by DI.global.instance<Json>()
+
    /** Download queue stats */
    fun downloadsWS(ws: WsConfig) {
        ws.onConnect { ctx ->
@@ -84,20 +92,20 @@ object DownloadController {
        }
    )

-    /** Queue chapter for download */
+    /** Queue single chapter for download */
    val queueChapter = handler(
        pathParam<Int>("chapterIndex"),
        pathParam<Int>("mangaId"),
        documentWith = {
            withOperation {
-                summary("Downloader add chapter")
-                description("Queue chapter for download")
+                summary("Downloader add single chapter")
+                description("Queue single chapter for download")
            }
        },
        behaviorOf = { ctx, chapterIndex, mangaId ->
            ctx.future(
                future {
-                    DownloadManager.enqueue(chapterIndex, mangaId)
+                    DownloadManager.enqueueWithChapterIndex(mangaId, chapterIndex)
                }
            )
        },
@@ -107,6 +115,27 @@ object DownloadController {
        }
    )

+    val queueChapters = handler(
+        documentWith = {
+            withOperation {
+                summary("Downloader add multiple chapters")
+                description("Queue multiple chapters for download")
+            }
+            body<EnqueueInput>()
+        },
+        behaviorOf = { ctx ->
+            val inputs = json.decodeFromString<EnqueueInput>(ctx.body())
+            ctx.future(
+                future {
+                    DownloadManager.enqueue(inputs)
+                }
+            )
+        },
+        withResults = {
+            httpCode(HttpCode.OK)
+        }
+    )
+
    /** delete chapter from download queue */
    val unqueueChapter = handler(
        pathParam<Int>("chapterIndex"),
@@ -140,6 +140,7 @@ object Chapter {
            val dbChapter = dbChapterMap.getValue(it.url)

            ChapterDataClass(
+                dbChapter[ChapterTable.id].value,
                it.url,
                it.name,
                it.date_upload,
@@ -9,18 +9,24 @@ package suwayomi.tachidesk.manga.impl.download

 import io.javalin.websocket.WsContext
 import io.javalin.websocket.WsMessageContext
+import kotlinx.serialization.Serializable
+import mu.KotlinLogging
 import org.jetbrains.exposed.sql.and
 import org.jetbrains.exposed.sql.select
 import org.jetbrains.exposed.sql.transactions.transaction
-import suwayomi.tachidesk.manga.impl.Manga.getManga
 import suwayomi.tachidesk.manga.impl.download.model.DownloadChapter
 import suwayomi.tachidesk.manga.impl.download.model.DownloadState.Downloading
 import suwayomi.tachidesk.manga.impl.download.model.DownloadStatus
+import suwayomi.tachidesk.manga.model.dataclass.ChapterDataClass
+import suwayomi.tachidesk.manga.model.dataclass.MangaDataClass
 import suwayomi.tachidesk.manga.model.table.ChapterTable
+import suwayomi.tachidesk.manga.model.table.MangaTable
 import suwayomi.tachidesk.manga.model.table.toDataClass
 import java.util.concurrent.ConcurrentHashMap
 import java.util.concurrent.CopyOnWriteArrayList

+private val logger = KotlinLogging.logger {}
+
 object DownloadManager {
    private val clients = ConcurrentHashMap<String, WsContext>()
    private val downloadQueue = CopyOnWriteArrayList<DownloadChapter>()
@@ -75,24 +81,81 @@ object DownloadManager {
        )
    }

-    suspend fun enqueue(chapterIndex: Int, mangaId: Int) {
-        if (downloadQueue.none { it.mangaId == mangaId && it.chapterIndex == chapterIndex }) {
-            downloadQueue.add(
-                DownloadChapter(
-                    chapterIndex,
-                    mangaId,
-                    chapter = ChapterTable.toDataClass(
-                        transaction {
-                            ChapterTable.select { (ChapterTable.manga eq mangaId) and (ChapterTable.sourceOrder eq chapterIndex) }
-                                .first()
-                        }
-                    ),
-                    manga = getManga(mangaId)
-                )
-            )
-            start()
+    fun enqueueWithChapterIndex(mangaId: Int, chapterIndex: Int) {
+        val chapter = transaction {
+            ChapterTable
+                .slice(ChapterTable.id)
+                .select { ChapterTable.manga.eq(mangaId) and ChapterTable.sourceOrder.eq(chapterIndex) }
+                .first()
        }
-        notifyAllClients()
+        enqueue(EnqueueInput(chapterIds = listOf(chapter[ChapterTable.id].value)))
+    }
+
+    @Serializable
+    // Input might have additional formats in the future, such as "All for mangaID" or "Unread for categoryID"
+    // Having this input format is just future-proofing
+    data class EnqueueInput(
+        val chapterIds: List<Int>?
+    )
+
+    fun enqueue(input: EnqueueInput) {
+        if (input.chapterIds == null) return
+
+        val chapters = transaction {
+            (ChapterTable innerJoin MangaTable)
+                .select { ChapterTable.id inList input.chapterIds }
+                .toList()
+        }
+
+        val mangas = transaction {
+            chapters.distinctBy { chapter -> chapter[MangaTable.id] }
+                .map { MangaTable.toDataClass(it) }
+                .associateBy { it.id }
+        }
+
+        val inputPairs = transaction {
+            chapters.map {
+                Pair(
+                    // this should be safe because mangas is created above from chapters
+                    mangas[it[ChapterTable.manga].value]!!,
+                    ChapterTable.toDataClass(it)
+                )
+            }
+        }
+
+        addMultipleToQueue(inputPairs)
+    }
+
+    /**
+     * Tries to add multiple inputs to queue
+     * If any of inputs was actually added to queue, starts the queue
+     */
+    private fun addMultipleToQueue(inputs: List<Pair<MangaDataClass, ChapterDataClass>>) {
+        val addedChapters = inputs.mapNotNull { addToQueue(it.first, it.second) }
+        if (addedChapters.isNotEmpty()) {
+            start()
+            notifyAllClients()
+        }
+    }
+
+    /**
+     * Tries to add chapter to queue.
+     * If chapter is added, returns the created DownloadChapter, otherwise returns null
+     */
+    private fun addToQueue(manga: MangaDataClass, chapter: ChapterDataClass): DownloadChapter? {
+        if (downloadQueue.none { it.mangaId == manga.id && it.chapterIndex == chapter.index }) {
+            val downloadChapter = DownloadChapter(
+                chapter.index,
+                manga.id,
+                chapter,
+                manga
+            )
+            downloadQueue.add(downloadChapter)
+            logger.debug { "Added chapter ${chapter.id} to download queue (${manga.title} | ${chapter.name})" }
+            return downloadChapter
+        }
+        logger.debug { "Chapter ${chapter.id} already present in queue (${manga.title} | ${chapter.name})" }
+        return null
    }

    fun unqueue(chapterIndex: Int, mangaId: Int) {
@@ -8,6 +8,7 @@ package suwayomi.tachidesk.manga.model.dataclass
 * file, You can obtain one at https://mozilla.org/MPL/2.0/. */

 data class ChapterDataClass(
+    val id: Int,
    val url: String,
    val name: String,
    val uploadDate: Long,
@@ -38,6 +38,7 @@ object ChapterTable : IntIdTable() {

 fun ChapterTable.toDataClass(chapterEntry: ResultRow) =
    ChapterDataClass(
+        chapterEntry[id].value,
        chapterEntry[url],
        chapterEntry[name],
        chapterEntry[date_upload],