From 5baf54335b3c3551687bd6169f9001c3b418f2fd Mon Sep 17 00:00:00 2001 From: schroda <50052685+schroda@users.noreply.github.com> Date: Tue, 15 Aug 2023 23:51:21 +0200 Subject: [PATCH] Feature/updater provide more info about update (#657) * Provide last global update timestamp * Provide skipped mangas in update status * Extract update status logic into function * Rename update "statusMap" to "mangaStatusMap" * Provide info about categories in update status --- .../graphql/dataLoaders/CategoryDataLoader.kt | 16 +++++++ .../tachidesk/graphql/queries/UpdateQuery.kt | 17 +++---- .../TachideskDataLoaderRegistryFactory.kt | 2 + .../tachidesk/graphql/types/UpdateType.kt | 26 ++++++++--- .../tachidesk/manga/impl/update/IUpdater.kt | 1 + .../tachidesk/manga/impl/update/UpdateJob.kt | 3 +- .../manga/impl/update/UpdateStatus.kt | 17 ++++--- .../tachidesk/manga/impl/update/Updater.kt | 45 +++++++++++++------ .../server/util/WebInterfaceManager.kt | 3 +- 9 files changed, 94 insertions(+), 36 deletions(-) diff --git a/server/src/main/kotlin/suwayomi/tachidesk/graphql/dataLoaders/CategoryDataLoader.kt b/server/src/main/kotlin/suwayomi/tachidesk/graphql/dataLoaders/CategoryDataLoader.kt index e1d0da0b..6b36a709 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/graphql/dataLoaders/CategoryDataLoader.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/graphql/dataLoaders/CategoryDataLoader.kt @@ -36,6 +36,22 @@ class CategoryDataLoader : KotlinDataLoader { } } +class CategoryForIdsDataLoader : KotlinDataLoader, CategoryNodeList> { + override val dataLoaderName = "CategoryForIdsDataLoader" + override fun getDataLoader(): DataLoader, CategoryNodeList> = DataLoaderFactory.newDataLoader { categoryIds -> + future { + transaction { + addLogger(Slf4jSqlDebugLogger) + val ids = categoryIds.flatten().distinct() + val categories = CategoryTable.select { CategoryTable.id inList ids }.map { CategoryType(it) } + categoryIds.map { categoryIds -> + categories.filter { it.id in categoryIds }.toNodeList() + } + } + } + } +} + class CategoriesForMangaDataLoader : KotlinDataLoader { override val dataLoaderName = "CategoriesForMangaDataLoader" override fun getDataLoader(): DataLoader = DataLoaderFactory.newDataLoader { ids -> diff --git a/server/src/main/kotlin/suwayomi/tachidesk/graphql/queries/UpdateQuery.kt b/server/src/main/kotlin/suwayomi/tachidesk/graphql/queries/UpdateQuery.kt index 88b50dd7..344c054b 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/graphql/queries/UpdateQuery.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/graphql/queries/UpdateQuery.kt @@ -4,21 +4,18 @@ import org.kodein.di.DI import org.kodein.di.conf.global import org.kodein.di.instance import suwayomi.tachidesk.graphql.types.UpdateStatus -import suwayomi.tachidesk.graphql.types.UpdateStatusType import suwayomi.tachidesk.manga.impl.update.IUpdater -import suwayomi.tachidesk.manga.impl.update.JobStatus class UpdateQuery { private val updater by DI.global.instance() fun updateStatus(): UpdateStatus { - val status = updater.status.value - return UpdateStatus( - isRunning = status.running, - pendingJobs = UpdateStatusType(status.statusMap[JobStatus.PENDING]?.map { it.id }.orEmpty()), - runningJobs = UpdateStatusType(status.statusMap[JobStatus.RUNNING]?.map { it.id }.orEmpty()), - completeJobs = UpdateStatusType(status.statusMap[JobStatus.COMPLETE]?.map { it.id }.orEmpty()), - failedJobs = UpdateStatusType(status.statusMap[JobStatus.FAILED]?.map { it.id }.orEmpty()) - ) + return UpdateStatus(updater.status.value) + } + + data class LastUpdateTimestampPayload(val timestamp: Long) + + fun lastUpdateTimestamp(): LastUpdateTimestampPayload { + return LastUpdateTimestampPayload(updater.getLastUpdateTimestamp()) } } diff --git a/server/src/main/kotlin/suwayomi/tachidesk/graphql/server/TachideskDataLoaderRegistryFactory.kt b/server/src/main/kotlin/suwayomi/tachidesk/graphql/server/TachideskDataLoaderRegistryFactory.kt index e362ea8e..b4b8ede4 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/graphql/server/TachideskDataLoaderRegistryFactory.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/graphql/server/TachideskDataLoaderRegistryFactory.kt @@ -10,6 +10,7 @@ package suwayomi.tachidesk.graphql.server import com.expediagroup.graphql.dataloader.KotlinDataLoaderRegistryFactory import suwayomi.tachidesk.graphql.dataLoaders.CategoriesForMangaDataLoader import suwayomi.tachidesk.graphql.dataLoaders.CategoryDataLoader +import suwayomi.tachidesk.graphql.dataLoaders.CategoryForIdsDataLoader import suwayomi.tachidesk.graphql.dataLoaders.CategoryMetaDataLoader import suwayomi.tachidesk.graphql.dataLoaders.ChapterDataLoader import suwayomi.tachidesk.graphql.dataLoaders.ChapterMetaDataLoader @@ -39,6 +40,7 @@ class TachideskDataLoaderRegistryFactory { MangaForSourceDataLoader(), MangaForIdsDataLoader(), CategoryDataLoader(), + CategoryForIdsDataLoader(), CategoryMetaDataLoader(), CategoriesForMangaDataLoader(), SourceDataLoader(), diff --git a/server/src/main/kotlin/suwayomi/tachidesk/graphql/types/UpdateType.kt b/server/src/main/kotlin/suwayomi/tachidesk/graphql/types/UpdateType.kt index 713205c0..410089ec 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/graphql/types/UpdateType.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/graphql/types/UpdateType.kt @@ -3,26 +3,42 @@ package suwayomi.tachidesk.graphql.types import com.expediagroup.graphql.generator.annotations.GraphQLIgnore import com.expediagroup.graphql.server.extensions.getValueFromDataLoader import graphql.schema.DataFetchingEnvironment +import suwayomi.tachidesk.manga.impl.update.CategoryUpdateStatus import suwayomi.tachidesk.manga.impl.update.JobStatus import suwayomi.tachidesk.manga.impl.update.UpdateStatus import java.util.concurrent.CompletableFuture class UpdateStatus( val isRunning: Boolean, + val skippedCategories: UpdateStatusCategoryType, + val updatingCategories: UpdateStatusCategoryType, val pendingJobs: UpdateStatusType, val runningJobs: UpdateStatusType, val completeJobs: UpdateStatusType, - val failedJobs: UpdateStatusType + val failedJobs: UpdateStatusType, + val skippedJobs: UpdateStatusType ) { constructor(status: UpdateStatus) : this( isRunning = status.running, - pendingJobs = UpdateStatusType(status.statusMap[JobStatus.PENDING]?.map { it.id }.orEmpty()), - runningJobs = UpdateStatusType(status.statusMap[JobStatus.RUNNING]?.map { it.id }.orEmpty()), - completeJobs = UpdateStatusType(status.statusMap[JobStatus.COMPLETE]?.map { it.id }.orEmpty()), - failedJobs = UpdateStatusType(status.statusMap[JobStatus.FAILED]?.map { it.id }.orEmpty()) + skippedCategories = UpdateStatusCategoryType(status.categoryStatusMap[CategoryUpdateStatus.SKIPPED]?.map { it.id }.orEmpty()), + updatingCategories = UpdateStatusCategoryType(status.categoryStatusMap[CategoryUpdateStatus.UPDATING]?.map { it.id }.orEmpty()), + pendingJobs = UpdateStatusType(status.mangaStatusMap[JobStatus.PENDING]?.map { it.id }.orEmpty()), + runningJobs = UpdateStatusType(status.mangaStatusMap[JobStatus.RUNNING]?.map { it.id }.orEmpty()), + completeJobs = UpdateStatusType(status.mangaStatusMap[JobStatus.COMPLETE]?.map { it.id }.orEmpty()), + failedJobs = UpdateStatusType(status.mangaStatusMap[JobStatus.FAILED]?.map { it.id }.orEmpty()), + skippedJobs = UpdateStatusType(status.mangaStatusMap[JobStatus.SKIPPED]?.map { it.id }.orEmpty()) ) } +class UpdateStatusCategoryType( + @get:GraphQLIgnore + val categoryIds: List +) { + fun categories(dataFetchingEnvironment: DataFetchingEnvironment): CompletableFuture { + return dataFetchingEnvironment.getValueFromDataLoader("CategoryForIdsDataLoader", categoryIds) + } +} + class UpdateStatusType( @get:GraphQLIgnore val mangaIds: List diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/IUpdater.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/IUpdater.kt index 66dd25c1..b804e7f4 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/IUpdater.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/IUpdater.kt @@ -4,6 +4,7 @@ import kotlinx.coroutines.flow.StateFlow import suwayomi.tachidesk.manga.model.dataclass.CategoryDataClass interface IUpdater { + fun getLastUpdateTimestamp(): Long fun addCategoriesToUpdateQueue(categories: List, clear: Boolean?, forceAll: Boolean) val status: StateFlow fun reset() diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/UpdateJob.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/UpdateJob.kt index 928fb98f..ba07836f 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/UpdateJob.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/UpdateJob.kt @@ -6,7 +6,8 @@ enum class JobStatus { PENDING, RUNNING, COMPLETE, - FAILED + FAILED, + SKIPPED } data class UpdateJob( diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/UpdateStatus.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/UpdateStatus.kt index 7f0a0999..3ee1e3ab 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/UpdateStatus.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/UpdateStatus.kt @@ -1,22 +1,27 @@ package suwayomi.tachidesk.manga.impl.update import com.fasterxml.jackson.annotation.JsonIgnore -import mu.KotlinLogging +import suwayomi.tachidesk.manga.model.dataclass.CategoryDataClass import suwayomi.tachidesk.manga.model.dataclass.MangaDataClass -val logger = KotlinLogging.logger {} +enum class CategoryUpdateStatus { + UPDATING, SKIPPED +} + data class UpdateStatus( - val statusMap: Map> = emptyMap(), + val categoryStatusMap: Map> = emptyMap(), + val mangaStatusMap: Map> = emptyMap(), val running: Boolean = false, @JsonIgnore val numberOfJobs: Int = 0 ) { - constructor(jobs: List, running: Boolean) : this( - statusMap = jobs.groupBy { it.status } + constructor(categories: Map>, jobs: List, skippedMangas: List, running: Boolean) : this( + categories, + mangaStatusMap = jobs.groupBy { it.status } .mapValues { entry -> entry.value.map { it.manga } - }, + }.plus(Pair(JobStatus.SKIPPED, skippedMangas)), running = running, numberOfJobs = jobs.size ) diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/Updater.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/Updater.kt index 6883267a..4da90d97 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/Updater.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/Updater.kt @@ -18,9 +18,6 @@ import kotlinx.coroutines.launch import kotlinx.coroutines.sync.Semaphore import kotlinx.coroutines.sync.withPermit import mu.KotlinLogging -import org.kodein.di.DI -import org.kodein.di.conf.global -import org.kodein.di.instance import suwayomi.tachidesk.manga.impl.Category import suwayomi.tachidesk.manga.impl.CategoryManga import suwayomi.tachidesk.manga.impl.Chapter @@ -49,6 +46,7 @@ class Updater : IUpdater { private var maxSourcesInParallel = 20 // max permits, necessary to be set to be able to release up to 20 permits private val semaphore = Semaphore(maxSourcesInParallel) + private val lastUpdateKey = "lastUpdateKey" private val lastAutomatedUpdateKey = "lastAutomatedUpdateKey" private val preferences = Preferences.userNodeForPackage(Updater::class.java) @@ -76,6 +74,10 @@ class Updater : IUpdater { ) } + override fun getLastUpdateTimestamp(): Long { + return preferences.getLong(lastUpdateKey, 0) + } + private fun autoUpdateTask() { val lastAutomatedUpdate = preferences.getLong(lastAutomatedUpdateKey, 0) preferences.putLong(lastAutomatedUpdateKey, System.currentTimeMillis()) @@ -109,6 +111,15 @@ class Updater : IUpdater { HAScheduler.schedule(::autoUpdateTask, updateInterval, timeToNextExecution, "global-update") } + /** + * Updates the status and sustains the "skippedMangas" + */ + private fun updateStatus(jobs: List, running: Boolean, categories: Map>? = null, skippedMangas: List? = null) { + val updateStatusCategories = categories ?: _status.value.categoryStatusMap + val tmpSkippedMangas = skippedMangas ?: _status.value.mangaStatusMap[JobStatus.SKIPPED] ?: emptyList() + _status.update { UpdateStatus(updateStatusCategories, jobs, tmpSkippedMangas, running) } + } + private fun getOrCreateUpdateChannelFor(source: String): Channel { return updateChannels.getOrPut(source) { logger.debug { "getOrCreateUpdateChannelFor: created channel for $source - channels: ${updateChannels.size + 1}" } @@ -121,7 +132,7 @@ class Updater : IUpdater { channel.consumeAsFlow() .onEach { job -> semaphore.withPermit { - _status.value = UpdateStatus( + updateStatus( process(job), tracker.any { (_, job) -> job.status == JobStatus.PENDING || job.status == JobStatus.RUNNING @@ -136,7 +147,7 @@ class Updater : IUpdater { private suspend fun process(job: UpdateJob): List { tracker[job.manga.id] = job.copy(status = JobStatus.RUNNING) - _status.update { UpdateStatus(tracker.values.toList(), true) } + updateStatus(tracker.values.toList(), true) tracker[job.manga.id] = try { logger.info { "Updating \"${job.manga.title}\" (source: ${job.manga.sourceId})" } Chapter.getChapterList(job.manga.id, true) @@ -150,9 +161,10 @@ class Updater : IUpdater { } override fun addCategoriesToUpdateQueue(categories: List, clear: Boolean?, forceAll: Boolean) { - val updater by DI.global.instance() + preferences.putLong(lastUpdateKey, System.currentTimeMillis()) + if (clear == true) { - updater.reset() + reset() } val includeInUpdateStatusToCategoryMap = categories.groupBy { it.includeInUpdate } @@ -164,6 +176,11 @@ class Updater : IUpdater { } else { includedCategories.ifEmpty { unsetCategories } } + val skippedCategories = categories.subtract(categoriesToUpdate.toSet()).toList() + val updateStatusCategories = mapOf( + Pair(CategoryUpdateStatus.UPDATING, categoriesToUpdate), + Pair(CategoryUpdateStatus.SKIPPED, skippedCategories) + ) logger.debug { "Updating categories: '${categoriesToUpdate.joinToString("', '") { it.name }}'" } @@ -179,10 +196,12 @@ class Updater : IUpdater { .filter { if (serverConfig.excludeCompleted.value) { it.status != MangaStatus.COMPLETED.name } else true } .filter { forceAll || !excludedCategories.any { category -> mangasToCategoriesMap[it.id]?.contains(category) == true } } .toList() + val skippedMangas = categoriesToUpdateMangas.subtract(mangasToUpdate.toSet()).toList() // In case no manga gets updated and no update job was running before, the client would never receive an info about its update request + updateStatus(emptyList(), mangasToUpdate.isNotEmpty(), updateStatusCategories, skippedMangas) + if (mangasToUpdate.isEmpty()) { - UpdaterSocket.notifyAllClients(UpdateStatus()) return } @@ -192,10 +211,10 @@ class Updater : IUpdater { ) } - private fun addMangasToQueue(mangas: List) { - mangas.forEach { tracker[it.id] = UpdateJob(it) } - _status.update { UpdateStatus(tracker.values.toList(), mangas.isNotEmpty()) } - mangas.forEach { addMangaToQueue(it) } + private fun addMangasToQueue(mangasToUpdate: List) { + mangasToUpdate.forEach { tracker[it.id] = UpdateJob(it) } + updateStatus(tracker.values.toList(), mangasToUpdate.isNotEmpty()) + mangasToUpdate.forEach { addMangaToQueue(it) } } private fun addMangaToQueue(manga: MangaDataClass) { @@ -208,7 +227,7 @@ class Updater : IUpdater { override fun reset() { scope.coroutineContext.cancelChildren() tracker.clear() - _status.update { UpdateStatus() } + updateStatus(emptyList(), false) updateChannels.forEach { (_, channel) -> channel.cancel() } updateChannels.clear() } diff --git a/server/src/main/kotlin/suwayomi/tachidesk/server/util/WebInterfaceManager.kt b/server/src/main/kotlin/suwayomi/tachidesk/server/util/WebInterfaceManager.kt index 3863a8e4..0b3dfde7 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/server/util/WebInterfaceManager.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/server/util/WebInterfaceManager.kt @@ -87,7 +87,8 @@ enum class WebUIChannel { } enum class WebUIFlavor( - val uiName: String, val repoUrl: String, + val uiName: String, + val repoUrl: String, val versionMappingUrl: String, val latestReleaseInfoUrl: String, val baseFileName: String