admin/Suwayomi-Server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Feature/updater provide more info about update (#657)

Browse Source 
* Provide last global update timestamp

* Provide skipped mangas in update status

* Extract update status logic into function

* Rename update "statusMap" to "mangaStatusMap"

* Provide info about categories in update status
This commit is contained in:
schroda
2023-08-15 23:51:21 +02:00
committed by GitHub
parent d9019b8f46
commit 5baf54335b
9 changed files with 94 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/src/main/kotlin/suwayomi/tachidesk/graphql/dataLoaders/CategoryDataLoader.kt
+16
View File
@@ -36,6 +36,22 @@ class CategoryDataLoader : KotlinDataLoader<Int, CategoryType> {
}
}
class CategoryForIdsDataLoader : KotlinDataLoader<List<Int>, CategoryNodeList> {
override val dataLoaderName = "CategoryForIdsDataLoader"
override fun getDataLoader(): DataLoader<List<Int>, CategoryNodeList> = DataLoaderFactory.newDataLoader { categoryIds ->
future {
transaction {
addLogger(Slf4jSqlDebugLogger)
val ids = categoryIds.flatten().distinct()
val categories = CategoryTable.select { CategoryTable.id inList ids }.map { CategoryType(it) }
categoryIds.map { categoryIds ->
categories.filter { it.id in categoryIds }.toNodeList()
}
}
}
}
}
class CategoriesForMangaDataLoader : KotlinDataLoader<Int, CategoryNodeList> {
override val dataLoaderName = "CategoriesForMangaDataLoader"
override fun getDataLoader(): DataLoader<Int, CategoryNodeList> = DataLoaderFactory.newDataLoader<Int, CategoryNodeList> { ids ->
server/src/main/kotlin/suwayomi/tachidesk/graphql/queries/UpdateQuery.kt
+7 -10
View File
@@ -4,21 +4,18 @@ import org.kodein.di.DI
import org.kodein.di.conf.global
import org.kodein.di.instance
import suwayomi.tachidesk.graphql.types.UpdateStatus
import suwayomi.tachidesk.graphql.types.UpdateStatusType
import suwayomi.tachidesk.manga.impl.update.IUpdater
import suwayomi.tachidesk.manga.impl.update.JobStatus
class UpdateQuery {
private val updater by DI.global.instance<IUpdater>()
fun updateStatus(): UpdateStatus {
val status = updater.status.value
return UpdateStatus(
isRunning = status.running,
pendingJobs = UpdateStatusType(status.statusMap[JobStatus.PENDING]?.map { it.id }.orEmpty()),
runningJobs = UpdateStatusType(status.statusMap[JobStatus.RUNNING]?.map { it.id }.orEmpty()),
completeJobs = UpdateStatusType(status.statusMap[JobStatus.COMPLETE]?.map { it.id }.orEmpty()),
failedJobs = UpdateStatusType(status.statusMap[JobStatus.FAILED]?.map { it.id }.orEmpty())
)
return UpdateStatus(updater.status.value)
}
data class LastUpdateTimestampPayload(val timestamp: Long)
fun lastUpdateTimestamp(): LastUpdateTimestampPayload {
return LastUpdateTimestampPayload(updater.getLastUpdateTimestamp())
}
}
server/src/main/kotlin/suwayomi/tachidesk/graphql/server/TachideskDataLoaderRegistryFactory.kt
+2
View File
@@ -10,6 +10,7 @@ package suwayomi.tachidesk.graphql.server
import com.expediagroup.graphql.dataloader.KotlinDataLoaderRegistryFactory
import suwayomi.tachidesk.graphql.dataLoaders.CategoriesForMangaDataLoader
import suwayomi.tachidesk.graphql.dataLoaders.CategoryDataLoader
import suwayomi.tachidesk.graphql.dataLoaders.CategoryForIdsDataLoader
import suwayomi.tachidesk.graphql.dataLoaders.CategoryMetaDataLoader
import suwayomi.tachidesk.graphql.dataLoaders.ChapterDataLoader
import suwayomi.tachidesk.graphql.dataLoaders.ChapterMetaDataLoader
@@ -39,6 +40,7 @@ class TachideskDataLoaderRegistryFactory {
MangaForSourceDataLoader(),
MangaForIdsDataLoader(),
CategoryDataLoader(),
CategoryForIdsDataLoader(),
CategoryMetaDataLoader(),
CategoriesForMangaDataLoader(),
SourceDataLoader(),
server/src/main/kotlin/suwayomi/tachidesk/graphql/types/UpdateType.kt
+21 -5
View File
@@ -3,26 +3,42 @@ package suwayomi.tachidesk.graphql.types
import com.expediagroup.graphql.generator.annotations.GraphQLIgnore
import com.expediagroup.graphql.server.extensions.getValueFromDataLoader
import graphql.schema.DataFetchingEnvironment
import suwayomi.tachidesk.manga.impl.update.CategoryUpdateStatus
import suwayomi.tachidesk.manga.impl.update.JobStatus
import suwayomi.tachidesk.manga.impl.update.UpdateStatus
import java.util.concurrent.CompletableFuture
class UpdateStatus(
val isRunning: Boolean,
val skippedCategories: UpdateStatusCategoryType,
val updatingCategories: UpdateStatusCategoryType,
val pendingJobs: UpdateStatusType,
val runningJobs: UpdateStatusType,
val completeJobs: UpdateStatusType,
val failedJobs: UpdateStatusType
val failedJobs: UpdateStatusType,
val skippedJobs: UpdateStatusType
) {
constructor(status: UpdateStatus) : this(
isRunning = status.running,
pendingJobs = UpdateStatusType(status.statusMap[JobStatus.PENDING]?.map { it.id }.orEmpty()),
runningJobs = UpdateStatusType(status.statusMap[JobStatus.RUNNING]?.map { it.id }.orEmpty()),
completeJobs = UpdateStatusType(status.statusMap[JobStatus.COMPLETE]?.map { it.id }.orEmpty()),
failedJobs = UpdateStatusType(status.statusMap[JobStatus.FAILED]?.map { it.id }.orEmpty())
skippedCategories = UpdateStatusCategoryType(status.categoryStatusMap[CategoryUpdateStatus.SKIPPED]?.map { it.id }.orEmpty()),
updatingCategories = UpdateStatusCategoryType(status.categoryStatusMap[CategoryUpdateStatus.UPDATING]?.map { it.id }.orEmpty()),
pendingJobs = UpdateStatusType(status.mangaStatusMap[JobStatus.PENDING]?.map { it.id }.orEmpty()),
runningJobs = UpdateStatusType(status.mangaStatusMap[JobStatus.RUNNING]?.map { it.id }.orEmpty()),
completeJobs = UpdateStatusType(status.mangaStatusMap[JobStatus.COMPLETE]?.map { it.id }.orEmpty()),
failedJobs = UpdateStatusType(status.mangaStatusMap[JobStatus.FAILED]?.map { it.id }.orEmpty()),
skippedJobs = UpdateStatusType(status.mangaStatusMap[JobStatus.SKIPPED]?.map { it.id }.orEmpty())
)
}
class UpdateStatusCategoryType(
@get:GraphQLIgnore
val categoryIds: List<Int>
) {
fun categories(dataFetchingEnvironment: DataFetchingEnvironment): CompletableFuture<CategoryNodeList> {
return dataFetchingEnvironment.getValueFromDataLoader("CategoryForIdsDataLoader", categoryIds)
}
}
class UpdateStatusType(
@get:GraphQLIgnore
val mangaIds: List<Int>
server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/IUpdater.kt
+1
View File
@@ -4,6 +4,7 @@ import kotlinx.coroutines.flow.StateFlow
import suwayomi.tachidesk.manga.model.dataclass.CategoryDataClass
interface IUpdater {
fun getLastUpdateTimestamp(): Long
fun addCategoriesToUpdateQueue(categories: List<CategoryDataClass>, clear: Boolean?, forceAll: Boolean)
val status: StateFlow<UpdateStatus>
fun reset()
server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/UpdateJob.kt
+2 -1
View File
@@ -6,7 +6,8 @@ enum class JobStatus {
PENDING,
RUNNING,
COMPLETE,
FAILED
FAILED,
SKIPPED
}
data class UpdateJob(
server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/UpdateStatus.kt
+11 -6
View File
@@ -1,22 +1,27 @@
package suwayomi.tachidesk.manga.impl.update
import com.fasterxml.jackson.annotation.JsonIgnore
import mu.KotlinLogging
import suwayomi.tachidesk.manga.model.dataclass.CategoryDataClass
import suwayomi.tachidesk.manga.model.dataclass.MangaDataClass
val logger = KotlinLogging.logger {}
enum class CategoryUpdateStatus {
UPDATING, SKIPPED
}
data class UpdateStatus(
val statusMap: Map<JobStatus, List<MangaDataClass>> = emptyMap(),
val categoryStatusMap: Map<CategoryUpdateStatus, List<CategoryDataClass>> = emptyMap(),
val mangaStatusMap: Map<JobStatus, List<MangaDataClass>> = emptyMap(),
val running: Boolean = false,
@JsonIgnore
val numberOfJobs: Int = 0
) {
constructor(jobs: List<UpdateJob>, running: Boolean) : this(
statusMap = jobs.groupBy { it.status }
constructor(categories: Map<CategoryUpdateStatus, List<CategoryDataClass>>, jobs: List<UpdateJob>, skippedMangas: List<MangaDataClass>, running: Boolean) : this(
categories,
mangaStatusMap = jobs.groupBy { it.status }
.mapValues { entry ->
entry.value.map { it.manga }
},
}.plus(Pair(JobStatus.SKIPPED, skippedMangas)),
running = running,
numberOfJobs = jobs.size
)
server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/Updater.kt
+32 -13
View File
@@ -18,9 +18,6 @@ import kotlinx.coroutines.launch
import kotlinx.coroutines.sync.Semaphore
import kotlinx.coroutines.sync.withPermit
import mu.KotlinLogging
import org.kodein.di.DI
import org.kodein.di.conf.global
import org.kodein.di.instance
import suwayomi.tachidesk.manga.impl.Category
import suwayomi.tachidesk.manga.impl.CategoryManga
import suwayomi.tachidesk.manga.impl.Chapter
@@ -49,6 +46,7 @@ class Updater : IUpdater {
private var maxSourcesInParallel = 20 // max permits, necessary to be set to be able to release up to 20 permits
private val semaphore = Semaphore(maxSourcesInParallel)
private val lastUpdateKey = "lastUpdateKey"
private val lastAutomatedUpdateKey = "lastAutomatedUpdateKey"
private val preferences = Preferences.userNodeForPackage(Updater::class.java)
@@ -76,6 +74,10 @@ class Updater : IUpdater {
)
}
override fun getLastUpdateTimestamp(): Long {
return preferences.getLong(lastUpdateKey, 0)
}
private fun autoUpdateTask() {
val lastAutomatedUpdate = preferences.getLong(lastAutomatedUpdateKey, 0)
preferences.putLong(lastAutomatedUpdateKey, System.currentTimeMillis())
@@ -109,6 +111,15 @@ class Updater : IUpdater {
HAScheduler.schedule(::autoUpdateTask, updateInterval, timeToNextExecution, "global-update")
}
/**
* Updates the status and sustains the "skippedMangas"
*/
private fun updateStatus(jobs: List<UpdateJob>, running: Boolean, categories: Map<CategoryUpdateStatus, List<CategoryDataClass>>? = null, skippedMangas: List<MangaDataClass>? = null) {
val updateStatusCategories = categories ?: _status.value.categoryStatusMap
val tmpSkippedMangas = skippedMangas ?: _status.value.mangaStatusMap[JobStatus.SKIPPED] ?: emptyList()
_status.update { UpdateStatus(updateStatusCategories, jobs, tmpSkippedMangas, running) }
}
private fun getOrCreateUpdateChannelFor(source: String): Channel<UpdateJob> {
return updateChannels.getOrPut(source) {
logger.debug { "getOrCreateUpdateChannelFor: created channel for $source - channels: ${updateChannels.size + 1}" }
@@ -121,7 +132,7 @@ class Updater : IUpdater {
channel.consumeAsFlow()
.onEach { job ->
semaphore.withPermit {
_status.value = UpdateStatus(
updateStatus(
process(job),
tracker.any { (_, job) ->
job.status == JobStatus.PENDING || job.status == JobStatus.RUNNING
@@ -136,7 +147,7 @@ class Updater : IUpdater {
private suspend fun process(job: UpdateJob): List<UpdateJob> {
tracker[job.manga.id] = job.copy(status = JobStatus.RUNNING)
_status.update { UpdateStatus(tracker.values.toList(), true) }
updateStatus(tracker.values.toList(), true)
tracker[job.manga.id] = try {
logger.info { "Updating \"${job.manga.title}\" (source: ${job.manga.sourceId})" }
Chapter.getChapterList(job.manga.id, true)
@@ -150,9 +161,10 @@ class Updater : IUpdater {
}
override fun addCategoriesToUpdateQueue(categories: List<CategoryDataClass>, clear: Boolean?, forceAll: Boolean) {
val updater by DI.global.instance<IUpdater>()
preferences.putLong(lastUpdateKey, System.currentTimeMillis())
if (clear == true) {
updater.reset()
reset()
}
val includeInUpdateStatusToCategoryMap = categories.groupBy { it.includeInUpdate }
@@ -164,6 +176,11 @@ class Updater : IUpdater {
} else {
includedCategories.ifEmpty { unsetCategories }
}
val skippedCategories = categories.subtract(categoriesToUpdate.toSet()).toList()
val updateStatusCategories = mapOf(
Pair(CategoryUpdateStatus.UPDATING, categoriesToUpdate),
Pair(CategoryUpdateStatus.SKIPPED, skippedCategories)
)
logger.debug { "Updating categories: '${categoriesToUpdate.joinToString("', '") { it.name }}'" }
@@ -179,10 +196,12 @@ class Updater : IUpdater {
.filter { if (serverConfig.excludeCompleted.value) { it.status != MangaStatus.COMPLETED.name } else true }
.filter { forceAll || !excludedCategories.any { category -> mangasToCategoriesMap[it.id]?.contains(category) == true } }
.toList()
val skippedMangas = categoriesToUpdateMangas.subtract(mangasToUpdate.toSet()).toList()
// In case no manga gets updated and no update job was running before, the client would never receive an info about its update request
updateStatus(emptyList(), mangasToUpdate.isNotEmpty(), updateStatusCategories, skippedMangas)
if (mangasToUpdate.isEmpty()) {
UpdaterSocket.notifyAllClients(UpdateStatus())
return
}
@@ -192,10 +211,10 @@ class Updater : IUpdater {
)
}
private fun addMangasToQueue(mangas: List<MangaDataClass>) {
mangas.forEach { tracker[it.id] = UpdateJob(it) }
_status.update { UpdateStatus(tracker.values.toList(), mangas.isNotEmpty()) }
mangas.forEach { addMangaToQueue(it) }
private fun addMangasToQueue(mangasToUpdate: List<MangaDataClass>) {
mangasToUpdate.forEach { tracker[it.id] = UpdateJob(it) }
updateStatus(tracker.values.toList(), mangasToUpdate.isNotEmpty())
mangasToUpdate.forEach { addMangaToQueue(it) }
}
private fun addMangaToQueue(manga: MangaDataClass) {
@@ -208,7 +227,7 @@ class Updater : IUpdater {
override fun reset() {
scope.coroutineContext.cancelChildren()
tracker.clear()
_status.update { UpdateStatus() }
updateStatus(emptyList(), false)
updateChannels.forEach { (_, channel) -> channel.cancel() }
updateChannels.clear()
}
server/src/main/kotlin/suwayomi/tachidesk/server/util/WebInterfaceManager.kt
+2 -1
View File
@@ -87,7 +87,8 @@ enum class WebUIChannel {
}
enum class WebUIFlavor(
val uiName: String, val repoUrl: String,
val uiName: String,
val repoUrl: String,
val versionMappingUrl: String,
val latestReleaseInfoUrl: String,
val baseFileName: String