Feature/cleanup backup logic (#1701)

* Extract global metadata backup logic into BackupGlobalMetaHandler

* Extract category backup logic into BackupCategoryHandler

* Extract source backup logic into BackupSourceHandler

* Extract manga backup logic into BackupMangaHandler
This commit is contained in:
schroda
2025-10-15 01:37:29 +02:00
committed by GitHub
parent b802c6977a
commit 8d119fd710
6 changed files with 592 additions and 532 deletions
@@ -9,7 +9,6 @@ package suwayomi.tachidesk.manga.impl.backup.proto
import android.app.Application
import android.content.Context
import eu.kanade.tachiyomi.source.model.UpdateStrategy
import io.github.oshai.kotlinlogging.KotlinLogging
import kotlinx.coroutines.DelicateCoroutinesApi
import kotlinx.coroutines.Dispatchers
@@ -20,33 +19,14 @@ import okio.Buffer
import okio.Sink
import okio.buffer
import okio.gzip
import org.jetbrains.exposed.sql.Query
import org.jetbrains.exposed.sql.ResultRow
import org.jetbrains.exposed.sql.SortOrder
import org.jetbrains.exposed.sql.selectAll
import org.jetbrains.exposed.sql.transactions.transaction
import suwayomi.tachidesk.global.impl.GlobalMeta
import suwayomi.tachidesk.manga.impl.Category
import suwayomi.tachidesk.manga.impl.CategoryManga
import suwayomi.tachidesk.manga.impl.Chapter
import suwayomi.tachidesk.manga.impl.Manga
import suwayomi.tachidesk.manga.impl.Source
import suwayomi.tachidesk.manga.impl.backup.BackupFlags
import suwayomi.tachidesk.manga.impl.backup.proto.handlers.BackupCategoryHandler
import suwayomi.tachidesk.manga.impl.backup.proto.handlers.BackupGlobalMetaHandler
import suwayomi.tachidesk.manga.impl.backup.proto.handlers.BackupMangaHandler
import suwayomi.tachidesk.manga.impl.backup.proto.handlers.BackupSettingsHandler
import suwayomi.tachidesk.manga.impl.backup.proto.handlers.BackupSourceHandler
import suwayomi.tachidesk.manga.impl.backup.proto.models.Backup
import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupCategory
import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupChapter
import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupHistory
import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupManga
import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupSource
import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupTracking
import suwayomi.tachidesk.manga.impl.track.Track
import suwayomi.tachidesk.manga.model.table.CategoryTable
import suwayomi.tachidesk.manga.model.table.ChapterTable
import suwayomi.tachidesk.manga.model.table.MangaStatus
import suwayomi.tachidesk.manga.model.table.MangaTable
import suwayomi.tachidesk.manga.model.table.SourceTable
import suwayomi.tachidesk.manga.model.table.toDataClass
import suwayomi.tachidesk.server.ApplicationDirs
import suwayomi.tachidesk.server.serverConfig
import suwayomi.tachidesk.util.HAScheduler
@@ -56,7 +36,6 @@ import uy.kohesive.injekt.injectLazy
import java.io.File
import java.io.InputStream
import kotlin.time.Duration.Companion.days
import kotlin.time.Duration.Companion.seconds
object ProtoBackupExport : ProtoBackupBase() {
private val logger = KotlinLogging.logger { }
@@ -170,20 +149,15 @@ object ProtoBackupExport : ProtoBackupBase() {
fun createBackup(flags: BackupFlags): InputStream {
// Create root object
val databaseManga =
if (flags.includeManga) {
transaction { MangaTable.selectAll().where { MangaTable.inLibrary eq true }.toList() }
} else {
emptyList()
}
val backupMangas = BackupMangaHandler.backup(flags)
val backup: Backup =
transaction {
Backup(
backupManga(databaseManga, flags),
backupCategories(flags),
backupExtensionInfo(databaseManga, flags),
backupGlobalMeta(flags),
BackupMangaHandler.backup(flags),
BackupCategoryHandler.backup(flags),
BackupSourceHandler.backup(backupMangas, flags),
BackupGlobalMetaHandler.backup(flags),
BackupSettingsHandler.backup(flags),
)
}
@@ -198,171 +172,4 @@ object ProtoBackupExport : ProtoBackupBase() {
return byteStream.inputStream()
}
private fun backupManga(
databaseManga: List<ResultRow>,
flags: BackupFlags,
): List<BackupManga> =
databaseManga.map { mangaRow ->
val backupManga =
BackupManga(
source = mangaRow[MangaTable.sourceReference],
url = mangaRow[MangaTable.url],
title = mangaRow[MangaTable.title],
artist = mangaRow[MangaTable.artist],
author = mangaRow[MangaTable.author],
description = mangaRow[MangaTable.description],
genre = mangaRow[MangaTable.genre]?.split(", ") ?: emptyList(),
status = MangaStatus.valueOf(mangaRow[MangaTable.status]).value,
thumbnailUrl = mangaRow[MangaTable.thumbnail_url],
dateAdded = mangaRow[MangaTable.inLibraryAt].seconds.inWholeMilliseconds,
viewer = 0, // not supported in Tachidesk
updateStrategy = UpdateStrategy.valueOf(mangaRow[MangaTable.updateStrategy]),
)
val mangaId = mangaRow[MangaTable.id].value
if (flags.includeClientData) {
backupManga.meta = Manga.getMangaMetaMap(mangaId)
}
if (flags.includeChapters || flags.includeHistory) {
val chapters =
transaction {
ChapterTable
.selectAll()
.where { ChapterTable.manga eq mangaId }
.orderBy(ChapterTable.sourceOrder to SortOrder.DESC)
.map {
ChapterTable.toDataClass(it)
}
}
if (flags.includeChapters) {
val chapterToMeta = Chapter.getChaptersMetaMaps(chapters.map { it.id })
backupManga.chapters =
chapters.map {
BackupChapter(
it.url,
it.name,
it.scanlator,
it.read,
it.bookmarked,
it.lastPageRead,
it.fetchedAt.seconds.inWholeMilliseconds,
it.uploadDate,
it.chapterNumber,
chapters.size - it.index,
).apply {
if (flags.includeClientData) {
this.meta = chapterToMeta[it.id] ?: emptyMap()
}
}
}
}
if (flags.includeHistory) {
backupManga.history =
chapters.mapNotNull {
if (it.lastReadAt > 0) {
BackupHistory(
url = it.url,
lastRead = it.lastReadAt.seconds.inWholeMilliseconds,
)
} else {
null
}
}
}
}
if (flags.includeCategories) {
backupManga.categories = CategoryManga.getMangaCategories(mangaId).map { it.order }
}
if (flags.includeTracking) {
val tracks =
Track.getTrackRecordsByMangaId(mangaRow[MangaTable.id].value).mapNotNull {
if (it.record == null) {
null
} else {
BackupTracking(
syncId = it.record.trackerId,
// forced not null so its compatible with 1.x backup system
libraryId = it.record.libraryId ?: 0,
mediaId = it.record.remoteId,
title = it.record.title,
lastChapterRead = it.record.lastChapterRead.toFloat(),
totalChapters = it.record.totalChapters,
score = it.record.score.toFloat(),
status = it.record.status,
startedReadingDate = it.record.startDate,
finishedReadingDate = it.record.finishDate,
trackingUrl = it.record.remoteUrl,
private = it.record.private,
)
}
}
if (tracks.isNotEmpty()) {
backupManga.tracking = tracks
}
}
backupManga
}
private fun backupCategories(flags: BackupFlags): List<BackupCategory> {
val categories =
CategoryTable
.selectAll()
.orderBy(CategoryTable.order to SortOrder.ASC)
.map { CategoryTable.toDataClass(it) }
val categoryToMeta = Category.getCategoriesMetaMaps(categories.map { it.id })
return categories.map {
BackupCategory(
it.name,
it.order,
0, // not supported in Tachidesk
).apply {
if (flags.includeClientData) {
this.meta = categoryToMeta[it.id] ?: emptyMap()
}
}
}
}
private fun backupExtensionInfo(
mangas: List<ResultRow>,
flags: BackupFlags,
): List<BackupSource> {
val inLibraryMangaSourceIds =
mangas
.asSequence()
.map { it[MangaTable.sourceReference] }
.distinct()
.toList()
val sources = SourceTable.selectAll().where { SourceTable.id inList inLibraryMangaSourceIds }
val sourceToMeta = Source.getSourcesMetaMaps(sources.map { it[SourceTable.id].value })
return inLibraryMangaSourceIds
.map { mangaSourceId ->
val source = sources.firstOrNull { it[SourceTable.id].value == mangaSourceId }
BackupSource(
source?.get(SourceTable.name) ?: "",
mangaSourceId,
).apply {
if (flags.includeClientData) {
this.meta = sourceToMeta[mangaSourceId] ?: emptyMap()
}
}
}.toList()
}
private fun backupGlobalMeta(flags: BackupFlags): Map<String, String> {
if (!flags.includeClientData) {
return emptyMap()
}
return GlobalMeta.getMetaMap()
}
}
@@ -21,50 +21,21 @@ import kotlinx.coroutines.sync.withLock
import okio.buffer
import okio.gzip
import okio.source
import org.jetbrains.exposed.dao.id.EntityID
import org.jetbrains.exposed.sql.ResultRow
import org.jetbrains.exposed.sql.and
import org.jetbrains.exposed.sql.batchInsert
import org.jetbrains.exposed.sql.insertAndGetId
import org.jetbrains.exposed.sql.selectAll
import org.jetbrains.exposed.sql.statements.BatchUpdateStatement
import org.jetbrains.exposed.sql.transactions.transaction
import org.jetbrains.exposed.sql.update
import suwayomi.tachidesk.global.impl.GlobalMeta
import suwayomi.tachidesk.graphql.types.toStatus
import suwayomi.tachidesk.manga.impl.Category
import suwayomi.tachidesk.manga.impl.Category.modifyCategoriesMetas
import suwayomi.tachidesk.manga.impl.CategoryManga
import suwayomi.tachidesk.manga.impl.Chapter.modifyChaptersMetas
import suwayomi.tachidesk.manga.impl.Manga.clearThumbnail
import suwayomi.tachidesk.manga.impl.Manga.modifyMangasMetas
import suwayomi.tachidesk.manga.impl.Source.modifySourceMetas
import suwayomi.tachidesk.manga.impl.backup.BackupFlags
import suwayomi.tachidesk.manga.impl.backup.proto.ProtoBackupValidator.ValidationResult
import suwayomi.tachidesk.manga.impl.backup.proto.ProtoBackupValidator.validate
import suwayomi.tachidesk.manga.impl.backup.proto.handlers.BackupCategoryHandler
import suwayomi.tachidesk.manga.impl.backup.proto.handlers.BackupGlobalMetaHandler
import suwayomi.tachidesk.manga.impl.backup.proto.handlers.BackupMangaHandler
import suwayomi.tachidesk.manga.impl.backup.proto.handlers.BackupSettingsHandler
import suwayomi.tachidesk.manga.impl.backup.proto.handlers.BackupSourceHandler
import suwayomi.tachidesk.manga.impl.backup.proto.models.Backup
import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupCategory
import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupChapter
import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupHistory
import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupManga
import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupSource
import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupTracking
import suwayomi.tachidesk.manga.impl.track.tracker.TrackerManager
import suwayomi.tachidesk.manga.impl.track.tracker.model.toTrack
import suwayomi.tachidesk.manga.impl.track.tracker.model.toTrackRecordDataClass
import suwayomi.tachidesk.manga.model.dataclass.TrackRecordDataClass
import suwayomi.tachidesk.manga.model.table.ChapterTable
import suwayomi.tachidesk.manga.model.table.MangaTable
import suwayomi.tachidesk.server.database.dbTransaction
import java.io.InputStream
import java.util.Date
import java.util.Timer
import java.util.TimerTask
import java.util.concurrent.ConcurrentHashMap
import kotlin.math.max
import kotlin.time.Duration.Companion.milliseconds
import suwayomi.tachidesk.manga.impl.track.Track as Tracker
object ProtoBackupImport : ProtoBackupBase() {
private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
@@ -73,11 +44,6 @@ object ProtoBackupImport : ProtoBackupBase() {
private val backupMutex = Mutex()
enum class RestoreMode {
NEW,
EXISTING,
}
sealed class BackupRestoreState {
data object Idle : BackupRestoreState()
@@ -215,7 +181,7 @@ object ProtoBackupImport : ProtoBackupBase() {
val categoryMapping =
if (flags.includeCategories) {
updateRestoreState(id, BackupRestoreState.RestoringCategories(restoreSettings + restoreCategories, restoreAmount))
restoreCategories(backup.backupCategories)
BackupCategoryHandler.restore(backup.backupCategories)
} else {
emptyMap()
}
@@ -223,9 +189,9 @@ object ProtoBackupImport : ProtoBackupBase() {
if (flags.includeClientData) {
updateRestoreState(id, BackupRestoreState.RestoringMeta(restoreSettings + restoreCategories + restoreMeta, restoreAmount))
restoreGlobalMeta(backup.meta)
BackupGlobalMetaHandler.restore(backup.meta)
restoreSourceMeta(backup.backupSources)
BackupSourceHandler.restore(backup.backupSources)
}
// Store source mapping for error messages
@@ -245,7 +211,7 @@ object ProtoBackupImport : ProtoBackupBase() {
),
)
restoreManga(
BackupMangaHandler.restore(
backupManga = manga,
categoryMapping = categoryMapping,
sourceMapping = sourceMapping,
@@ -273,292 +239,4 @@ object ProtoBackupImport : ProtoBackupBase() {
return validationResult
}
private fun restoreCategories(backupCategories: List<BackupCategory>): Map<Int, Int> {
val categoryIds = Category.createCategories(backupCategories.map { it.name })
val metaEntryByCategoryId =
categoryIds
.zip(backupCategories)
.associate { (categoryId, backupCategory) ->
categoryId to backupCategory.meta
}
modifyCategoriesMetas(metaEntryByCategoryId)
return backupCategories.withIndex().associate { (index, backupCategory) ->
backupCategory.order to categoryIds[index]
}
}
private fun restoreManga(
backupManga: BackupManga,
categoryMapping: Map<Int, Int>,
sourceMapping: Map<Long, String>,
errors: MutableList<Pair<Date, String>>,
flags: BackupFlags,
) {
val chapters = backupManga.chapters
val categories = backupManga.categories
val history = backupManga.history
val tracking = backupManga.tracking
val dbCategoryIds = categories.mapNotNull { categoryMapping[it] }
try {
restoreMangaData(backupManga, chapters, dbCategoryIds, history, tracking, flags)
} catch (e: Exception) {
val sourceName = sourceMapping[backupManga.source] ?: backupManga.source.toString()
errors.add(Date() to "${backupManga.title} [$sourceName]: ${e.message}")
}
}
private fun restoreMangaData(
manga: BackupManga,
chapters: List<BackupChapter>,
categoryIds: List<Int>,
history: List<BackupHistory>,
tracks: List<BackupTracking>,
flags: BackupFlags,
) {
val dbManga =
transaction {
MangaTable
.selectAll()
.where { (MangaTable.url eq manga.url) and (MangaTable.sourceReference eq manga.source) }
.firstOrNull()
}
val restoreMode = if (dbManga != null) RestoreMode.EXISTING else RestoreMode.NEW
val mangaId =
transaction {
val mangaId =
if (dbManga == null) {
// insert manga to database
MangaTable
.insertAndGetId {
it[url] = manga.url
it[title] = manga.title
it[artist] = manga.artist
it[author] = manga.author
it[description] = manga.description
it[genre] = manga.genre.joinToString()
it[status] = manga.status
it[thumbnail_url] = manga.thumbnailUrl
it[updateStrategy] = manga.updateStrategy.name
it[sourceReference] = manga.source
it[initialized] = manga.description != null
it[inLibrary] = manga.favorite
it[inLibraryAt] = manga.dateAdded.milliseconds.inWholeSeconds
}.value
} else {
val dbMangaId = dbManga[MangaTable.id].value
// Merge manga data
MangaTable.update({ MangaTable.id eq dbMangaId }) {
it[artist] = manga.artist ?: dbManga[artist]
it[author] = manga.author ?: dbManga[author]
it[description] = manga.description ?: dbManga[description]
it[genre] = manga.genre.ifEmpty { null }?.joinToString() ?: dbManga[genre]
it[status] = manga.status
it[thumbnail_url] = manga.thumbnailUrl ?: dbManga[thumbnail_url]
it[updateStrategy] = manga.updateStrategy.name
it[initialized] = dbManga[initialized] || manga.description != null
it[inLibrary] = manga.favorite || dbManga[inLibrary]
it[inLibraryAt] = manga.dateAdded.milliseconds.inWholeSeconds
}
dbMangaId
}
// delete thumbnail in case cached data still exists
clearThumbnail(mangaId)
if (flags.includeClientData && manga.meta.isNotEmpty()) {
modifyMangasMetas(mapOf(mangaId to manga.meta))
}
// merge chapter data
if (flags.includeChapters || flags.includeHistory) {
restoreMangaChapterData(mangaId, restoreMode, chapters, history, flags)
}
// merge categories
if (flags.includeCategories) {
restoreMangaCategoryData(mangaId, categoryIds)
}
mangaId
}
if (flags.includeTracking) {
restoreMangaTrackerData(mangaId, tracks)
}
// TODO: insert/merge history
}
private fun getMangaChapterToRestoreInfo(
mangaId: Int,
restoreMode: RestoreMode,
chapters: List<BackupChapter>,
): Pair<List<BackupChapter>, List<Pair<BackupChapter, ResultRow>>> {
val uniqueChapters = chapters.distinctBy { it.url }
if (restoreMode == RestoreMode.NEW) {
return Pair(uniqueChapters, emptyList())
}
val dbChaptersByUrl = ChapterTable.selectAll().where { ChapterTable.manga eq mangaId }.associateBy { it[ChapterTable.url] }
val (chaptersToUpdate, chaptersToInsert) = uniqueChapters.partition { dbChaptersByUrl.contains(it.url) }
val chaptersToUpdateToDbChapter = chaptersToUpdate.map { it to dbChaptersByUrl[it.url]!! }
return chaptersToInsert to chaptersToUpdateToDbChapter
}
private fun restoreMangaChapterData(
mangaId: Int,
restoreMode: RestoreMode,
chapters: List<BackupChapter>,
history: List<BackupHistory>,
flags: BackupFlags,
) = dbTransaction {
val (chaptersToInsert, chaptersToUpdateToDbChapter) = getMangaChapterToRestoreInfo(mangaId, restoreMode, chapters)
val historyByChapter = history.groupBy({ it.url }, { it.lastRead })
val insertedChapterIds =
if (flags.includeChapters) {
ChapterTable
.batchInsert(chaptersToInsert) { chapter ->
this[ChapterTable.url] = chapter.url
this[ChapterTable.name] = chapter.name
if (chapter.dateUpload == 0L) {
this[ChapterTable.date_upload] = chapter.dateFetch
} else {
this[ChapterTable.date_upload] = chapter.dateUpload
}
this[ChapterTable.chapter_number] = chapter.chapterNumber
this[ChapterTable.scanlator] = chapter.scanlator
this[ChapterTable.sourceOrder] = chaptersToInsert.size - chapter.sourceOrder
this[ChapterTable.manga] = mangaId
this[ChapterTable.isRead] = chapter.read
this[ChapterTable.lastPageRead] = chapter.lastPageRead.coerceAtLeast(0)
this[ChapterTable.isBookmarked] = chapter.bookmark
this[ChapterTable.fetchedAt] = chapter.dateFetch.milliseconds.inWholeSeconds
if (flags.includeHistory) {
this[ChapterTable.lastReadAt] =
historyByChapter[chapter.url]?.maxOrNull()?.milliseconds?.inWholeSeconds ?: 0
}
}.map { it[ChapterTable.id].value }
} else {
emptyList()
}
if (chaptersToUpdateToDbChapter.isNotEmpty()) {
BatchUpdateStatement(ChapterTable).apply {
chaptersToUpdateToDbChapter.forEach { (backupChapter, dbChapter) ->
addBatch(EntityID(dbChapter[ChapterTable.id].value, ChapterTable))
if (flags.includeChapters) {
this[ChapterTable.isRead] = backupChapter.read || dbChapter[ChapterTable.isRead]
this[ChapterTable.lastPageRead] =
max(backupChapter.lastPageRead, dbChapter[ChapterTable.lastPageRead]).coerceAtLeast(0)
this[ChapterTable.isBookmarked] = backupChapter.bookmark || dbChapter[ChapterTable.isBookmarked]
}
if (flags.includeHistory) {
this[ChapterTable.lastReadAt] =
(historyByChapter[backupChapter.url]?.maxOrNull()?.milliseconds?.inWholeSeconds ?: 0)
.coerceAtLeast(dbChapter[ChapterTable.lastReadAt])
}
}
execute(this@dbTransaction)
}
}
if (flags.includeClientData) {
val chaptersToInsertByChapterId = insertedChapterIds.zip(chaptersToInsert)
val chapterToUpdateByChapterId =
chaptersToUpdateToDbChapter.map { (backupChapter, dbChapter) ->
dbChapter[ChapterTable.id].value to
backupChapter
}
val metaEntryByChapterId =
(chaptersToInsertByChapterId + chapterToUpdateByChapterId)
.associate { (chapterId, backupChapter) ->
chapterId to backupChapter.meta
}
modifyChaptersMetas(metaEntryByChapterId)
}
}
private fun restoreMangaCategoryData(
mangaId: Int,
categoryIds: List<Int>,
) {
CategoryManga.addMangaToCategories(mangaId, categoryIds)
}
private fun restoreMangaTrackerData(
mangaId: Int,
tracks: List<BackupTracking>,
) {
val dbTrackRecordsByTrackerId =
Tracker
.getTrackRecordsByMangaId(mangaId)
.mapNotNull { it.record?.toTrack() }
.associateBy { it.tracker_id }
val (existingTracks, newTracks) =
tracks
.mapNotNull { backupTrack ->
val track = backupTrack.toTrack(mangaId)
val isUnsupportedTracker = TrackerManager.getTracker(track.tracker_id) == null
if (isUnsupportedTracker) {
return@mapNotNull null
}
val dbTrack =
dbTrackRecordsByTrackerId[backupTrack.syncId]
?: // new track
return@mapNotNull track
if (track.toTrackRecordDataClass().forComparison() == dbTrack.toTrackRecordDataClass().forComparison()) {
return@mapNotNull null
}
dbTrack.also {
it.remote_id = track.remote_id
it.library_id = track.library_id
it.last_chapter_read = max(dbTrack.last_chapter_read, track.last_chapter_read)
}
}.partition { (it.id ?: -1) > 0 }
Tracker.updateTrackRecords(existingTracks)
Tracker.insertTrackRecords(newTracks)
}
private fun restoreGlobalMeta(meta: Map<String, String>) {
GlobalMeta.modifyMetas(meta)
}
private fun restoreSourceMeta(backupSources: List<BackupSource>) {
modifySourceMetas(backupSources.associateBy { it.sourceId }.mapValues { it.value.meta })
}
private fun TrackRecordDataClass.forComparison() = this.copy(id = 0, mangaId = 0)
}
@@ -0,0 +1,63 @@
package suwayomi.tachidesk.manga.impl.backup.proto.handlers
/*
* Copyright (C) Contributors to the Suwayomi project
*
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
import org.jetbrains.exposed.sql.SortOrder
import org.jetbrains.exposed.sql.selectAll
import suwayomi.tachidesk.manga.impl.Category
import suwayomi.tachidesk.manga.impl.Category.modifyCategoriesMetas
import suwayomi.tachidesk.manga.impl.backup.BackupFlags
import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupCategory
import suwayomi.tachidesk.manga.model.table.CategoryTable
import suwayomi.tachidesk.manga.model.table.toDataClass
import suwayomi.tachidesk.server.database.dbTransaction
object BackupCategoryHandler {
fun backup(flags: BackupFlags): List<BackupCategory> =
dbTransaction {
val categories =
CategoryTable
.selectAll()
.orderBy(CategoryTable.order to SortOrder.ASC)
.map { CategoryTable.toDataClass(it) }
val categoryToMeta =
if (flags.includeClientData) {
Category.getCategoriesMetaMaps(categories.map { it.id })
} else {
emptyMap()
}
categories.map {
BackupCategory(
it.name,
it.order,
0, // not supported in Tachidesk
).apply {
this.meta = categoryToMeta[it.id] ?: emptyMap()
}
}
}
fun restore(backupCategories: List<BackupCategory>): Map<Int, Int> {
val categoryIds = Category.createCategories(backupCategories.map { it.name })
val metaEntryByCategoryId =
categoryIds
.zip(backupCategories)
.associate { (categoryId, backupCategory) ->
categoryId to backupCategory.meta
}
modifyCategoriesMetas(metaEntryByCategoryId)
return backupCategories.withIndex().associate { (index, backupCategory) ->
backupCategory.order to categoryIds[index]
}
}
}
@@ -0,0 +1,25 @@
package suwayomi.tachidesk.manga.impl.backup.proto.handlers
/*
* Copyright (C) Contributors to the Suwayomi project
*
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
import suwayomi.tachidesk.global.impl.GlobalMeta
import suwayomi.tachidesk.manga.impl.backup.BackupFlags
object BackupGlobalMetaHandler {
fun backup(flags: BackupFlags): Map<String, String> {
if (!flags.includeClientData) {
return emptyMap()
}
return GlobalMeta.getMetaMap()
}
fun restore(meta: Map<String, String>) {
GlobalMeta.modifyMetas(meta)
}
}
@@ -0,0 +1,431 @@
package suwayomi.tachidesk.manga.impl.backup.proto.handlers
/*
* Copyright (C) Contributors to the Suwayomi project
*
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
import eu.kanade.tachiyomi.source.model.UpdateStrategy
import org.jetbrains.exposed.dao.id.EntityID
import org.jetbrains.exposed.sql.ResultRow
import org.jetbrains.exposed.sql.SortOrder
import org.jetbrains.exposed.sql.and
import org.jetbrains.exposed.sql.batchInsert
import org.jetbrains.exposed.sql.insertAndGetId
import org.jetbrains.exposed.sql.selectAll
import org.jetbrains.exposed.sql.statements.BatchUpdateStatement
import org.jetbrains.exposed.sql.transactions.transaction
import org.jetbrains.exposed.sql.update
import suwayomi.tachidesk.manga.impl.CategoryManga
import suwayomi.tachidesk.manga.impl.Chapter
import suwayomi.tachidesk.manga.impl.Chapter.modifyChaptersMetas
import suwayomi.tachidesk.manga.impl.Manga
import suwayomi.tachidesk.manga.impl.Manga.clearThumbnail
import suwayomi.tachidesk.manga.impl.Manga.modifyMangasMetas
import suwayomi.tachidesk.manga.impl.backup.BackupFlags
import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupChapter
import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupHistory
import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupManga
import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupTracking
import suwayomi.tachidesk.manga.impl.track.tracker.TrackerManager
import suwayomi.tachidesk.manga.impl.track.tracker.model.toTrack
import suwayomi.tachidesk.manga.impl.track.tracker.model.toTrackRecordDataClass
import suwayomi.tachidesk.manga.model.dataclass.TrackRecordDataClass
import suwayomi.tachidesk.manga.model.table.ChapterTable
import suwayomi.tachidesk.manga.model.table.MangaStatus
import suwayomi.tachidesk.manga.model.table.MangaTable
import suwayomi.tachidesk.manga.model.table.toDataClass
import suwayomi.tachidesk.server.database.dbTransaction
import java.util.Date
import kotlin.math.max
import kotlin.time.Duration.Companion.milliseconds
import kotlin.time.Duration.Companion.seconds
import suwayomi.tachidesk.manga.impl.track.Track as Tracker
object BackupMangaHandler {
private enum class RestoreMode {
NEW,
EXISTING,
}
fun backup(flags: BackupFlags): List<BackupManga> =
dbTransaction {
if (!flags.includeManga) {
return@dbTransaction emptyList()
}
val manga = MangaTable.selectAll().where { MangaTable.inLibrary eq true }.toList()
manga.map { mangaRow ->
val backupManga =
BackupManga(
source = mangaRow[MangaTable.sourceReference],
url = mangaRow[MangaTable.url],
title = mangaRow[MangaTable.title],
artist = mangaRow[MangaTable.artist],
author = mangaRow[MangaTable.author],
description = mangaRow[MangaTable.description],
genre = mangaRow[MangaTable.genre]?.split(", ") ?: emptyList(),
status = MangaStatus.valueOf(mangaRow[MangaTable.status]).value,
thumbnailUrl = mangaRow[MangaTable.thumbnail_url],
dateAdded = mangaRow[MangaTable.inLibraryAt].seconds.inWholeMilliseconds,
viewer = 0, // not supported in Tachidesk
updateStrategy = UpdateStrategy.valueOf(mangaRow[MangaTable.updateStrategy]),
)
val mangaId = mangaRow[MangaTable.id].value
if (flags.includeClientData) {
backupManga.meta = Manga.getMangaMetaMap(mangaId)
}
if (flags.includeChapters || flags.includeHistory) {
val chapters =
transaction {
ChapterTable
.selectAll()
.where { ChapterTable.manga eq mangaId }
.orderBy(ChapterTable.sourceOrder to SortOrder.DESC)
.map {
ChapterTable.toDataClass(it)
}
}
if (flags.includeChapters) {
val chapterToMeta = Chapter.getChaptersMetaMaps(chapters.map { it.id })
backupManga.chapters =
chapters.map {
BackupChapter(
it.url,
it.name,
it.scanlator,
it.read,
it.bookmarked,
it.lastPageRead,
it.fetchedAt.seconds.inWholeMilliseconds,
it.uploadDate,
it.chapterNumber,
chapters.size - it.index,
).apply {
if (flags.includeClientData) {
this.meta = chapterToMeta[it.id] ?: emptyMap()
}
}
}
}
if (flags.includeHistory) {
backupManga.history =
chapters.mapNotNull {
if (it.lastReadAt > 0) {
BackupHistory(
url = it.url,
lastRead = it.lastReadAt.seconds.inWholeMilliseconds,
)
} else {
null
}
}
}
}
if (flags.includeCategories) {
backupManga.categories = CategoryManga.getMangaCategories(mangaId).map { it.order }
}
if (flags.includeTracking) {
val tracks =
Tracker.getTrackRecordsByMangaId(mangaRow[MangaTable.id].value).mapNotNull {
if (it.record == null) {
null
} else {
BackupTracking(
syncId = it.record.trackerId,
// forced not null so its compatible with 1.x backup system
libraryId = it.record.libraryId ?: 0,
mediaId = it.record.remoteId,
title = it.record.title,
lastChapterRead = it.record.lastChapterRead.toFloat(),
totalChapters = it.record.totalChapters,
score = it.record.score.toFloat(),
status = it.record.status,
startedReadingDate = it.record.startDate,
finishedReadingDate = it.record.finishDate,
trackingUrl = it.record.remoteUrl,
private = it.record.private,
)
}
}
if (tracks.isNotEmpty()) {
backupManga.tracking = tracks
}
}
backupManga
}
}
fun restore(
backupManga: BackupManga,
categoryMapping: Map<Int, Int>,
sourceMapping: Map<Long, String>,
errors: MutableList<Pair<Date, String>>,
flags: BackupFlags,
) {
val chapters = backupManga.chapters
val categories = backupManga.categories
val history = backupManga.history
val tracking = backupManga.tracking
val dbCategoryIds = categories.mapNotNull { categoryMapping[it] }
try {
restoreMangaData(backupManga, chapters, dbCategoryIds, history, tracking, flags)
} catch (e: Exception) {
val sourceName = sourceMapping[backupManga.source] ?: backupManga.source.toString()
errors.add(Date() to "${backupManga.title} [$sourceName]: ${e.message}")
}
}
private fun restoreMangaData(
manga: BackupManga,
chapters: List<BackupChapter>,
categoryIds: List<Int>,
history: List<BackupHistory>,
tracks: List<BackupTracking>,
flags: BackupFlags,
) {
val dbManga =
transaction {
MangaTable
.selectAll()
.where { (MangaTable.url eq manga.url) and (MangaTable.sourceReference eq manga.source) }
.firstOrNull()
}
val restoreMode = if (dbManga != null) RestoreMode.EXISTING else RestoreMode.NEW
val mangaId =
transaction {
val mangaId =
if (dbManga == null) {
// insert manga to database
MangaTable
.insertAndGetId {
it[url] = manga.url
it[title] = manga.title
it[artist] = manga.artist
it[author] = manga.author
it[description] = manga.description
it[genre] = manga.genre.joinToString()
it[status] = manga.status
it[thumbnail_url] = manga.thumbnailUrl
it[updateStrategy] = manga.updateStrategy.name
it[sourceReference] = manga.source
it[initialized] = manga.description != null
it[inLibrary] = manga.favorite
it[inLibraryAt] = manga.dateAdded.milliseconds.inWholeSeconds
}.value
} else {
val dbMangaId = dbManga[MangaTable.id].value
// Merge manga data
MangaTable.update({ MangaTable.id eq dbMangaId }) {
it[artist] = manga.artist ?: dbManga[artist]
it[author] = manga.author ?: dbManga[author]
it[description] = manga.description ?: dbManga[description]
it[genre] = manga.genre.ifEmpty { null }?.joinToString() ?: dbManga[genre]
it[status] = manga.status
it[thumbnail_url] = manga.thumbnailUrl ?: dbManga[thumbnail_url]
it[updateStrategy] = manga.updateStrategy.name
it[initialized] = dbManga[initialized] || manga.description != null
it[inLibrary] = manga.favorite || dbManga[inLibrary]
it[inLibraryAt] = manga.dateAdded.milliseconds.inWholeSeconds
}
dbMangaId
}
// delete thumbnail in case cached data still exists
clearThumbnail(mangaId)
if (flags.includeClientData && manga.meta.isNotEmpty()) {
modifyMangasMetas(mapOf(mangaId to manga.meta))
}
// merge chapter data
if (flags.includeChapters || flags.includeHistory) {
restoreMangaChapterData(mangaId, restoreMode, chapters, history, flags)
}
// merge categories
if (flags.includeCategories) {
restoreMangaCategoryData(mangaId, categoryIds)
}
mangaId
}
if (flags.includeTracking) {
restoreMangaTrackerData(mangaId, tracks)
}
// TODO: insert/merge history
}
private fun getMangaChapterToRestoreInfo(
mangaId: Int,
restoreMode: RestoreMode,
chapters: List<BackupChapter>,
): Pair<List<BackupChapter>, List<Pair<BackupChapter, ResultRow>>> {
val uniqueChapters = chapters.distinctBy { it.url }
if (restoreMode == RestoreMode.NEW) {
return Pair(uniqueChapters, emptyList())
}
val dbChaptersByUrl = ChapterTable.selectAll().where { ChapterTable.manga eq mangaId }.associateBy { it[ChapterTable.url] }
val (chaptersToUpdate, chaptersToInsert) = uniqueChapters.partition { dbChaptersByUrl.contains(it.url) }
val chaptersToUpdateToDbChapter = chaptersToUpdate.map { it to dbChaptersByUrl[it.url]!! }
return chaptersToInsert to chaptersToUpdateToDbChapter
}
private fun restoreMangaChapterData(
mangaId: Int,
restoreMode: RestoreMode,
chapters: List<BackupChapter>,
history: List<BackupHistory>,
flags: BackupFlags,
) = dbTransaction {
val (chaptersToInsert, chaptersToUpdateToDbChapter) = getMangaChapterToRestoreInfo(mangaId, restoreMode, chapters)
val historyByChapter = history.groupBy({ it.url }, { it.lastRead })
val insertedChapterIds =
if (flags.includeChapters) {
ChapterTable
.batchInsert(chaptersToInsert) { chapter ->
this[ChapterTable.url] = chapter.url
this[ChapterTable.name] = chapter.name
if (chapter.dateUpload == 0L) {
this[ChapterTable.date_upload] = chapter.dateFetch
} else {
this[ChapterTable.date_upload] = chapter.dateUpload
}
this[ChapterTable.chapter_number] = chapter.chapterNumber
this[ChapterTable.scanlator] = chapter.scanlator
this[ChapterTable.sourceOrder] = chaptersToInsert.size - chapter.sourceOrder
this[ChapterTable.manga] = mangaId
this[ChapterTable.isRead] = chapter.read
this[ChapterTable.lastPageRead] = chapter.lastPageRead.coerceAtLeast(0)
this[ChapterTable.isBookmarked] = chapter.bookmark
this[ChapterTable.fetchedAt] = chapter.dateFetch.milliseconds.inWholeSeconds
if (flags.includeHistory) {
this[ChapterTable.lastReadAt] =
historyByChapter[chapter.url]?.maxOrNull()?.milliseconds?.inWholeSeconds ?: 0
}
}.map { it[ChapterTable.id].value }
} else {
emptyList()
}
if (chaptersToUpdateToDbChapter.isNotEmpty()) {
BatchUpdateStatement(ChapterTable).apply {
chaptersToUpdateToDbChapter.forEach { (backupChapter, dbChapter) ->
addBatch(EntityID(dbChapter[ChapterTable.id].value, ChapterTable))
if (flags.includeChapters) {
this[ChapterTable.isRead] = backupChapter.read || dbChapter[ChapterTable.isRead]
this[ChapterTable.lastPageRead] =
max(backupChapter.lastPageRead, dbChapter[ChapterTable.lastPageRead]).coerceAtLeast(0)
this[ChapterTable.isBookmarked] = backupChapter.bookmark || dbChapter[ChapterTable.isBookmarked]
}
if (flags.includeHistory) {
this[ChapterTable.lastReadAt] =
(historyByChapter[backupChapter.url]?.maxOrNull()?.milliseconds?.inWholeSeconds ?: 0)
.coerceAtLeast(dbChapter[ChapterTable.lastReadAt])
}
}
execute(this@dbTransaction)
}
}
if (flags.includeClientData) {
val chaptersToInsertByChapterId = insertedChapterIds.zip(chaptersToInsert)
val chapterToUpdateByChapterId =
chaptersToUpdateToDbChapter.map { (backupChapter, dbChapter) ->
dbChapter[ChapterTable.id].value to
backupChapter
}
val metaEntryByChapterId =
(chaptersToInsertByChapterId + chapterToUpdateByChapterId)
.associate { (chapterId, backupChapter) ->
chapterId to backupChapter.meta
}
modifyChaptersMetas(metaEntryByChapterId)
}
}
private fun restoreMangaCategoryData(
mangaId: Int,
categoryIds: List<Int>,
) {
CategoryManga.addMangaToCategories(mangaId, categoryIds)
}
private fun restoreMangaTrackerData(
mangaId: Int,
tracks: List<BackupTracking>,
) {
val dbTrackRecordsByTrackerId =
Tracker
.getTrackRecordsByMangaId(mangaId)
.mapNotNull { it.record?.toTrack() }
.associateBy { it.tracker_id }
val (existingTracks, newTracks) =
tracks
.mapNotNull { backupTrack ->
val track = backupTrack.toTrack(mangaId)
val isUnsupportedTracker = TrackerManager.getTracker(track.tracker_id) == null
if (isUnsupportedTracker) {
return@mapNotNull null
}
val dbTrack =
dbTrackRecordsByTrackerId[backupTrack.syncId]
?: // new track
return@mapNotNull track
if (track.toTrackRecordDataClass().forComparison() == dbTrack.toTrackRecordDataClass().forComparison()) {
return@mapNotNull null
}
dbTrack.also {
it.remote_id = track.remote_id
it.library_id = track.library_id
it.last_chapter_read = max(dbTrack.last_chapter_read, track.last_chapter_read)
}
}.partition { (it.id ?: -1) > 0 }
Tracker.updateTrackRecords(existingTracks)
Tracker.insertTrackRecords(newTracks)
}
private fun TrackRecordDataClass.forComparison() = this.copy(id = 0, mangaId = 0)
}
@@ -0,0 +1,56 @@
package suwayomi.tachidesk.manga.impl.backup.proto.handlers
/*
* Copyright (C) Contributors to the Suwayomi project
*
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
import org.jetbrains.exposed.sql.selectAll
import suwayomi.tachidesk.manga.impl.Source
import suwayomi.tachidesk.manga.impl.Source.modifySourceMetas
import suwayomi.tachidesk.manga.impl.backup.BackupFlags
import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupManga
import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupSource
import suwayomi.tachidesk.manga.model.table.SourceTable
import suwayomi.tachidesk.server.database.dbTransaction
object BackupSourceHandler {
fun backup(
backupMangas: List<BackupManga>,
flags: BackupFlags,
): List<BackupSource> =
dbTransaction {
val inLibraryMangaSourceIds =
backupMangas
.asSequence()
.map { it.source }
.distinct()
.toList()
val sources = SourceTable.selectAll().where { SourceTable.id inList inLibraryMangaSourceIds }
val sourceToMeta =
if (flags.includeClientData) {
Source.getSourcesMetaMaps(sources.map { it[SourceTable.id].value })
} else {
emptyMap()
}
inLibraryMangaSourceIds
.map { mangaSourceId ->
val source = sources.firstOrNull { it[SourceTable.id].value == mangaSourceId }
BackupSource(
source?.get(SourceTable.name) ?: "",
mangaSourceId,
).apply {
if (flags.includeClientData) {
this.meta = sourceToMeta[mangaSourceId] ?: emptyMap()
}
}
}.toList()
}
fun restore(backupSources: List<BackupSource>) {
modifySourceMetas(backupSources.associateBy { it.sourceId }.mapValues { it.value.meta })
}
}