refactor: improve sync merging categories (#1559)

* feat: Add versioning to categories * feat: use random UID for categories. For legacy and migration we should assign uid on insert, and modify existing one as well in the migration. * feat: sync category metadata Add version, uid and lastModifiedAt fields to Category model to allow syncing. * chore: fix category merging logic Improve the category merging logic by matching using UIDs first, with a fallback to matching by name for legacy remote categories. Previously, categories were only matched by name, which could lead to incorrect merges if names were changed. This change ensures more accurate synchronization by prioritizing the unique identifier. Conflict resolution is now based on the `version` field, and logging has been added for better visibility into the merging process. * refactor: prioritize UID when restoring categories If a category with the same UID exists, update it instead of creating a new one. Fallback to matching by name if no UID match is found. * chore: add isSyncing flag like before. This make sure the version is consistent, and it's not accidentally appended by the trigger, if it does then one device will always be ahead, than previous, and they need to make multiple changes to increase the version. * Apply suggestion from @jobobby04 Use SY specific numbers(601, 602 for now) Co-authored-by: jobobby04 <jobobby04@users.noreply.github.com> * chore: commit review, re-order. * chore: surround changes in // SY --> // SY <-- * refactor: fallback to existing category UID if backup UID is 0 during restore. when dealing with old backups (backups created before we added UIDs). In those old backups, backupCategory.uid defaults to 0. If a user restored an old backup, it would match by name, and then overwrite the newly generated local UID with 0. This would break the synchronization. * refactor: change to 6xx * feat: improve sync reliability for categories and settings - Refactor `mergeCategoriesLists` to correctly match categories by name when UID matching fails, ensuring better reconciliation across devices. - Fix a bug in category merging where multiple categories with UID 0 (common for non-synced items) caused data loss. - Update `SyncManager` to detect changes in categories, sources, preferences, saved searches, and extension repos, ensuring they synchronize even when the library favorites haven't changed. - Convert `BackupCategory` and `BackupExtensionRepos` to data classes to support robust content-aware comparison during the sync process. - Fix data loss in `mergeSourcesLists`, `mergePreferencesLists`, and `mergeSavedSearchesLists` by retaining local versions when conflicting with remote data. * fix(sync): properly sync category deletions across devices Previously, the sync system could not distinguish between a category that was deleted locally and a new category created on another device, causing deleted categories to be restored from the remote backup. - Update `SyncService` to use `lastSyncTimestamp` to deduce if a missing local category was deleted (if modified before last sync) or newly created remotely (if modified after). - Update `SyncManager` to explicitly delete local categories that are absent from the merged remote backup, propagating deletions to other devices. - Fix `RestoreOptions` in `SyncManager` to respect the user's sync preferences instead of hardcoding `categories = true`. * chore: change it to 6xx and not 600. * chore: don't need to change this. * chore: use kotlin time duration units --------- Co-authored-by: jobobby04 <jobobby04@users.noreply.github.com>
2026-04-04 03:49:37 +11:00
parent fe0b14ab97
commit ba75395648
11 changed files with 252 additions and 47 deletions
@@ -13,12 +13,18 @@ class BackupCategory(
    @ProtoNumber(100) var flags: Long = 0,
    // SY specific values
    /*@ProtoNumber(600) var mangaOrder: List<Long> = emptyList(),*/
+    @ProtoNumber(601) var version: Long = 0,
+    @ProtoNumber(602) var uid: Long = 0,
+    @ProtoNumber(603) var lastModifiedAt: Long = 0,
 ) {
    fun toCategory(id: Long) = Category(
        id = id,
        name = this@BackupCategory.name,
        flags = this@BackupCategory.flags,
        order = this@BackupCategory.order,
+        version = this@BackupCategory.version,
+        uid = this@BackupCategory.uid,
+        lastModifiedAt = this@BackupCategory.lastModifiedAt,
        /*mangaOrder = this@BackupCategory.mangaOrder*/
    )
 }
@@ -29,5 +35,8 @@ val backupCategoryMapper = { category: Category ->
        name = category.name,
        order = category.order,
        flags = category.flags,
+        version = category.version,
+        uid = category.uid,
+        lastModifiedAt = category.lastModifiedAt,
    )
 }
@@ -17,20 +17,63 @@ class CategoriesRestorer(
        if (backupCategories.isNotEmpty()) {
            val dbCategories = getCategories.await()
            val dbCategoriesByName = dbCategories.associateBy { it.name }
+            // SY -->
+            val dbCategoriesByUid = dbCategories.associateBy { it.uid } // Map by UID
+            // SY <--
+
            var nextOrder = dbCategories.maxOfOrNull { it.order }?.plus(1) ?: 0

            val categories = backupCategories
                .sortedBy { it.order }
-                .map {
-                    val dbCategory = dbCategoriesByName[it.name]
-                    if (dbCategory != null) return@map dbCategory
+                // SY -->
+                .map { backupCategory ->
+                    var dbCategory = if (backupCategory.uid != 0L) {
+                        dbCategoriesByUid[backupCategory.uid]
+                    } else {
+                        null
+                    }
+
+                    if (dbCategory == null) {
+                        dbCategory = dbCategoriesByName[backupCategory.name]
+                    }
+
+                    if (dbCategory != null) {
+                        handler.await {
+                            categoriesQueries.update(
+                                name = backupCategory.name,
+                                order = backupCategory.order,
+                                flags = backupCategory.flags,
+                                version = backupCategory.version,
+                                uid = if (backupCategory.uid != 0L) backupCategory.uid else dbCategory.uid,
+                                last_modified_at = backupCategory.lastModifiedAt,
+                                isSyncing = 1,
+                                categoryId = dbCategory.id,
+                            )
+                        }
+                        return@map dbCategory
+                    }
+
                    val order = nextOrder++
                    handler.awaitOneExecutable {
-                        categoriesQueries.insert(it.name, order, it.flags)
+                        categoriesQueries.insert(
+                            backupCategory.name,
+                            order,
+                            backupCategory.flags,
+                            backupCategory.version,
+                            backupCategory.uid,
+                            backupCategory.lastModifiedAt,
+                        )
                        categoriesQueries.selectLastInsertedRowId()
                    }
-                        .let { id -> it.toCategory(id).copy(order = order) }
+                        .let { id -> backupCategory.toCategory(id).copy(order = order) }
                }
+            // SY <--
+
+            // SY -->
+            handler.await {
+                categoriesQueries.resetIsSyncing()
+            }
+            // SY <--

            libraryPreferences.categorizedDisplaySettings().set(
                (dbCategories + categories)
@@ -73,6 +73,7 @@ class SyncManager(
        handler.await(inTransaction = true) {
            mangasQueries.resetIsSyncing()
            chaptersQueries.resetIsSyncing()
+            categoriesQueries.resetIsSyncing()
        }

        val syncOptions = syncPreferences.getSyncSettings()
@@ -156,7 +157,7 @@ class SyncManager(
        }

        // Stop the sync early if the remote backup is null or empty
-        if (remoteBackup.backupManga.size == 0) {
+        if (remoteBackup.backupManga.isEmpty() && remoteBackup.backupCategories.isEmpty() && remoteBackup.backupSources.isEmpty()) {
            notifier.showSyncError("No data found on remote server.")
            return
        }
@@ -185,14 +186,40 @@ class SyncManager(
            // SY <--
        )

-        // It's local sync no need to restore data. (just update remote data)
-        if (filteredFavorites.isEmpty()) {
+        val hasMangaChanges = filteredFavorites.isNotEmpty()
+        val hasCategoryChanges = remoteBackup.backupCategories != backup.backupCategories
+        val hasSourceChanges = remoteBackup.backupSources != backup.backupSources
+        val hasPreferenceChanges = remoteBackup.backupPreferences != backup.backupPreferences
+        val hasSourcePreferenceChanges = remoteBackup.backupSourcePreferences != backup.backupSourcePreferences
+        val hasExtensionRepoChanges = remoteBackup.backupExtensionRepo != backup.backupExtensionRepo
+        val hasSavedSearchChanges = remoteBackup.backupSavedSearches != backup.backupSavedSearches
+
+        if (!hasMangaChanges && !hasCategoryChanges && !hasSourceChanges &&
+            !hasPreferenceChanges && !hasSourcePreferenceChanges &&
+            !hasExtensionRepoChanges && !hasSavedSearchChanges
+        ) {
            // update the sync timestamp
            syncPreferences.lastSyncTimestamp().set(Date().time)
            notifier.showSyncSuccess("Sync completed successfully")
            return
        }

+        if (syncOptions.categories) {
+            val mergedUids = newSyncData.backupCategories.map { it.uid }.toSet()
+            val mergedNames = newSyncData.backupCategories.map { it.name }.toSet()
+            val localCategories = getCategories.await().filterNot { it.id == 0L } // Exclude system category
+            val categoriesToDelete = localCategories.filter {
+                it.uid !in mergedUids && it.name !in mergedNames
+            }
+            if (categoriesToDelete.isNotEmpty()) {
+                handler.await(inTransaction = true) {
+                    categoriesToDelete.forEach {
+                        categoriesQueries.delete(it.id)
+                    }
+                }
+            }
+        }
+
        val backupUri = writeSyncDataToCache(context, newSyncData)
        logcat(LogPriority.DEBUG) { "Got Backup Uri: $backupUri" }
        if (backupUri != null) {
@@ -201,10 +228,14 @@ class SyncManager(
                backupUri,
                sync = true,
                options = RestoreOptions(
-                    appSettings = true,
-                    sourceSettings = true,
-                    libraryEntries = true,
-                    extensionRepoSettings = true,
+                    appSettings = syncOptions.appSettings,
+                    sourceSettings = syncOptions.sourceSettings,
+                    libraryEntries = syncOptions.libraryEntries,
+                    categories = syncOptions.categories,
+                    extensionRepoSettings = syncOptions.extensionRepoSettings,
+                    // SY -->
+                    savedSearches = syncOptions.savedSearches,
+                    // SY <--
                ),
            )

@@ -14,6 +14,7 @@ import kotlinx.serialization.Serializable
 import kotlinx.serialization.json.Json
 import logcat.LogPriority
 import logcat.logcat
+import kotlin.time.Duration.Companion.seconds

@Serializable
 data class SyncData(
@@ -274,37 +275,70 @@ abstract class SyncService(
        localCategoriesList: List<BackupCategory>?,
        remoteCategoriesList: List<BackupCategory>?,
    ): List<BackupCategory> {
+        val logTag = "MergeCategories"
        if (localCategoriesList == null) return remoteCategoriesList ?: emptyList()
        if (remoteCategoriesList == null) return localCategoriesList
-        val localCategoriesMap = localCategoriesList.associateBy { it.name }
-        val remoteCategoriesMap = remoteCategoriesList.associateBy { it.name }

-        val mergedCategoriesMap = mutableMapOf<String, BackupCategory>()
+        val result = mutableListOf<BackupCategory>()
+        val processedLocals = mutableSetOf<BackupCategory>()

-        localCategoriesMap.forEach { (name, localCategory) ->
-            val remoteCategory = remoteCategoriesMap[name]
-            if (remoteCategory != null) {
-                // Compare and merge local and remote categories
-                val mergedCategory = if (localCategory.order > remoteCategory.order) {
-                    localCategory
+        val localMapByUid = localCategoriesList.filter { it.uid != 0L }.associateBy { it.uid }
+        val localMapByName = localCategoriesList.associateBy { it.name }
+
+        val lastSyncTime = syncPreferences.lastSyncTimestamp().get()
+
+        remoteCategoriesList.forEach { remote ->
+            var localMatch: BackupCategory? = null
+
+            // 1. Try match by UID
+            if (remote.uid != 0L) {
+                localMatch = localMapByUid[remote.uid]
+            }
+
+            // 2. Try match by Name (fallback)
+            if (localMatch == null) {
+                localMatch = localMapByName[remote.name]
+            }
+
+            if (localMatch != null) {
+                processedLocals.add(localMatch)
+                // Conflict resolution
+                if (localMatch.version >= remote.version) {
+                    logcat(LogPriority.DEBUG, logTag) { "Keeping local category: ${localMatch.name} (UID: ${localMatch.uid})" }
+                    result.add(localMatch)
                } else {
-                    remoteCategory
+                    logcat(LogPriority.DEBUG, logTag) { "Keeping remote category: ${remote.name} (UID: ${remote.uid})" }
+                    // Preserve Local UID if Remote was 0
+                    if (remote.uid == 0L) {
+                        remote.uid = localMatch.uid
+                    }
+                    result.add(remote)
                }
-                mergedCategoriesMap[name] = mergedCategory
            } else {
-                // If the category is only in the local list, add it to the merged list
-                mergedCategoriesMap[name] = localCategory
+                val remoteModifiedTimeMillis = remote.lastModifiedAt.seconds.inWholeMilliseconds
+                if (lastSyncTime == 0L || remoteModifiedTimeMillis > lastSyncTime) {
+                    logcat(LogPriority.DEBUG, logTag) { "Adding new remote category: ${remote.name} (UID: ${remote.uid})" }
+                    result.add(remote)
+                } else {
+                    logcat(LogPriority.DEBUG, logTag) { "Dropping deleted remote category: ${remote.name} (UID: ${remote.uid})" }
+                }
            }
        }

-        // Add any categories from the remote list that are not in the local list
-        remoteCategoriesMap.forEach { (name, remoteCategory) ->
-            if (!mergedCategoriesMap.containsKey(name)) {
-                mergedCategoriesMap[name] = remoteCategory
+        // Add remaining Local Categories
+        localCategoriesList.forEach { local ->
+            if (local !in processedLocals) {
+                val localModifiedTimeMillis = local.lastModifiedAt.seconds.inWholeMilliseconds
+                if (lastSyncTime == 0L || localModifiedTimeMillis > lastSyncTime) {
+                    logcat(LogPriority.DEBUG, logTag) { "Keeping local only category: ${local.name} (UID: ${local.uid})" }
+                    result.add(local)
+                } else {
+                    logcat(LogPriority.DEBUG, logTag) { "Dropping local category deleted on remote: ${local.name} (UID: ${local.uid})" }
+                }
            }
        }

-        return mergedCategoriesMap.values.toList()
+        return result.sortedBy { it.order }
    }

    private fun mergeSourcesLists(
@@ -341,8 +375,8 @@ abstract class SyncService(
                    remoteSource
                }
                else -> {
-                    logcat(LogPriority.DEBUG, logTag) { "Remote and local is not empty: $sourceId. Skipping." }
-                    null
+                    logcat(LogPriority.DEBUG, logTag) { "Remote and local have the same source ID: $sourceId. Keeping local." }
+                    localSource
                }
            }
        }
@@ -387,8 +421,8 @@ abstract class SyncService(
                    remotePreference
                }
                else -> {
-                    logcat(LogPriority.DEBUG, logTag) { "Both remote and local have keys. Skipping: $key" }
-                    null
+                    logcat(LogPriority.DEBUG, logTag) { "Both remote and local have the same preference key: $key. Keeping local." }
+                    localPreference
                }
            }
        }
@@ -507,10 +541,8 @@ abstract class SyncService(
                }

                else -> {
-                    logcat(LogPriority.DEBUG, logTag) {
-                        "No saved search found for composite key: $compositeKey. Skipping."
-                    }
-                    null
+                    logcat(LogPriority.DEBUG, logTag) { "Both remote and local have the same saved search key: $compositeKey. Keeping local." }
+                    localSearch
                }
            }
        }
@@ -39,6 +39,10 @@ class MoveSortingModeSettingsMigration : Migration {
                        categoryId = it.id,
                        flags = it.flags and 0b00111100L.inv(),
                        name = null,
+                        version = it.version,
+                        uid = it.uid,
+                        last_modified_at = null,
+                        isSyncing = null,
                        order = null,
                    )
                }