refactor: improve sync merging categories (#1559)

* feat: Add versioning to categories

* feat: use random UID for categories.

For legacy and migration we should assign uid on insert, and modify existing one as well in the migration.

* feat: sync category metadata

Add version, uid and lastModifiedAt fields to Category model to allow syncing.

* chore: fix category merging logic

Improve the category merging logic by matching using UIDs first, with a fallback to matching by name for legacy remote categories.

Previously, categories were only matched by name, which could lead to incorrect merges if names were changed. This change ensures more accurate synchronization by prioritizing the unique identifier. Conflict resolution is now based on the `version` field, and logging has been added for better visibility into the merging process.

* refactor: prioritize UID when restoring categories

If a category with the same UID exists, update it instead of creating a new one. Fallback to matching by name if no UID match is found.

* chore: add isSyncing flag like before.

This make sure the version is consistent, and it's not accidentally appended by the trigger, if it does then one device will always be ahead, than previous, and they need to make multiple changes to increase the version.

* Apply suggestion from @jobobby04

Use SY specific numbers(601, 602 for now)

Co-authored-by: jobobby04 <jobobby04@users.noreply.github.com>

* chore: commit review, re-order.

* chore: surround changes in // SY --> // SY <--

* refactor: fallback to existing category UID if backup UID is 0 during restore.

when dealing with old backups (backups created before we added UIDs). In those old backups, backupCategory.uid defaults to 0.
If a user restored an old backup, it would match by name, and then overwrite the newly generated local UID with 0. This would break the synchronization.

* refactor: change to 6xx

* feat: improve sync reliability for categories and settings

- Refactor `mergeCategoriesLists` to correctly match categories by name when UID matching fails, ensuring better reconciliation across devices.
- Fix a bug in category merging where multiple categories with UID 0 (common for non-synced items) caused data loss.
- Update `SyncManager` to detect changes in categories, sources, preferences, saved searches, and extension repos, ensuring they synchronize even when the library favorites haven't changed.
- Convert `BackupCategory` and `BackupExtensionRepos` to data classes to support robust content-aware comparison during the sync process.
- Fix data loss in `mergeSourcesLists`, `mergePreferencesLists`, and `mergeSavedSearchesLists` by retaining local versions when conflicting with remote data.

* fix(sync): properly sync category deletions across devices

Previously, the sync system could not distinguish between a category that was deleted locally and a new category created on another device, causing deleted categories to be restored from the remote backup.

- Update `SyncService` to use `lastSyncTimestamp` to deduce if a missing local category was deleted (if modified before last sync) or newly created remotely (if modified after).
- Update `SyncManager` to explicitly delete local categories that are absent from the merged remote backup, propagating deletions to other devices.
- Fix `RestoreOptions` in `SyncManager` to respect the user's sync preferences instead of hardcoding `categories = true`.

* chore: change it to 6xx and not 600.

* chore: don't need to change this.

* chore: use kotlin time duration units

---------

Co-authored-by: jobobby04 <jobobby04@users.noreply.github.com>

app/src/main/java/eu/kanade/tachiyomi/data/backup/models/BackupCategory.kt

@@ -13,12 +13,18 @@ class BackupCategory(
     @ProtoNumber(100) var flags: Long = 0,
     // SY specific values
     /*@ProtoNumber(600) var mangaOrder: List<Long> = emptyList(),*/
+    @ProtoNumber(601) var version: Long = 0,
+    @ProtoNumber(602) var uid: Long = 0,
+    @ProtoNumber(603) var lastModifiedAt: Long = 0,
 ) {
     fun toCategory(id: Long) = Category(
         id = id,
         name = this@BackupCategory.name,
         flags = this@BackupCategory.flags,
         order = this@BackupCategory.order,
+        version = this@BackupCategory.version,
+        uid = this@BackupCategory.uid,
+        lastModifiedAt = this@BackupCategory.lastModifiedAt,
         /*mangaOrder = this@BackupCategory.mangaOrder*/
     )
 }
@@ -29,5 +35,8 @@ val backupCategoryMapper = { category: Category ->
         name = category.name,
         order = category.order,
         flags = category.flags,
+        version = category.version,
+        uid = category.uid,
+        lastModifiedAt = category.lastModifiedAt,
     )
 }

app/src/main/java/eu/kanade/tachiyomi/data/backup/restore/restorers/CategoriesRestorer.kt

@@ -17,20 +17,63 @@ class CategoriesRestorer(
         if (backupCategories.isNotEmpty()) {
             val dbCategories = getCategories.await()
             val dbCategoriesByName = dbCategories.associateBy { it.name }
+            // SY -->
+            val dbCategoriesByUid = dbCategories.associateBy { it.uid } // Map by UID
+            // SY <--
+
             var nextOrder = dbCategories.maxOfOrNull { it.order }?.plus(1) ?: 0
 
             val categories = backupCategories
                 .sortedBy { it.order }
-                .map {
+                // SY -->
-                    val dbCategory = dbCategoriesByName[it.name]
+                .map { backupCategory ->
-                    if (dbCategory != null) return@map dbCategory
+                    var dbCategory = if (backupCategory.uid != 0L) {
+                        dbCategoriesByUid[backupCategory.uid]
+                    } else {
+                        null
+                    }
+
+                    if (dbCategory == null) {
+                        dbCategory = dbCategoriesByName[backupCategory.name]
+                    }
+
+                    if (dbCategory != null) {
+                        handler.await {
+                            categoriesQueries.update(
+                                name = backupCategory.name,
+                                order = backupCategory.order,
+                                flags = backupCategory.flags,
+                                version = backupCategory.version,
+                                uid = if (backupCategory.uid != 0L) backupCategory.uid else dbCategory.uid,
+                                last_modified_at = backupCategory.lastModifiedAt,
+                                isSyncing = 1,
+                                categoryId = dbCategory.id,
+                            )
+                        }
+                        return@map dbCategory
+                    }
+
                     val order = nextOrder++
                     handler.awaitOneExecutable {
-                        categoriesQueries.insert(it.name, order, it.flags)
+                        categoriesQueries.insert(
+                            backupCategory.name,
+                            order,
+                            backupCategory.flags,
+                            backupCategory.version,
+                            backupCategory.uid,
+                            backupCategory.lastModifiedAt,
+                        )
                         categoriesQueries.selectLastInsertedRowId()
                     }
-                        .let { id -> it.toCategory(id).copy(order = order) }
+                        .let { id -> backupCategory.toCategory(id).copy(order = order) }
                 }
+            // SY <--
+
+            // SY -->
+            handler.await {
+                categoriesQueries.resetIsSyncing()
+            }
+            // SY <--
 
             libraryPreferences.categorizedDisplaySettings().set(
                 (dbCategories + categories)

app/src/main/java/eu/kanade/tachiyomi/data/sync/SyncManager.kt

@@ -73,6 +73,7 @@ class SyncManager(
         handler.await(inTransaction = true) {
             mangasQueries.resetIsSyncing()
             chaptersQueries.resetIsSyncing()
+            categoriesQueries.resetIsSyncing()
         }
 
         val syncOptions = syncPreferences.getSyncSettings()
@@ -156,7 +157,7 @@ class SyncManager(
         }
 
         // Stop the sync early if the remote backup is null or empty
-        if (remoteBackup.backupManga.size == 0) {
+        if (remoteBackup.backupManga.isEmpty() && remoteBackup.backupCategories.isEmpty() && remoteBackup.backupSources.isEmpty()) {
             notifier.showSyncError("No data found on remote server.")
             return
         }
@@ -185,14 +186,40 @@ class SyncManager(
             // SY <--
         )
 
-        // It's local sync no need to restore data. (just update remote data)
+        val hasMangaChanges = filteredFavorites.isNotEmpty()
-        if (filteredFavorites.isEmpty()) {
+        val hasCategoryChanges = remoteBackup.backupCategories != backup.backupCategories
+        val hasSourceChanges = remoteBackup.backupSources != backup.backupSources
+        val hasPreferenceChanges = remoteBackup.backupPreferences != backup.backupPreferences
+        val hasSourcePreferenceChanges = remoteBackup.backupSourcePreferences != backup.backupSourcePreferences
+        val hasExtensionRepoChanges = remoteBackup.backupExtensionRepo != backup.backupExtensionRepo
+        val hasSavedSearchChanges = remoteBackup.backupSavedSearches != backup.backupSavedSearches
+
+        if (!hasMangaChanges && !hasCategoryChanges && !hasSourceChanges &&
+            !hasPreferenceChanges && !hasSourcePreferenceChanges &&
+            !hasExtensionRepoChanges && !hasSavedSearchChanges
+        ) {
             // update the sync timestamp
             syncPreferences.lastSyncTimestamp().set(Date().time)
             notifier.showSyncSuccess("Sync completed successfully")
             return
         }
 
+        if (syncOptions.categories) {
+            val mergedUids = newSyncData.backupCategories.map { it.uid }.toSet()
+            val mergedNames = newSyncData.backupCategories.map { it.name }.toSet()
+            val localCategories = getCategories.await().filterNot { it.id == 0L } // Exclude system category
+            val categoriesToDelete = localCategories.filter {
+                it.uid !in mergedUids && it.name !in mergedNames
+            }
+            if (categoriesToDelete.isNotEmpty()) {
+                handler.await(inTransaction = true) {
+                    categoriesToDelete.forEach {
+                        categoriesQueries.delete(it.id)
+                    }
+                }
+            }
+        }
+
         val backupUri = writeSyncDataToCache(context, newSyncData)
         logcat(LogPriority.DEBUG) { "Got Backup Uri: $backupUri" }
         if (backupUri != null) {
@@ -201,10 +228,14 @@ class SyncManager(
                 backupUri,
                 sync = true,
                 options = RestoreOptions(
-                    appSettings = true,
+                    appSettings = syncOptions.appSettings,
-                    sourceSettings = true,
+                    sourceSettings = syncOptions.sourceSettings,
-                    libraryEntries = true,
+                    libraryEntries = syncOptions.libraryEntries,
-                    extensionRepoSettings = true,
+                    categories = syncOptions.categories,
+                    extensionRepoSettings = syncOptions.extensionRepoSettings,
+                    // SY -->
+                    savedSearches = syncOptions.savedSearches,
+                    // SY <--
                 ),
             )
 

app/src/main/java/eu/kanade/tachiyomi/data/sync/service/SyncService.kt

@@ -14,6 +14,7 @@ import kotlinx.serialization.Serializable
 import kotlinx.serialization.json.Json
 import logcat.LogPriority
 import logcat.logcat
+import kotlin.time.Duration.Companion.seconds
 
 @Serializable
 data class SyncData(
@@ -274,37 +275,70 @@ abstract class SyncService(
         localCategoriesList: List<BackupCategory>?,
         remoteCategoriesList: List<BackupCategory>?,
     ): List<BackupCategory> {
+        val logTag = "MergeCategories"
         if (localCategoriesList == null) return remoteCategoriesList ?: emptyList()
         if (remoteCategoriesList == null) return localCategoriesList
-        val localCategoriesMap = localCategoriesList.associateBy { it.name }
-        val remoteCategoriesMap = remoteCategoriesList.associateBy { it.name }
 
-        val mergedCategoriesMap = mutableMapOf<String, BackupCategory>()
+        val result = mutableListOf<BackupCategory>()
+        val processedLocals = mutableSetOf<BackupCategory>()
 
-        localCategoriesMap.forEach { (name, localCategory) ->
+        val localMapByUid = localCategoriesList.filter { it.uid != 0L }.associateBy { it.uid }
-            val remoteCategory = remoteCategoriesMap[name]
+        val localMapByName = localCategoriesList.associateBy { it.name }
-            if (remoteCategory != null) {
+
-                // Compare and merge local and remote categories
+        val lastSyncTime = syncPreferences.lastSyncTimestamp().get()
-                val mergedCategory = if (localCategory.order > remoteCategory.order) {
+
-                    localCategory
+        remoteCategoriesList.forEach { remote ->
+            var localMatch: BackupCategory? = null
+
+            // 1. Try match by UID
+            if (remote.uid != 0L) {
+                localMatch = localMapByUid[remote.uid]
+            }
+
+            // 2. Try match by Name (fallback)
+            if (localMatch == null) {
+                localMatch = localMapByName[remote.name]
+            }
+
+            if (localMatch != null) {
+                processedLocals.add(localMatch)
+                // Conflict resolution
+                if (localMatch.version >= remote.version) {
+                    logcat(LogPriority.DEBUG, logTag) { "Keeping local category: ${localMatch.name} (UID: ${localMatch.uid})" }
+                    result.add(localMatch)
                 } else {
-                    remoteCategory
+                    logcat(LogPriority.DEBUG, logTag) { "Keeping remote category: ${remote.name} (UID: ${remote.uid})" }
+                    // Preserve Local UID if Remote was 0
+                    if (remote.uid == 0L) {
+                        remote.uid = localMatch.uid
+                    }
+                    result.add(remote)
                 }
-                mergedCategoriesMap[name] = mergedCategory
             } else {
-                // If the category is only in the local list, add it to the merged list
+                val remoteModifiedTimeMillis = remote.lastModifiedAt.seconds.inWholeMilliseconds
-                mergedCategoriesMap[name] = localCategory
+                if (lastSyncTime == 0L || remoteModifiedTimeMillis > lastSyncTime) {
+                    logcat(LogPriority.DEBUG, logTag) { "Adding new remote category: ${remote.name} (UID: ${remote.uid})" }
+                    result.add(remote)
+                } else {
+                    logcat(LogPriority.DEBUG, logTag) { "Dropping deleted remote category: ${remote.name} (UID: ${remote.uid})" }
+                }
             }
         }
 
-        // Add any categories from the remote list that are not in the local list
+        // Add remaining Local Categories
-        remoteCategoriesMap.forEach { (name, remoteCategory) ->
+        localCategoriesList.forEach { local ->
-            if (!mergedCategoriesMap.containsKey(name)) {
+            if (local !in processedLocals) {
-                mergedCategoriesMap[name] = remoteCategory
+                val localModifiedTimeMillis = local.lastModifiedAt.seconds.inWholeMilliseconds
+                if (lastSyncTime == 0L || localModifiedTimeMillis > lastSyncTime) {
+                    logcat(LogPriority.DEBUG, logTag) { "Keeping local only category: ${local.name} (UID: ${local.uid})" }
+                    result.add(local)
+                } else {
+                    logcat(LogPriority.DEBUG, logTag) { "Dropping local category deleted on remote: ${local.name} (UID: ${local.uid})" }
+                }
             }
         }
 
-        return mergedCategoriesMap.values.toList()
+        return result.sortedBy { it.order }
     }
 
     private fun mergeSourcesLists(
@@ -341,8 +375,8 @@ abstract class SyncService(
                     remoteSource
                 }
                 else -> {
-                    logcat(LogPriority.DEBUG, logTag) { "Remote and local is not empty: $sourceId. Skipping." }
+                    logcat(LogPriority.DEBUG, logTag) { "Remote and local have the same source ID: $sourceId. Keeping local." }
-                    null
+                    localSource
                 }
             }
         }
@@ -387,8 +421,8 @@ abstract class SyncService(
                     remotePreference
                 }
                 else -> {
-                    logcat(LogPriority.DEBUG, logTag) { "Both remote and local have keys. Skipping: $key" }
+                    logcat(LogPriority.DEBUG, logTag) { "Both remote and local have the same preference key: $key. Keeping local." }
-                    null
+                    localPreference
                 }
             }
         }
@@ -507,10 +541,8 @@ abstract class SyncService(
                 }
 
                 else -> {
-                    logcat(LogPriority.DEBUG, logTag) {
+                    logcat(LogPriority.DEBUG, logTag) { "Both remote and local have the same saved search key: $compositeKey. Keeping local." }
-                        "No saved search found for composite key: $compositeKey. Skipping."
+                    localSearch
-                    }
-                    null
                 }
             }
         }

app/src/main/java/mihon/core/migration/migrations/MoveSortingModeSettingsMigration.kt

@@ -39,6 +39,10 @@ class MoveSortingModeSettingsMigration : Migration {
                         categoryId = it.id,
                         flags = it.flags and 0b00111100L.inv(),
                         name = null,
+                        version = it.version,
+                        uid = it.uid,
+                        last_modified_at = null,
+                        isSyncing = null,
                         order = null,
                     )
                 }

data/src/main/java/tachiyomi/data/category/CategoryMapper.kt

@@ -8,12 +8,18 @@ object CategoryMapper {
         name: String,
         order: Long,
         flags: Long,
+        version: Long,
+        uid: Long,
+        lastModifiedAt: Long,
     ): Category {
         return Category(
             id = id,
             name = name,
             order = order,
             flags = flags,
+            version = version,
+            uid = uid,
+            lastModifiedAt = lastModifiedAt,
         )
     }
 }

data/src/main/java/tachiyomi/data/category/CategoryRepositoryImpl.kt

@@ -42,6 +42,9 @@ class CategoryRepositoryImpl(
                 name = category.name,
                 order = category.order,
                 flags = category.flags,
+                version = category.version,
+                uid = category.uid,
+                last_modified_at = category.lastModifiedAt,
             )
             categoriesQueries.selectLastInsertedRowId()
         }
@@ -67,6 +70,10 @@ class CategoryRepositoryImpl(
             name = update.name,
             order = update.order,
             flags = update.flags,
+            version = update.version,
+            uid = update.uid,
+            last_modified_at = update.lastModifiedAt,
+            isSyncing = null,
             categoryId = update.id,
         )
     }

data/src/main/sqldelight/tachiyomi/data/categories.sq

@@ -6,11 +6,15 @@ CREATE TABLE categories(
     name TEXT NOT NULL,
     sort INTEGER NOT NULL,
     flags INTEGER NOT NULL,
-    manga_order TEXT AS List<Long> NOT NULL
+    manga_order TEXT AS List<Long> NOT NULL,
+    version INTEGER NOT NULL DEFAULT 0,
+    uid INTEGER NOT NULL DEFAULT 0,
+    last_modified_at INTEGER NOT NULL DEFAULT 0,
+    is_syncing INTEGER NOT NULL DEFAULT 0
 );
 
 -- Insert system category
-INSERT OR IGNORE INTO categories(_id, name, sort, flags, manga_order) VALUES (0, "", -1, 0, "");
+INSERT OR IGNORE INTO categories(_id, name, sort, flags, manga_order, version, uid, last_modified_at, is_syncing) VALUES (0, "", -1, 0, "", 0, 0, 0, 0);
 -- Disallow deletion of default category
 CREATE TRIGGER IF NOT EXISTS system_category_delete_trigger BEFORE DELETE
 ON categories
@@ -20,8 +24,29 @@ BEGIN SELECT CASE
     END;
 END;
 
+CREATE TRIGGER update_category_version AFTER UPDATE ON categories
+WHEN new.is_syncing = 0 AND (
+    new.name != old.name OR
+    new.sort != old.sort OR
+    new.flags != old.flags
+)
+BEGIN
+    UPDATE categories
+    SET version = version + 1,
+        last_modified_at = strftime('%s', 'now')
+    WHERE _id = new._id;
+END;
+
+CREATE TRIGGER insert_category_uid AFTER INSERT ON categories
+BEGIN
+    UPDATE categories
+    SET uid = CASE WHEN uid = 0 THEN abs(random()) ELSE uid END,
+        last_modified_at = CASE WHEN last_modified_at = 0 THEN strftime('%s', 'now') ELSE last_modified_at END
+    WHERE _id = new._id;
+END;
+
 getCategory:
-SELECT _id,name,sort,flags
+SELECT _id,name,sort,flags,version,uid,last_modified_at
 FROM categories
 WHERE _id = :id
 LIMIT 1;
@@ -31,7 +56,10 @@ SELECT
 _id AS id,
 name,
 sort AS `order`,
-flags
+flags,
+version,
+uid,
+last_modified_at
 FROM categories
 ORDER BY sort;
 
@@ -40,15 +68,18 @@ SELECT
 C._id AS id,
 C.name,
 C.sort AS `order`,
-C.flags
+C.flags,
+C.version,
+C.uid,
+C.last_modified_at
 FROM categories C
 JOIN mangas_categories MC
 ON C._id = MC.category_id
 WHERE MC.manga_id = :mangaId;
 
 insert:
-INSERT INTO categories(name, sort, flags, manga_order)
+INSERT INTO categories(name, sort, flags, manga_order, version, uid, last_modified_at)
-VALUES (:name, :order, :flags, "");
+VALUES (:name, :order, :flags, "", :version, :uid, :last_modified_at);
 
 delete:
 DELETE FROM categories
@@ -58,7 +89,11 @@ update:
 UPDATE categories
 SET name = coalesce(:name, name),
     sort = coalesce(:order, sort),
-    flags = coalesce(:flags, flags)
+    flags = coalesce(:flags, flags),
+    version = coalesce(:version, version),
+    uid = coalesce(:uid, uid),
+    last_modified_at = coalesce(:last_modified_at, last_modified_at),
+    is_syncing = coalesce(:isSyncing, is_syncing)
 WHERE _id = :categoryId;
 
 updateAllFlags:
@@ -66,4 +101,9 @@ UPDATE categories SET
 flags = coalesce(?, flags);
 
 selectLastInsertedRowId:
-SELECT last_insert_rowid();
+SELECT last_insert_rowid();
+
+resetIsSyncing:
+UPDATE categories
+SET is_syncing = 0
+WHERE is_syncing = 1;

data/src/main/sqldelight/tachiyomi/migrations/39.sqm

@@ -0,0 +1,27 @@
+ALTER TABLE categories ADD COLUMN version INTEGER NOT NULL DEFAULT 0;
+ALTER TABLE categories ADD COLUMN uid INTEGER NOT NULL DEFAULT 0;
+ALTER TABLE categories ADD COLUMN last_modified_at INTEGER NOT NULL DEFAULT 0;
+ALTER TABLE categories ADD COLUMN is_syncing INTEGER NOT NULL DEFAULT 0;
+
+UPDATE categories SET uid = abs(random());
+
+CREATE TRIGGER insert_category_uid AFTER INSERT ON categories
+BEGIN
+    UPDATE categories
+    SET uid = CASE WHEN uid = 0 THEN abs(random()) ELSE uid END,
+        last_modified_at = CASE WHEN last_modified_at = 0 THEN strftime('%s', 'now') ELSE last_modified_at END
+    WHERE _id = new._id;
+END;
+
+CREATE TRIGGER update_category_version AFTER UPDATE ON categories
+WHEN new.is_syncing = 0 AND (
+    new.name != old.name OR
+    new.sort != old.sort OR
+    new.flags != old.flags
+)
+BEGIN
+    UPDATE categories
+    SET version = version + 1,
+        last_modified_at = strftime('%s', 'now')
+    WHERE _id = new._id;
+END;

domain/src/main/java/tachiyomi/domain/category/model/Category.kt

@@ -7,6 +7,9 @@ data class Category(
     val name: String,
     val order: Long,
     val flags: Long,
+    val version: Long = 0,
+    val uid: Long = 0,
+    val lastModifiedAt: Long = 0,
 ) : Serializable {
 
     val isSystemCategory: Boolean = id == UNCATEGORIZED_ID

domain/src/main/java/tachiyomi/domain/category/model/CategoryUpdate.kt

@@ -5,4 +5,7 @@ data class CategoryUpdate(
     val name: String? = null,
     val order: Long? = null,
     val flags: Long? = null,
+    val version: Long? = null,
+    val uid: Long? = null,
+    val lastModifiedAt: Long? = null,
 )