Database changes to support library syncing (#9683)

* feat: added migrations.

* feat: create triggers, account for new installs.

* feat: update mappers to include the new field.

* feat: update backupManga and backupChapter.

Include the new fields to be backed up as well.

* feat: add sql query to fetch all manga with `last_favorited_at` field.

* feat: version bump.

* chore: revert and refactor.

* chore: forgot to lower case the field name.

* chore: added getAllManga query as well renamed `fetchMangaWithLastFavorite` to `getMangasWithFavoriteTimestamp`

* chore: oops that's not meant to be there.

* feat: back fill and set last_modified_at to not null.

* chore: remove redundant triggers.

* fix: build error, accidentally removed insert.

* fix: build error, accidentally removed insert.

* refactor: review pointer, make fields not null.

(cherry picked from commit a577f5534f31086174b1cc851d8b489d69f557e8)

# Conflicts:
#	app/build.gradle.kts
#	app/src/main/java/eu/kanade/tachiyomi/data/backup/models/BackupManga.kt
#	data/src/main/java/tachiyomi/data/manga/MangaMapper.kt
#	data/src/main/sqldelight/tachiyomi/data/mangas.sq
#	data/src/main/sqldelight/tachiyomi/migrations/25.sqm
#	domain/src/main/java/tachiyomi/domain/manga/model/Manga.kt

data/src/main/java/tachiyomi/data/LibraryQuery.kt

@@ -30,13 +30,15 @@ private val mapper = { cursor: SqlCursor ->
         filtered_scanlators = cursor.getString(18)?.let(listOfStringsAndAdapter::decode),
         update_strategy = updateStrategyAdapter.decode(cursor.getLong(19)!!),
         calculate_interval = cursor.getLong(20)!!,
-        totalCount = cursor.getLong(21)!!,
-        readCount = cursor.getLong(22)!!,
-        latestUpload = cursor.getLong(23)!!,
-        chapterFetchedAt = cursor.getLong(24)!!,
-        lastRead = cursor.getLong(25)!!,
-        bookmarkCount = cursor.getLong(26)!!,
-        category = cursor.getLong(27)!!,
+        last_modified_at = cursor.getLong(21)!!,
+        favorite_modified_at = cursor.getLong(22)!!,
+        totalCount = cursor.getLong(23)!!,
+        readCount = cursor.getLong(24)!!,
+        latestUpload = cursor.getLong(25)!!,
+        chapterFetchedAt = cursor.getLong(26)!!,
+        lastRead = cursor.getLong(27)!!,
+        bookmarkCount = cursor.getLong(28)!!,
+        category = cursor.getLong(29)!!,
     )
 }
 

data/src/main/java/tachiyomi/data/chapter/ChapterMapper.kt

@@ -2,8 +2,8 @@ package tachiyomi.data.chapter
 
 import tachiyomi.domain.chapter.model.Chapter
 
-val chapterMapper: (Long, Long, String, String, String?, Boolean, Boolean, Long, Float, Long, Long, Long) -> Chapter =
-    { id, mangaId, url, name, scanlator, read, bookmark, lastPageRead, chapterNumber, sourceOrder, dateFetch, dateUpload ->
+val chapterMapper: (Long, Long, String, String, String?, Boolean, Boolean, Long, Float, Long, Long, Long, Long) -> Chapter =
+    { id, mangaId, url, name, scanlator, read, bookmark, lastPageRead, chapterNumber, sourceOrder, dateFetch, dateUpload, lastModifiedAt ->
         Chapter(
             id = id,
             mangaId = mangaId,
@@ -17,5 +17,6 @@ val chapterMapper: (Long, Long, String, String, String?, Boolean, Boolean, Long,
             dateUpload = dateUpload,
             chapterNumber = chapterNumber,
             scanlator = scanlator,
+            lastModifiedAt = lastModifiedAt,
         )
     }

data/src/main/java/tachiyomi/data/manga/MangaMapper.kt

@@ -5,8 +5,8 @@ import tachiyomi.domain.library.model.LibraryManga
 import tachiyomi.domain.manga.model.Manga
 import tachiyomi.view.LibraryView
 
-val mangaMapper: (Long, Long, String, String?, String?, String?, List<String>?, String, Long, String?, Boolean, Long?, Long?, Boolean, Long, Long, Long, Long, List<String>?, UpdateStrategy, Long) -> Manga =
-    { id, source, url, artist, author, description, genre, title, status, thumbnailUrl, favorite, lastUpdate, nextUpdate, initialized, viewerFlags, chapterFlags, coverLastModified, dateAdded, filteredScanlators, updateStrategy, calculateInterval ->
+val mangaMapper: (Long, Long, String, String?, String?, String?, List<String>?, String, Long, String?, Boolean, Long?, Long?, Boolean, Long, Long, Long, Long, List<String>?, UpdateStrategy, Long, Long, Long?) -> Manga =
+    { id, source, url, artist, author, description, genre, title, status, thumbnailUrl, favorite, lastUpdate, nextUpdate, initialized, viewerFlags, chapterFlags, coverLastModified, dateAdded, filteredScanlators, updateStrategy, calculateInterval, lastModifiedAt, favoriteModifiedAt ->
         Manga(
             id = id,
             source = source,
@@ -33,11 +33,13 @@ val mangaMapper: (Long, Long, String, String?, String?, String?, List<String>?,
             // SY -->
             filteredScanlators = filteredScanlators,
             // SY <--
+            lastModifiedAt = lastModifiedAt,
+            favoriteModifiedAt = favoriteModifiedAt,
         )
     }
 
-val libraryManga: (Long, Long, String, String?, String?, String?, List<String>?, String, Long, String?, Boolean, Long?, Long?, Boolean, Long, Long, Long, Long, List<String>?, UpdateStrategy, Long, Long, Long, Long, Long, Long, Long, Long) -> LibraryManga =
-    { id, source, url, artist, author, description, genre, title, status, thumbnailUrl, favorite, lastUpdate, nextUpdate, initialized, viewerFlags, chapterFlags, coverLastModified, dateAdded, filteredScanlators, updateStrategy, calculateInterval, totalCount, readCount, latestUpload, chapterFetchedAt, lastRead, bookmarkCount, category ->
+val libraryManga: (Long, Long, String, String?, String?, String?, List<String>?, String, Long, String?, Boolean, Long?, Long?, Boolean, Long, Long, Long, Long, List<String>?, UpdateStrategy, Long, Long, Long?, Long, Long, Long, Long, Long, Long, Long) -> LibraryManga =
+    { id, source, url, artist, author, description, genre, title, status, thumbnailUrl, favorite, lastUpdate, nextUpdate, initialized, viewerFlags, chapterFlags, coverLastModified, dateAdded, filteredScanlators, updateStrategy, calculateInterval, lastModifiedAt, favoriteModifiedAt, totalCount, readCount, latestUpload, chapterFetchedAt, lastRead, bookmarkCount, category ->
         LibraryManga(
             manga = mangaMapper(
                 id,
@@ -63,6 +65,8 @@ val libraryManga: (Long, Long, String, String?, String?, String?, List<String>?,
                 // SY <--
                 updateStrategy,
                 calculateInterval,
+                lastModifiedAt,
+                favoriteModifiedAt,
             ),
             category = category,
             totalChapters = totalCount,
@@ -98,6 +102,8 @@ val libraryViewMapper: (LibraryView) -> LibraryManga = {
             initialized = it.initialized,
             filteredScanlators = it.filtered_scanlators,
             calculateInterval = it.calculate_interval.toInt(),
+            lastModifiedAt = it.last_modified_at,
+            favoriteModifiedAt = it.favorite_modified_at,
         ),
         it.category,
         it.totalCount,

data/src/main/sqldelight/tachiyomi/data/chapters.sq

@@ -11,6 +11,7 @@ CREATE TABLE chapters(
     source_order INTEGER NOT NULL,
     date_fetch INTEGER AS Long NOT NULL,
     date_upload INTEGER AS Long NOT NULL,
+    last_modified_at INTEGER AS Long NOT NULL,
     FOREIGN KEY(manga_id) REFERENCES mangas (_id)
     ON DELETE CASCADE
 );
@@ -18,6 +19,15 @@ CREATE TABLE chapters(
 CREATE INDEX chapters_manga_id_index ON chapters(manga_id);
 CREATE INDEX chapters_unread_by_manga_index ON chapters(manga_id, read) WHERE read = 0;
 
+CREATE TRIGGER update_last_modified_at_chapters
+AFTER UPDATE ON chapters
+FOR EACH ROW
+BEGIN
+  UPDATE chapters
+  SET last_modified_at = strftime('%s', 'now')
+  WHERE _id = new._id;
+END;
+
 getChapterById:
 SELECT *
 FROM chapters
@@ -58,8 +68,8 @@ DELETE FROM chapters
 WHERE _id IN :chapterIds;
 
 insert:
-INSERT INTO chapters(manga_id,url,name,scanlator,read,bookmark,last_page_read,chapter_number,source_order,date_fetch,date_upload)
-VALUES (:mangaId,:url,:name,:scanlator,:read,:bookmark,:lastPageRead,:chapterNumber,:sourceOrder,:dateFetch,:dateUpload);
+INSERT INTO chapters(manga_id, url, name, scanlator, read, bookmark, last_page_read, chapter_number, source_order, date_fetch, date_upload, last_modified_at)
+VALUES (:mangaId, :url, :name, :scanlator, :read, :bookmark, :lastPageRead, :chapterNumber, :sourceOrder, :dateFetch, :dateUpload, strftime('%s', 'now'));
 
 update:
 UPDATE chapters

data/src/main/sqldelight/tachiyomi/data/mangas.sq

@@ -23,12 +23,31 @@ CREATE TABLE mangas(
     date_added INTEGER AS Long NOT NULL,
     filtered_scanlators TEXT AS List<String>,
     update_strategy INTEGER AS UpdateStrategy NOT NULL DEFAULT 0,
-    calculate_interval INTEGER DEFAULT 0 NOT NULL
+    calculate_interval INTEGER DEFAULT 0 NOT NULL,
+    last_modified_at INTEGER AS Long NOT NULL,
+    favorite_modified_at INTEGER AS Long
 );
 
 CREATE INDEX library_favorite_index ON mangas(favorite) WHERE favorite = 1;
 CREATE INDEX mangas_url_index ON mangas(url);
 
+CREATE TRIGGER update_favorite_modified_at_mangas
+AFTER UPDATE OF favorite ON mangas
+BEGIN
+  UPDATE mangas
+  SET favorite_modified_at = strftime('%s', 'now')
+  WHERE _id = new._id;
+END;
+
+CREATE TRIGGER update_last_modified_at_mangas
+AFTER UPDATE ON mangas
+FOR EACH ROW
+BEGIN
+  UPDATE mangas
+  SET last_modified_at = strftime('%s', 'now')
+  WHERE _id = new._id;
+END;
+
 getMangaById:
 SELECT *
 FROM mangas
@@ -53,6 +72,15 @@ WHERE favorite = 0 AND _id IN(
     SELECT chapters.manga_id FROM chapters WHERE read = 1 OR last_page_read != 0
 );
 
+getAllManga:
+SELECT *
+FROM mangas;
+
+getMangasWithFavoriteTimestamp:
+SELECT *
+FROM mangas
+WHERE favorite_modified_at IS NOT NULL;
+
 getSourceIdWithFavoriteCount:
 SELECT
 source,
@@ -99,8 +127,8 @@ WHERE favorite = 0 AND source IN :sourceIdsAND AND _id NOT IN (
 );
 
 insert:
-INSERT INTO mangas(source,url,artist,author,description,genre,title,status,thumbnail_url,favorite,last_update,next_update,initialized,viewer,chapter_flags,cover_last_modified,date_added,filtered_scanlators,update_strategy,calculate_interval)
-VALUES (:source,:url,:artist,:author,:description,:genre,:title,:status,:thumbnailUrl,:favorite,:lastUpdate,:nextUpdate,:initialized,:viewerFlags,:chapterFlags,:coverLastModified,:dateAdded,:filteredScanlators,:updateStrategy,:calculateInterval);
+INSERT INTO mangas(source,url,artist,author,description,genre,title,status,thumbnail_url,favorite,last_update,next_update,initialized,viewer,chapter_flags,cover_last_modified,date_added,filtered_scanlators,update_strategy,calculate_interval, last_modified_at)
+VALUES (:source,:url,:artist,:author,:description,:genre,:title,:status,:thumbnailUrl,:favorite,:lastUpdate,:nextUpdate,:initialized,:viewerFlags,:chapterFlags,:coverLastModified,:dateAdded,:filteredScanlators,:updateStrategy,:calculateInterval, strftime('%s', 'now'));
 
 update:
 UPDATE mangas SET

data/src/main/sqldelight/tachiyomi/data/mangas_categories.sq

@@ -2,15 +2,25 @@ CREATE TABLE mangas_categories(
     _id INTEGER NOT NULL PRIMARY KEY,
     manga_id INTEGER NOT NULL,
     category_id INTEGER NOT NULL,
+    last_modified_at INTEGER AS Long NOT NULL,
     FOREIGN KEY(category_id) REFERENCES categories (_id)
     ON DELETE CASCADE,
     FOREIGN KEY(manga_id) REFERENCES mangas (_id)
     ON DELETE CASCADE
 );
 
+CREATE TRIGGER update_last_modified_at_mangas_categories
+AFTER UPDATE ON mangas_categories
+FOR EACH ROW
+BEGIN
+  UPDATE mangas_categories
+  SET last_modified_at = strftime('%s', 'now')
+  WHERE _id = new._id;
+END;
+
 insert:
-INSERT INTO mangas_categories(manga_id, category_id)
-VALUES (:mangaId, :categoryId);
+INSERT INTO mangas_categories(manga_id, category_id, last_modified_at)
+VALUES (:mangaId, :categoryId, strftime('%s', 'now'));
 
 deleteMangaCategoryByMangaId:
 DELETE FROM mangas_categories

data/src/main/sqldelight/tachiyomi/migrations/28.sqm

@@ -0,0 +1,49 @@
+ALTER TABLE mangas ADD COLUMN last_modified_at INTEGER AS Long NOT NULL;
+ALTER TABLE mangas ADD COLUMN favorite_modified_at INTEGER AS Long;
+ALTER TABLE mangas_categories ADD COLUMN last_modified_at INTEGER AS Long NOT NULL;
+ALTER TABLE chapters ADD COLUMN last_modified_at INTEGER AS Long NOT NULL;
+
+UPDATE mangas SET last_modified_at = strftime('%s', 'now');
+UPDATE mangas SET favorite_modified_at = strftime('%s', 'now') WHERE favorite = 1;
+UPDATE mangas_categories SET last_modified_at = strftime('%s', 'now');
+UPDATE chapters SET last_modified_at = strftime('%s', 'now');
+
+-- Create triggers
+DROP TRIGGER IF EXISTS update_last_modified_at_mangas;
+CREATE TRIGGER update_last_modified_at_mangas
+AFTER UPDATE ON mangas
+FOR EACH ROW
+BEGIN
+  UPDATE mangas
+  SET last_modified_at = strftime('%s', 'now')
+  WHERE _id = new._id;
+END;
+
+DROP TRIGGER IF EXISTS update_favorite_modified_at_mangas;
+CREATE TRIGGER update_last_favorited_at_mangas
+AFTER UPDATE OF favorite ON mangas
+BEGIN
+  UPDATE mangas
+  SET favorite_modified_at = strftime('%s', 'now')
+  WHERE _id = new._id;
+END;
+
+DROP TRIGGER IF EXISTS update_last_modified_at_chapters;
+CREATE TRIGGER update_last_modified_at_chapters
+AFTER UPDATE ON chapters
+FOR EACH ROW
+BEGIN
+  UPDATE chapters
+  SET last_modified_at = strftime('%s', 'now')
+  WHERE _id = new._id;
+END;
+
+DROP TRIGGER IF EXISTS update_last_modified_at_mangas_categories;
+CREATE TRIGGER update_last_modified_at_mangas_categories
+AFTER UPDATE ON mangas_categories
+FOR EACH ROW
+BEGIN
+  UPDATE mangas_categories
+  SET last_modified_at = strftime('%s', 'now')
+  WHERE _id = new._id;
+END;