bump version

* Cleanup and update gradle, update dependencies

* Duplicate Jsoup

.github/workflows/build_push.yml

@@ -78,11 +78,14 @@ jobs:
           echo "$genTag"
           echo "::set-output name=value::$genTag"
 
-      - name: make windows packages
+      - name: make bundle packages
         run: |
           cd master/scripts
           ./windows-bundler.sh win32
           ./windows-bundler.sh win64
+          ./unix-bundler.sh linux-x64
+          ./unix-bundler.sh macOS-x64
+          ./unix-bundler.sh macOS-arm64
 
 #      - name: Mock make windows packages
 #        run: |
@@ -116,11 +119,7 @@ jobs:
         uses: ncipollo/release-action@v1
         with:
           token: ${{ secrets.DEPLOY_PREVIEW_TOKEN }}
-          artifacts: "master/server/build/*.jar,master/server/build/*.zip"
+          artifacts: "master/server/build/*.jar,master/server/build/*.zip,master/server/build/*.tar.gz"
           owner: "Suwayomi"
           repo: "Tachidesk-Server-preview"
           tag: ${{ steps.GenTagName.outputs.value }}
-
-      - name: Run Docker build workflow
-        run: |
-          curl -X POST -H "Accept: application/vnd.github.v3+json" -H "Authorization: token ${{ secrets.DEPLOY_PREVIEW_TOKEN }}" -d '{"ref":"main", "inputs":{"tachidesk_release_type": "preview"}}' https://api.github.com/repos/suwayomi/docker-tachidesk/actions/workflows/build_container_images.yml/dispatches

.github/workflows/docker_build_stable.yml

@@ -0,0 +1,15 @@
+name: Docker Build Stable
+
+on:
+  release:
+    types: [published]
+  
+jobs:
+  build_publish_docker_container:
+    runs-on: ubuntu-latest
+
+    steps:         
+      - name: run docker build and publish script
+        run: |
+          curl -X POST -H "Accept: application/vnd.github.v3+json" -H "Authorization: token ${{ secrets.DEPLOY_PREVIEW_TOKEN }}" -d '{"ref":"main", "inputs":{"tachidesk_release_type": "stable"}}' https://api.github.com/repos/suwayomi/docker-tachidesk/actions/workflows/build_container_images.yml/dispatches
+          

.github/workflows/publish.yml

@@ -51,13 +51,6 @@ jobs:
           cd master
           curl https://raw.githubusercontent.com/Suwayomi/Tachidesk/android-jar/android.jar -o AndroidCompat/lib/android.jar
 
-      - name: Cache node_modules
-        uses: actions/cache@v2
-        with:
-          path: |
-            **/webUI/node_modules
-          key: ${{ runner.os }}-${{ hashFiles('**/webUI/yarn.lock') }}
-
       - name: Build and copy webUI, Build Jar
         uses: eskatos/gradle-command-action@v1
         env:
@@ -70,23 +63,21 @@ jobs:
           dependencies-cache-enabled: true
           configuration-cache-enabled: true
 
-      - name: make windows packages
+      - name: make bundle packages
         run: |
           cd master/scripts
           ./windows-bundler.sh win32
           ./windows-bundler.sh win64
+          ./unix-bundler.sh linux-x64
+          ./unix-bundler.sh macOS-x64
+          ./unix-bundler.sh macOS-arm64
 
       - name: Upload Release
         uses: xresloader/upload-to-github-release@v1
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
-          file: "master/server/build/*.jar;master/server/build/*.zip"
+          file: "master/server/build/*.jar;master/server/build/*.zip;master/server/build/*.tar.gz"
           tags: true
           draft: true
           verbose: true
-
-      - name: Run Docker build workflow
-        run: |
-          curl -X POST -H "Accept: application/vnd.github.v3+json" -H "Authorization: token ${{ secrets.DEPLOY_PREVIEW_TOKEN }}" -d '{"ref":"main", "inputs":{"tachidesk_release_type": "stable"}}' https://api.github.com/repos/suwayomi/docker-tachidesk/actions/workflows/build_container_images.yml/dispatches
-          

.gitignore

@@ -11,6 +11,7 @@ server/tmp/
 server/tachiserver-data/
 
 # bundle asset downlaods
-OpenJDK*.zip
+OpenJDK*.*
-electron-*.zip
+zulu*jre*
+electron-*.*
 rcedit-*

AndroidCompat/Config/src/main/java/xyz/nulldev/ts/config/ApplicationRootDir.kt

@@ -9,11 +9,12 @@ package xyz.nulldev.ts.config
 
 import net.harawata.appdirs.AppDirsFactory
 
+const val CONFIG_PREFIX = "suwayomi.tachidesk.config"
 
 val ApplicationRootDir: String
     get(): String {
         return System.getProperty(
-                "suwayomi.tachidesk.server.rootDir",
+                "$CONFIG_PREFIX.server.rootDir",
                 AppDirsFactory.getInstance().getUserDataDir("Tachidesk", null, null)
         )
     }

AndroidCompat/Config/src/main/java/xyz/nulldev/ts/config/ConfigManager.kt

@@ -48,7 +48,7 @@ open class ConfigManager {
         val baseConfig =
                 ConfigFactory.parseMap(
                         mapOf(
-                                "ts.server.rootDir" to ApplicationRootDir
+                                "androidcompat.rootDir" to "$ApplicationRootDir/android-compat" // override AndroidCompat's rootDir
                         )
                 )
 

AndroidCompat/Config/src/main/java/xyz/nulldev/ts/config/ConfigModule.kt

@@ -14,26 +14,31 @@ import kotlin.reflect.KProperty
 /**
  * Abstract config module.
  */
-abstract class ConfigModule(config: Config, moduleName: String = "") {
+@Suppress("UNUSED_PARAMETER")
-    val overridableWithSysProperty = SystemPropertyOverrideDelegate(config, moduleName)
+abstract class ConfigModule(config: Config)
+
+/**
+ * Abstract jvm-commandline-argument-overridable config module.
+ */
+abstract class SystemPropertyOverridableConfigModule(config: Config, moduleName: String): ConfigModule(config) {
+    val overridableConfig = SystemPropertyOverrideDelegate(config, moduleName)
 }
 
+/** Defines a config property that is overridable with jvm `-D` commandline arguments prefixed with [CONFIG_PREFIX] */
 class SystemPropertyOverrideDelegate(val config: Config, val moduleName: String) {
     inline operator fun <R, reified T> getValue(thisRef: R, property: KProperty<*>): T {
         val configValue: T = config.getValue(thisRef, property)
 
         val combined = System.getProperty(
-            "suwayomi.tachidesk.config.$moduleName.${property.name}",
+            "$CONFIG_PREFIX.$moduleName.${property.name}",
             configValue.toString()
         )
 
-        val asT = when(T::class.simpleName) {
+        return when(T::class.simpleName) {
             "Int" -> combined.toInt()
             "Boolean" -> combined.toBoolean()
             // add more types as needed
-            else -> combined
+            else -> combined // covers String
-        }
+        } as T
-
-        return asT as T
     }
 }

AndroidCompat/build.gradle.kts

@@ -1,68 +1,28 @@
-
-plugins {
-    application
-    kotlin("plugin.serialization")
-}
-
-
-repositories {
-    mavenCentral()
-    maven {
-        url = uri("https://jitpack.io")
-    }
-
-    maven {
-        url = uri("https://maven.google.com")
-    }
-}
-
 dependencies {
     // Android stub library
     implementation(fileTree("lib/"))
 
-    // JSON
-    compileOnly("com.google.code.gson:gson:2.8.6")
-
     // XML
-    compileOnly(group= "xmlpull", name= "xmlpull", version= "1.1.3.1")
+    compileOnly("xmlpull:xmlpull:1.1.3.4a")
 
     // Config API
     implementation(project(":AndroidCompat:Config"))
 
     // APK sig verifier
-    compileOnly("com.android.tools.build:apksig:4.2.0-alpha13")
+    compileOnly("com.android.tools.build:apksig:7.1.0-alpha12")
 
     // AndroidX annotations
-    compileOnly("androidx.annotation:annotation:1.2.0-alpha01")
+    compileOnly("androidx.annotation:annotation:1.2.0")
 
     // substitute for duktape-android
     implementation("org.mozilla:rhino-runtime:1.7.13") // slimmer version of 'org.mozilla:rhino'
     implementation("org.mozilla:rhino-engine:1.7.13") // provides the same interface as 'javax.script' a.k.a Nashorn
 
     // Kotlin wrapper around Java Preferences, makes certain things easier
-    val multiplatformSettingsVersion = "0.7.7"
+    val multiplatformSettingsVersion = "0.8"
     implementation("com.russhwolf:multiplatform-settings-jvm:$multiplatformSettingsVersion")
     implementation("com.russhwolf:multiplatform-settings-serialization-jvm:$multiplatformSettingsVersion")
 
     // Android version of SimpleDateFormat
     implementation("com.ibm.icu:icu4j:69.1")
 }
-
-tasks {
-    withType<org.jetbrains.kotlin.gradle.tasks.KotlinCompile> {
-        kotlinOptions.freeCompilerArgs = listOf("-Xopt-in=kotlin.RequiresOptIn")
-    }
-}
-
-//def fatJarTask = tasks.getByPath(':AndroidCompat:JVMPatch:fatJar')
-//
-//// Copy JVM core patches
-//task copyJVMPatches(type: Copy) {
-//    from fatJarTask.outputs.files
-//    into 'src/main/resources/patches'
-//}
-//
-//compileOnly(Java.dependsOn gradle.includedBuild('dex2jar').task(':dex-translator:assemble')
-//compileOnly(Java.dependsOn copyJVMPatches
-//copyJVMPatches.dependsOn fatJarTask
-//

AndroidCompat/src/main/java/androidx/preference/TwoStatePreference.java

@@ -13,7 +13,10 @@ import com.fasterxml.jackson.annotation.JsonIgnore;
 public class TwoStatePreference extends Preference {
     // Note: remove @JsonIgnore and implement methods if any extension ever uses these methods or the variables behind them
 
-    public TwoStatePreference(Context context) { super(context); }
+    public TwoStatePreference(Context context) {
+        super(context);
+        setDefaultValue(false);
+    }
 
     @JsonIgnore
     public boolean isChecked() { throw new RuntimeException("Stub!"); }

AndroidCompat/src/main/java/xyz/nulldev/androidcompat/config/ApplicationInfoConfigModule.kt

@@ -1,6 +1,7 @@
 package xyz.nulldev.androidcompat.config
 
 import com.typesafe.config.Config
+import io.github.config4k.getValue
 import xyz.nulldev.ts.config.ConfigModule
 
 /**
@@ -8,8 +9,8 @@ import xyz.nulldev.ts.config.ConfigModule
  */
 
 class ApplicationInfoConfigModule(config: Config) : ConfigModule(config) {
-    val packageName = config.getString("packageName")!!
+    val packageName: String by config
-    val debug = config.getBoolean("debug")
+    val debug: Boolean by config
 
     companion object {
         fun register(config: Config)

AndroidCompat/src/main/java/xyz/nulldev/androidcompat/config/FilesConfigModule.kt

@@ -1,6 +1,7 @@
 package xyz.nulldev.androidcompat.config
 
 import com.typesafe.config.Config
+import io.github.config4k.getValue
 import xyz.nulldev.ts.config.ConfigModule
 
 /**
@@ -8,23 +9,23 @@ import xyz.nulldev.ts.config.ConfigModule
  */
 
 class FilesConfigModule(config: Config) : ConfigModule(config) {
-    val dataDir = config.getString("dataDir")!!
+    val dataDir:String by config
-    val filesDir = config.getString("filesDir")!!
+    val filesDir:String by config
-    val noBackupFilesDir = config.getString("noBackupFilesDir")!!
+    val noBackupFilesDir:String by config
-    val externalFilesDirs: MutableList<String> = config.getStringList("externalFilesDirs")!!
+    val externalFilesDirs: MutableList<String> by config
-    val obbDirs: MutableList<String> = config.getStringList("obbDirs")!!
+    val obbDirs: MutableList<String> by config
-    val cacheDir = config.getString("cacheDir")!!
+    val cacheDir:String by config
-    val codeCacheDir = config.getString("codeCacheDir")!!
+    val codeCacheDir:String by config
-    val externalCacheDirs: MutableList<String> = config.getStringList("externalCacheDirs")!!
+    val externalCacheDirs: MutableList<String> by config
-    val externalMediaDirs: MutableList<String> = config.getStringList("externalMediaDirs")!!
+    val externalMediaDirs: MutableList<String> by config
-    val rootDir = config.getString("rootDir")!!
+    val rootDir:String by config
-    val externalStorageDir = config.getString("externalStorageDir")!!
+    val externalStorageDir:String by config
-    val downloadCacheDir = config.getString("downloadCacheDir")!!
+    val downloadCacheDir:String by config
-    val databasesDir = config.getString("databasesDir")!!
+    val databasesDir:String by config
 
-    val prefsDir = config.getString("prefsDir")!!
+    val prefsDir:String by config
 
-    val packageDir = config.getString("packageDir")!!
+    val packageDir:String by config
 
     companion object {
         fun register(config: Config)

AndroidCompat/src/main/java/xyz/nulldev/androidcompat/config/SystemConfigModule.kt

@@ -2,9 +2,10 @@ package xyz.nulldev.androidcompat.config
 
 import com.typesafe.config.Config
 import xyz.nulldev.ts.config.ConfigModule
+import io.github.config4k.getValue
 
 class SystemConfigModule(val config: Config) : ConfigModule(config) {
-    val isDebuggable = config.getBoolean("isDebuggable")
+    val isDebuggable: Boolean by config
 
     val propertyPrefix = "properties."
 

AndroidCompat/src/main/resources/compat-reference.conf

@@ -1,36 +1,12 @@
+# AndroidComapt Root dir
+androidcompat.rootDir = androidcompat-root
 
-# Allow/disallow preference changes (useful for demos)
+####################### `android.files` (FilesConfigModule) #######################
-ts.server.allowConfigChanges = true
-
-# Enable the WebUI? Note: The API and multi-user sync server ui will remain available even if the WebUI is disabled
-ts.server.enableWebUi = true
-
-# 'true' to use the old, buggy/memory-leaking WebUI
-ts.server.useOldWebUi = false
-
-# 'true' to pretty print all JSON API responses
-ts.server.prettyPrintApi = false
-
-# List of blacklisted/whitelisted API endpoints/operation IDs
-ts.server.disabledApiEndpoints = []
-ts.server.enabledApiEndpoints = []
-
-# Message to print in the console when the API has finished booting
-ts.server.httpInitializedPrintMessage = ""
-
-# Use external folder for static files
-ts.server.useExternalStaticFiles = false
-ts.server.externalStaticFilesFolder = ""
-
-# Root storage dir
-ts.server.rootDir = tachiserver-data
-# Dir to store JVM patches
-ts.server.patchesDir = ${ts.server.rootDir}/patches
 
 # Storage dir for the emulated Android app
-android.files.rootDir = ${ts.server.rootDir}/android-compat/appdata
+android.files.rootDir = ${androidcompat.rootDir}/appdata
 # External storage dir for the emulated Android app's
-android.files.externalStorageDir = ${ts.server.rootDir}/android-compat/extappdata
+android.files.externalStorageDir = ${androidcompat.rootDir}/extappdata
 
 # Internal Android directories
 android.files.dataDir = ${android.files.rootDir}/data
@@ -48,37 +24,16 @@ android.files.externalCacheDirs = [${android.files.externalStorageDir}/cache]
 android.files.externalMediaDirs = [${android.files.externalStorageDir}/media]
 android.files.downloadCacheDir = ${android.files.externalStorageDir}/downloadCache
 
-android.files.packageDir = ${ts.server.rootDir}/android-compat/packages
+android.files.packageDir = ${androidcompat.rootDir}/android-compat/packages
+
+####################### `android.app` (ApplicationInfoConfigModule) #######################
 
 # Emulated Android app package name
 android.app.packageName = eu.kanade.tachiyomi
 # Debug mode for the emulated Android app
 android.app.debug = true
 
+####################### `android.system` (SystemConfigModule) #######################
+
 # Whether or not the emulated Android system is debuggable
 android.system.isDebuggable = true
-
-# Is the multi-user sync server enabled? Does not affect the single-user sync server included in the API.
-ts.syncd.enable = false
-
-# The URL of this server (displayed in the sync server web ui)
-ts.syncd.baseUrl = "http://example.com"
-
-# 'true' to disable the API and only enable the multi-user sync server
-ts.syncd.syncOnlyMode = false
-
-# The root directory to store synchronized data
-ts.syncd.rootDir = ${ts.server.rootDir}/sync/accounts
-
-# Location to store config files for the sandbox
-ts.syncd.sandboxedConfig = ${ts.server.rootDir}/sync/sandboxed_config.config
-
-# Recaptcha stuff for signup/login
-ts.syncd.recaptcha.siteKey = ""
-ts.syncd.recaptcha.secret = ""
-
-# Sync server display name
-ts.syncd.name = "Tachiyomi sync server"
-
-# Header used to forward the IP to the multi-user sync server if the server is behind a reverse proxy
-ts.syncd.ipHeader = ""

CHANGELOG-TEMPLATE.md

@@ -0,0 +1,28 @@
+# Server: v0.X.Y-rXXX + WebUI: rXXX
+## TL;DR
+<!-- TODO: fill before release -->
+
+## Tachidesk-Server
+### Public API
+#### Non-breaking changes
+- N/A
+
+#### Breaking changes
+- N/A
+
+#### Bug fixes
+- N/A
+
+### Private API
+- N/A
+
+
+## Tachidesk-WebUI
+#### Visible changes
+- N/A
+
+#### Bug fixes
+- N/A
+
+#### Internal changes
+- N/A

CHANGELOG.md

@@ -0,0 +1,212 @@
+# Server: v0.5.2 + WebUI: r807
+## TL;DR
+- Fixed Local source not working on Windows
+- Fixed Chapter numbers being shown incorrectly
+
+## Tachidesk-Server
+### Public API
+#### Non-breaking changes
+- N/A
+
+#### Breaking changes
+- N/A
+
+#### Bug fixes
+- N/A
+
+### Private API
+- (r942) Gradle Updates ([#199](https://github.com/Suwayomi/Tachidesk-WebUI/pull/199) by @Syer10)
+- (r941) Update BytecodeEditor to use Java NIO Paths ([#200](https://github.com/Suwayomi/Tachidesk-WebUI/pull/200) by @Syer10)
+
+
+## Tachidesk-WebUI
+#### Visible changes
+- (r804) update text positioning on Reader and Player ([#35](https://github.com/Suwayomi/Tachidesk-WebUI/pull/35) by @voltrare)
+- (r806) Source card for Local source is different
+- (r807) add Local source guide
+
+#### Bug fixes
+- (r805) fix chapter name
+
+#### Internal changes
+- N/A
+
+
+
+# Server: v0.5.1 + WebUI: r803
+## TL;DR
+- Loading sources' manga list is at least twice as fast
+- Added support for Tachiyomi's Local source
+- Added BasicAuth support, now you can protect your Tachidesk instance if you are running it on a public server
+- Added ability to turn off cache for image requests
+
+## Tachidesk-Server
+### Public API
+#### Non-breaking changes
+- (r915) add BasicAuth support
+- (r918) add ability to delete downloaded chapters
+- (r923-r930) add Local Source
+- (r938) add ability to turn off cache for image requests
+
+#### Breaking changes
+- N/A
+
+#### Bug fixes
+- (r917) detect if a downloaded chapter is missing
+
+### Private API
+- (r913) remove expand char limit on MangaTable columns
+- (r914) migrate to Javalin 4
+- (r921) depricate zero based chapters
+- (r937) add ChapterRecognition from tachiyomi, closes #10
+
+
+
+## Tachidesk-WebUI
+#### Visible changes
+- (r790) nice looking progress percentage
+- (r791) show a Delete button for downloaded chapters
+- (r792) Update hover effect using more of Material-UI color pallete ([#29](https://github.com/Suwayomi/Tachidesk-WebUI/pull/29) by @voltrare)
+- (r793) Optimize images ([#32](https://github.com/Suwayomi/Tachidesk-WebUI/pull/32) by @phanirithvij)
+- (r794) try fix #30 ([#31](https://github.com/Suwayomi/Tachidesk-WebUI/pull/31) by @phanirithvij)
+- (r795) fix viewing page number when the string is long
+- (r796) show proper display name for source
+- (r797) fail gracefully when a thumbnail has errors
+- (r798) fix when a source fails to load mangas
+- (r800) add Local source ([#31](https://github.com/Suwayomi/Tachidesk-WebUI/pull/31))
+- (r803) add support for useCache
+
+#### Bug fixes
+- N/A
+
+#### Internal changes
+- N/A
+
+
+
+# Server: v0.5.0 + WebUI: r789
+## TL;DR
+- You can now install APK extensions from the extensions page
+- WebUI now comes with an updated Material Design looks and is faster a little bit.
+- WebUI now shows Nsfw content by default, disable it in settings if you prefer to not see Nsfw stuff
+- Added support for configuration of sources, this enables MangaDex, Komga, Cubari and many other sources
+- Chapters in the Manga page and Sources in the source page now look nicer and will glow with mouse hover
+
+## Tachidesk-Server
+### Public API
+#### Non-breaking changes
+- (r888) add installing APK from external sources endpoint
+
+#### Breaking changes
+- (r877 [#188](https://github.com/Suwayomi/Tachidesk-Server/pull/188) by @Syer10) `MangaDataClass.genre` changed type to `List<String>`
+
+#### Bug fixes
+- (r899-r901) fix when an external apk is installed and it doesn't have the default tachiyomi-extensions name
+- (r905) fix a bug where if two sources return the same URL, a false duplicate might be detected
+
+### Private API
+- (r887) the `run` task won't call `downloadWebUI` now
+- (r902) cleanup print/ln instances
+- (r906) better handling of uninstalling Extensions
+
+## Tachidesk-WebUI
+#### Visible changes
+- (r770) add support for the new genre type
+- (r771) set the default value of `showNsfw` to `true` so we won't have visual artifacts with a clean install
+- (r774 [#21](https://github.com/Suwayomi/Tachidesk-WebUI/pull/21) by @voltrare) `ReaderNavbar.jsx`: Swap close and retract Navbar buttons
+- (r775 [#23](https://github.com/Suwayomi/Tachidesk-WebUI/pull/23) by @voltrare) `yarn.lock`: Fixes version inconsistency after commit 9b866811b
+- (r776 [#23](https://github.com/Suwayomi/Tachidesk-WebUI/pull/23) by @voltrare) add margin between Source and Extension cards, make the Search button look nicer
+- (r777) add support for installing external APK files
+- (r778) fix the makeToaster?
+- (r779) Action button for installing external extension
+- (r780 Suwayomi/Tachidesk-WebUI#25) add on hover, active effect to Chapter/Episode card
+- (r782-r785) updating material-ui to v5 changed the theme
+- (r785-r788) better `SourceCard` looks on mobile, move `SourceDataClass.isConfigurable` gear button to `SourceMangas`
+- (r789) implement source configuration
+
+#### Bug fixes
+- N/A
+
+#### Internal changes
+- (r782-r785) update dependencies, migrate material-ui from v4 to v5
+
+
+
+# Server: v0.4.9 + WebUI: r769
+## Tachidesk-Server
+### Public API
+#### Non-breaking changes
+- N/A
+
+#### Breaking changes
+- (r857) renamed: `SourceDataClass.isNSFW` -> `SourceDataClass.isNsfw`
+
+#### Bug fixes
+- N/A
+
+### Private API
+- (r850) Bump WebUI version to r767
+- (r861) Bump WebUI version to r769
+
+#### Non-code changes
+- (r851) Add this changelog file and `CHANGELOG-TEMPLATE.md`
+- (r852-r853) `CONTRIBUTING.md`: Add a note about this maintaining this file changelog
+- (r855) `CONTRIBUTING.md`: text cleanup
+- (r859) `CONTRIBUTING.md`: remove dumb rule
+- (r862) `windows-bundler.sh`: update jre
+- (r864) add linux and macOS bundler script and launcher scripts
+- (r865) fix macOS bundler script and launcher scripts
+- (r866) bump electron version to v14.0.0
+- (r868) add linux and macOS bundlers to the publish workflow
+- (r871) `publish.yml`: remove node module cache, won't need it anymore
+- (r873) `publish.yml` and `build_push.yml`: fix oopsies
+
+
+## Tachidesk-WebUI
+#### Visible changes
+- (r767-r769) Support for hiding NSFW content in settings screen, extensions screen, sources screen
+
+#### Bug fixes
+- N/A
+
+#### Internal changes
+- (r767) Remove some duplicate dependency declaration from `package.json`
+
+#### Non-code changes
+- (r42-r45) Change `README.md`: some links and stuff 
+- (r45-r765) Add all of the commit history from when WebUI was separated from Server, jumping from r45 to r765 (r45 is exactly the same as r765)
+- (r766) Steal `.gitattributes` from Tachidesk-Server
+- (r767) Dependency cleanup in `package.json`
+
+
+
+
+# Server: v0.4.8 + WebUI: r41
+## Tachidesk-Server
+### Public API
+#### Non-breaking changes
+- Added support for serializing Search Filters
+- `SourceDataClass` now has a `isNsfw` key
+
+#### Breaking changes
+- N/A
+
+#### Bug fixes
+- Fixed a bug where backup restore reversed chapter order
+- Open Site feature now works properly (https://github.com/Suwayomi/Tachidesk-WebUI/issues/19)
+
+### Private API
+- Added `CloudflareInterceptor` from TachiWeb-Server
+- Restoring backup for mangas in library(merging manga data) is now supported
+
+## Tachidesk-WebUI
+#### Visible changes
+- Better looking manga card titles
+- Better reader title, next, prev buttons
+
+#### Bug fixes
+- Open Site feature now works properly (https://github.com/Suwayomi/Tachidesk-WebUI/issues/19)
+- Re-ordering categories now works
+
+#### Internal changes
+- N/A

CONTRIBUTING.md

@@ -2,51 +2,38 @@
 ## Where should I start?
 Checkout [This Kanban Board](https://github.com/Suwayomi/Tachidesk/projects/1) to see the rough development roadmap.
 
-**Note to potential contributors:** Notify the developers on Suwayomi discord (#programming channel) or open a WIP pull request before starting if you decide to take on working on anything from/not from the roadmap in order to avoid parallel efforts on the same issue/feature.
+**Note 1:** Notify the developers on [Suwayomi discord](https://discord.gg/DDZdqZWaHA) (#programming channel) or open a WIP pull request before starting if you decide to take on working on anything from/not from the roadmap in order to avoid parallel efforts on the same issue/feature.
 
-## How does Tachidesk work?
+**Note 2:** Store all changes with each direct commit/PR in [CHANGELOG.md](./CHANGELOG.md).
+
+## How does Tachidesk-Server work?
 This project has two components: 
-1. **server:** contains the implementation of [tachiyomi's extensions library](https://github.com/tachiyomiorg/extensions-lib) and uses an Android compatibility library to run apk extensions. All this concludes to serving a REST API to `webUI`.
+1. **Server:** contains the implementation of [tachiyomi's extensions library](https://github.com/tachiyomiorg/extensions-lib) and uses an Android compatibility library to run jar libraries converted from apk extensions. All this concludes to serving a REST API.
-2. **webUI:** A react SPA(`create-react-app`) project that works with the server to do the presentation.
+2. **WebUI:** A React SPA(`create-react-app`) project that works with the server to do the presentation located at https://github.com/Suwayomi/Tachidesk-WebUI
 
 ## Why a web app?
 This structure is chosen to
 - Achieve the maximum multi-platform-ness
-- Gives the ability to acces Tachidesk from a remote web browser e.g. your phone, tablet or smart TV
+- Gives the ability to access Tachidesk-Server from a remote client e.g., your phone, tablet or smart TV
-- Eaise development of alternative user intefaces for Tachidesk
+- Ease development of user interfaces for Tachidesk
-
-## User Interfaces for Tachidesk server
-Currently, there are three known interfaces for Tachidesk:
-1. [webUI](https://github.com/Suwayomi/Tachidesk/tree/master/webUI/react): The react SPA that Tachidesk is traditionally shipped with.
-2. [TachideskJUI](https://github.com/Suwayomi/TachideskJUI): A Jetbrains Compose Native app, re-uses components made for the upcoming Tachiyomi 1.x
-3. [Equinox](https://github.com/Suwayomi/Equinox): A web user interface made with Vue.js, in super early stages of development.
 
 ## Building from source
 ### Prerequisites
 You need these software packages installed in order to build the project
-### Server
+
 - Java Development Kit and Java Runtime Environment version 8 or newer(both Oracle JDK and OpenJDK works)
 - Android stubs jar
-    - Manual download: Download [android.jar](https://raw.githubusercontent.com/Suwayomi/Tachidesk/android-jar/android.jar) and put it under `AndroidCompat/lib`.
+    - **Manual download:** Download [android.jar](https://raw.githubusercontent.com/Suwayomi/Tachidesk/android-jar/android.jar) and put it under `AndroidCompat/lib`.
-    - Automated download: Run `AndroidCompat/getAndroid.sh`(MacOS/Linux) or `AndroidCompat/getAndroid.ps1`(Windows) from project's root directory to download and rebuild the jar file from Google's repository.
+    - **Automated download:** Run `AndroidCompat/getAndroid.sh`(macOS/Linux) or `AndroidCompat/getAndroid.ps1`(Windows) from project's root directory to download and rebuild the jar file from Google's repository.
-### webUI
-- Nodejs LTS or latest
-- Yarn
-- Git
-### building the full-blown jar
-Run `./gradlew :webUI:copyBuild server:shadowJar`, the resulting built jar file will be `server/build/Tachidesk-vX.Y.Z-rxxx.jar`.
-### building without `webUI` bundled(server only)
-Delete the `server/src/main/resources/react` directory if exists from previous runs, then run `./gradlew server:shadowJar`, the resulting built jar file will be `server/build/Tachidesk-vX.Y.Z-rxxx.jar`.
-### building the Windows package
-First Build the jar, then cd into the `scripts` directory and run `./windows<bits>-bundler.sh` (or `./windows<bits>-bundler.ps1` if you are on windows), the resulting built zip package file will be `server/build/Tachidesk-vX.Y.Z-rxxx-win64.zip`.
-## Running in development mode
-First satisfy [the prerequisites](#prerequisites)
-### server
-run `./gradlew :server:run --stacktrace` to run the server
-### webUI
-How to do it is described in `webUI/react/README.md` but for short,
- first cd into `webUI/react` then run `yarn` to install the node modules(do this only once)
- then `yarn start` to start the development server, if a new browser window doesn't get opened automatically,
- then open `http://127.0.0.1:3000` in a modern browser. This is a `create-react-app` project
- and supports HMR and all the other goodies you'll need.
 
+### building the full-blown jar (Tachidesk-Server + Tachidesk-WebUI bundle)
+Run `./gradlew server:downloadWebUI server:shadowJar`, the resulting built jar file will be `server/build/Tachidesk-Server-vX.Y.Z-rxxx.jar`.
+
+### building without `webUI` bundled (server only)
+Delete `server/src/main/resources/WebUI.zip` if exists from previous runs, then run `./gradlew server:shadowJar`, the resulting built jar file will be `server/build/Tachidesk-Server-vX.Y.Z-rxxx.jar`.
+
+### building the Windows package
+First Build the jar, then cd into the `scripts` directory and run `./windows-bundler.sh win32` or `./windows-bundler.sh win64` depending on the target architecture, the resulting built zip package file will be `server/build/Tachidesk-Server-vX.Y.Z-rxxx-winXX.zip`.
+
+## Running in development mode
+run `./gradlew :server:run --stacktrace` to run the server

README.md

@@ -3,48 +3,83 @@
 |-------|----------|---------|---------|
 | ![CI](https://github.com/Suwayomi/Tachidesk/actions/workflows/build_push.yml/badge.svg) | [![stable release](https://img.shields.io/github/release/Suwayomi/Tachidesk.svg?maxAge=3600&label=download)](https://github.com/Suwayomi/Tachidesk/releases) | [![preview](https://img.shields.io/badge/dynamic/json?url=https://github.com/Suwayomi/Tachidesk-preview/raw/main/index.json&label=download&query=$.latest&color=blue)](https://github.com/Suwayomi/Tachidesk-preview/releases/latest) | [![Discord](https://img.shields.io/discord/801021177333940224.svg?label=discord&labelColor=7289da&color=2c2f33&style=flat)](https://discord.gg/DDZdqZWaHA) |
 
-# Tachidesk
+# Tachidesk-Server is a server app! You may not want to Download Tachidesk-Server directly.
+Yes, you need a client/user interface app as a front-end for Tachidesk-Server, if you Directly Download Tachidesk-Server you'll get a bundled version of [Tachidesk-WebUI](https://github.com/Suwayomi/Tachidesk-WebUI) with it.
+
+Here's a list of known clients/user interfaces for Tachidesk-Server:
+- [Tachidesk-JUI](https://github.com/Suwayomi/Tachidesk-JUI): The "official" front-end for Tachidesk-Server, A native desktop Application.
+- [Tachidesk-WebUI](https://github.com/Suwayomi/Tachidesk-WebUI): The web/electrion front-end that Tachidesk-Server is traditionally shipped with.
+- [Tachidesk-qtui](https://github.com/Suwayomi/Tachidesk-qtui): A C++/Qt front-end for mobile devices(Android/linux), in super early stage of development.
+- [Equinox](https://github.com/Suwayomi/Equinox): A web user interface made with Vue.js, in super early stage of development.
+
+# What is Tachidesk then?
 <img src="https://github.com/Suwayomi/Tachidesk/raw/master/server/src/main/resources/icon/faviconlogo.png" alt="drawing" width="200"/>
 
-A free and open source manga reader that runs extensions built for [Tachiyomi](https://tachiyomi.org/). 
+A free and open source manga reader server that runs extensions built for [Tachiyomi](https://tachiyomi.org/). 
 
 Tachidesk is an independent Tachiyomi compatible software and is **not a Fork of** Tachiyomi.
 
-Tachidesk is as multi-platform as you can get. Any platform that runs java and/or has a modern browser can run it. This includes Windows, Linux, macOS, chrome OS, etc. Follow [Downloading and Running the app](#downloading-and-running-the-app) for installation instructions.
+Tachidesk-Server is as multi-platform as you can get. Any platform that runs java and/or has a modern browser can run it. This includes Windows, Linux, macOS, chrome OS, etc. Follow [Downloading and Running the app](#downloading-and-running-the-app) for installation instructions.
 
 Ability to read and write Tachiyomi compatible backups and syncing is a planned feature.
 
-**Tachidesk needs serious front-end dev help for it's reader and other parts, if you like the app and want to see it become better please don't hesitate to contribute some code!**
-
 ## Is this application usable? Should I test it?
 Here is a list of current features:
 
-- Installing and executing Tachiyomi's Extensions, So you'll get the same sources.
+- From Tachiyomi
-- A library to save your mangas and categories to put them into.
+    - Installing and executing Tachiyomi's Extensions, So you'll get the same sources
-- Searching and browsing installed sources.
+    - A library to save your mangas and categories to put them into
-- A decent chapter reader.
+    - Searching and browsing installed sources
-- Ability to download Manga for offline read
+    - Ability to download Manga for offline read
-- Backup and restore support powered by Tachiyomi Legacy Backups
+    - Backup and restore support powered by Tachiyomi Backups
+- From Aniyomi
+    - Installing and executing Aniyomi's Extensions
+    - Searching and browsing installed sources.
+    - Viewing an anime and it's episodes
 
-**Note:** Keep in mind that Tachidesk is alpha software and can break rarely and/or with each update. See [Troubleshooting](https://github.com/Suwayomi/Tachidesk/wiki/Troubleshooting) if it happens.
+**Note:** These are capabilities of Tachidesk-Server, the actual working support is provided by each front-end app, checkout their respective readme for more info.
 
-## Downloading and Running the app
+**Note:** Tachidesk-Server is alpha software and can break rarely and/or with each update. See [Troubleshooting](https://github.com/Suwayomi/Tachidesk-Server/wiki/Troubleshooting) if it happens.
-### All Operating Systems
-You should have The Java Runtime Environment(JRE) 8 or newer and a modern browser installed(Google is your friend for seeking assitance). Also an internet connection is required as almost everything this app does is downloading stuff. 
 
-Download the latest "Stable" jar release from [the releases section](https://github.com/Suwayomi/Tachidesk/releases) or a preview jar build from [the preview repository](https://github.com/Suwayomi/Tachidesk-preview/releases).
+# Downloading and Running the app
+## General Requirements
+In order to use the app effectively you need the following:
+- The jar release of Tachideesk-Server
+- The Java Runtime Environment(JRE) 8 or newer (included in bundle releases)
+- A Modern Browser like Google Chrome, Firefox, etc.
+- ElectronJS (optional) (included in bundle releases)
+- An internet connection (when you want to use online features)
+## Using the jar release directly 
+Download the latest `.jar` release from [the releases section](https://github.com/Suwayomi/Tachidesk-Server/releases) or a preview jar build from [the preview repository](https://github.com/Suwayomi/Tachidesk-preview/releases).
 
-Double click on the jar file or run `java -jar Tachidesk-vX.Y.Z-rxxx.jar` (or `java -jar Tachidesk-latest.jar` if you have the latest preview) from a Terminal/Command Prompt window to run the app which will open a new browser window automatically. Also the System Tray Icon is your friend if you need to open the browser window again or close Tachidesk.
+Make sure you have The Java Runtime Environment installed on your system, Double click on the jar file or run `java -jar Tachidesk-vX.Y.Z-rxxx.jar` (or `java -jar Tachidesk-latest.jar` if you have the latest preview) from a Terminal/Command Prompt window to run the app which will open a new browser window automatically. Also the System Tray Icon is your friend if you need to open the browser window again or close Tachidesk.
+
+## Using Operating System Specific Bundles
+To facilitate the use of Tachidesk we provide bundle releases that include The Java Runtime Environment, ElectronJS and 3 Tachidesk Launcher Scripts.
+
+#### Launcher Scripts
+- `Tachidesk Electron Launcher`: Launches Tachidesk inside Electron as a desktop applicaton
+- `Tachidesk Browser Launcher`: Launches Tachidesk in a browser window
+- `Tachidesk Debug Launcher`: Launches Tachidesk with debug logs attached. If Tachidesk doesn't work for you, running this can give you insight into why.
+
+**Node:** Linux launcher scripts are named a bit differently but work the same. 
 
 ### Windows
-Download the latest "Stable" win32 or win64 (depending on your system, usually you want win64) release from [the releases section](https://github.com/Suwayomi/Tachidesk/releases) or a preview one from [the preview repository](https://github.com/Suwayomi/Tachidesk-preview/releases).
+Download the latest `win32`(Windows 32-bit) or `win64`(Windows 64-bit) release from [the releases section](https://github.com/Suwayomi/Tachidesk/releases) or a preview one from [the preview repository](https://github.com/Suwayomi/Tachidesk-preview/releases).
 
-The Windows specific build has java bundled inside, so you don't have to install java to use it. Unzip `Tachidesk-vX.Y.Z-rxxx-win64.zip` and run one of the Launcher files depending on what you want(see bellow). The rest works like the previous section.
+Unzip the downloaded file and double click on one of the launcher scripts.
-#### Windows Launchers
-- `Tachidesk Electron Launcher.bat`: Launches Tachidesk inside Electron as a desktop applicaton
-- `Tachidesk Browser Launcher.bat`: Launches Tachidesk in a browser window
-- `Tachidesk Debug Launcher.bat`: Launches Tachidesk with debug logs attached. If Tachidesk doesn't work for you, running this can give you insight into why.
 
+### macOS
+Download the latest `macOS-x64`(older macOS systems) or `macOS-arm64`(Apple M1) release from [the releases section](https://github.com/Suwayomi/Tachidesk/releases) or a preview one from [the preview repository](https://github.com/Suwayomi/Tachidesk-preview/releases).
+
+Unzip the downloaded file and double click on one of the launcher scripts.
+
+### GNU/Linux
+Download the latest `linux-x64`(x86_64) release from [the releases section](https://github.com/Suwayomi/Tachidesk/releases) or a preview one from [the preview repository](https://github.com/Suwayomi/Tachidesk-preview/releases).
+
+`tar xvf` the downloaded file and double click on one of the launcher scripts or run them using the terminal.
+
+## Other methods of getting Tachidesk
 ### Arch Linux
 You can install Tachidesk from the AUR
 ```
@@ -52,7 +87,7 @@ yay -S tachidesk
 ```
 
 ### Docker
-Check our Offical Docker release [Tachidesk Container](https://github.com/orgs/Suwayomi/packages/container/package/tachidesk) or use [arbuilder's](https://github.com/arbuilder/Tachidesk-docker) tachidesk docker repo for installation. Source code for our container is available at [docker-tachidesk](https://github.com/Suwayomi/docker-tachidesk). By default the server will be running on http://localhost:4567 open this url in your browser.
+Check our Official Docker release [Tachidesk Container](https://github.com/orgs/Suwayomi/packages/container/package/tachidesk) for running Tachidesk Server in a docker container. Source code for our container is available at [docker-tachidesk](https://github.com/Suwayomi/docker-tachidesk). By default the server will be running on http://localhost:4567 open this url in your browser.
 
 Install from the command line:
 ```
@@ -75,9 +110,9 @@ See [CONTRIBUTING.md](./CONTRIBUTING.md).
 ## Credit
 This project is a spiritual successor of [TachiWeb-Server](https://github.com/Tachiweb/TachiWeb-server), Many of the ideas and the groundwork adopted in this project comes from TachiWeb.
 
-The `AndroidCompat` module was originally developed by [@null-dev](https://github.com/null-dev) for [TachiWeb-Server](https://github.com/Tachiweb/TachiWeb-server) and is licensed under `Apache License Version 2.0`.
+The `AndroidCompat` module was originally developed by [@null-dev](https://github.com/null-dev) for [TachiWeb-Server](https://github.com/Tachiweb/TachiWeb-server) and is licensed under `Apache License Version 2.0` and `Copyright 2019 Andy Bao and contributors`.
 
-Parts of [tachiyomi](https://github.com/tachiyomiorg/tachiyomi) is adopted into this codebase, also licensed under `Apache License Version 2.0`.
+Parts of [tachiyomi](https://github.com/tachiyomiorg/tachiyomi) is adopted into this codebase, also licensed under `Apache License Version 2.0` and `Copyright 2015 Javier Tomás`.
 
 You can obtain a copy of `Apache License Version 2.0` from  http://www.apache.org/licenses/LICENSE-2.0
 

build.gradle.kts

@@ -1,8 +1,11 @@
 import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
+import org.jmailen.gradle.kotlinter.tasks.FormatTask
+import org.jmailen.gradle.kotlinter.tasks.LintTask
 
 plugins {
     kotlin("jvm") version kotlinVersion
     kotlin("plugin.serialization") version kotlinVersion
+    id("org.jmailen.kotlinter") version "3.6.0"
 }
 
 allprojects {
@@ -12,10 +15,8 @@ allprojects {
 
     repositories {
         mavenCentral()
-        maven("https://maven.google.com/")
+        google()
         maven("https://jitpack.io")
-        maven("https://oss.sonatype.org/content/repositories/snapshots/")
-        maven("https://dl.google.com/dl/android/maven2/")
     }
 }
 
@@ -27,18 +28,36 @@ val projects = listOf(
 
 configure(projects) {
     apply(plugin = "org.jetbrains.kotlin.jvm")
+    apply(plugin = "org.jetbrains.kotlin.plugin.serialization")
 
     java {
         sourceCompatibility = JavaVersion.VERSION_1_8
         targetCompatibility = JavaVersion.VERSION_1_8
     }
 
-    tasks.withType<KotlinCompile> {
+    tasks {
-        kotlinOptions {
+        withType<KotlinCompile> {
-            jvmTarget = JavaVersion.VERSION_1_8.toString()
+            kotlinOptions {
+                jvmTarget = JavaVersion.VERSION_1_8.toString()
+                freeCompilerArgs = listOf(
+                    "-Xopt-in=kotlin.RequiresOptIn",
+                    "-Xopt-in=kotlinx.coroutines.ExperimentalCoroutinesApi",
+                    "-Xopt-in=kotlinx.coroutines.InternalCoroutinesApi",
+                    "-Xopt-in=kotlinx.serialization.ExperimentalSerializationApi",
+                )
+            }
+        }
+
+        withType<LintTask> {
+            source(files("src/kotlin"))
+        }
+
+        withType<FormatTask> {
+            source(files("src/kotlin"))
         }
     }
 
+
     dependencies {
         // Kotlin
         implementation(kotlin("stdlib-jdk8"))
@@ -46,31 +65,30 @@ configure(projects) {
         testImplementation(kotlin("test-junit5"))
 
         // coroutines
-        val coroutinesVersion = "1.5.0"
+        val coroutinesVersion = "1.5.2"
         implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:$coroutinesVersion")
         implementation("org.jetbrains.kotlinx:kotlinx-coroutines-jdk8:$coroutinesVersion")
         implementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:$coroutinesVersion")
 
-        val kotlinSerializationVersion = "1.2.1"
+        val kotlinSerializationVersion = "1.3.0-RC"
         implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:$kotlinSerializationVersion")
         implementation("org.jetbrains.kotlinx:kotlinx-serialization-protobuf:$kotlinSerializationVersion")
 
-
         // Dependency Injection
-        implementation("org.kodein.di:kodein-di-conf-jvm:7.5.0")
+        implementation("org.kodein.di:kodein-di-conf-jvm:7.8.0")
 
         // Logging
-        implementation("org.slf4j:slf4j-api:1.7.30")
+        implementation("org.slf4j:slf4j-api:1.7.32")
-        implementation("ch.qos.logback:logback-classic:1.2.3")
+        implementation("ch.qos.logback:logback-classic:1.2.6")
-        implementation("io.github.microutils:kotlin-logging:2.0.6")
+        implementation("io.github.microutils:kotlin-logging:2.0.11")
 
         // ReactiveX
         implementation("io.reactivex:rxjava:1.3.8")
         implementation("io.reactivex:rxkotlin:1.0.0")
         implementation("com.jakewharton.rxrelay:rxrelay:1.2.0")
 
-        // JSoup
+        // dependency both in AndroidCompat and extensions, version locked by Tachiyomi app/extensions
-        implementation("org.jsoup:jsoup:1.13.1")
+        implementation("org.jsoup:jsoup:1.14.2")
 
         // dependency of :AndroidCompat:Config
         implementation("com.typesafe:config:1.4.1")
@@ -79,14 +97,15 @@ configure(projects) {
         // to get application content root
         implementation("net.harawata:appdirs:1.2.1")
 
-        // dex2jar: https://github.com/DexPatcher/dex2jar/releases/tag/v2.1-20190905-lanchon
+        // dex2jar
-        // note: watch https://github.com/ThexXTURBOXx/dex2jar for future development
+        val dex2jarVersion = "v26"
-        implementation("com.github.DexPatcher.dex2jar:dex-tools:v2.1-20190905-lanchon")
+        implementation("com.github.ThexXTURBOXx.dex2jar:dex-translator:$dex2jarVersion")
+        implementation("com.github.ThexXTURBOXx.dex2jar:dex-tools:$dex2jarVersion")
 
         // APK parser
         implementation("net.dongliu:apk-parser:2.6.10")
 
-        // Jackson
+        // dependency both in AndroidCompat and server, version locked by javalin
-        implementation("com.fasterxml.jackson.core:jackson-annotations:2.10.3")
+        implementation("com.fasterxml.jackson.core:jackson-annotations:2.12.4")
     }
 }

buildSrc/src/main/kotlin/Constants.kt

@@ -7,16 +7,16 @@ import java.io.BufferedReader
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
 
-const val kotlinVersion = "1.5.21"
+const val kotlinVersion = "1.5.30"
 
 const val MainClass = "suwayomi.tachidesk.MainKt"
 
 // should be bumped with each stable release
-val tachideskVersion = System.getenv("ProductVersion") ?: "v0.4.6"
+val tachideskVersion = System.getenv("ProductVersion") ?: "v0.5.2"
 
-val webUIRevisionTag = System.getenv("WebUIRevision") ?: "r24"
+val webUIRevisionTag = System.getenv("WebUIRevision") ?: "r807"
 
-// counts commit count on master
+// counts commits on the master branch
 val tachideskRevision = runCatching {
     System.getenv("ProductRevision") ?: Runtime
         .getRuntime()

gradle/wrapper/gradle-wrapper.properties

@@ -1,5 +1,5 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-7.1.1-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-7.2-all.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists

scripts/resources/Tachidesk Browser Launcher.command

@@ -0,0 +1,3 @@
+cd "`dirname "$0"`"
+
+./jre/Contents/Home/bin/java -jar Tachidesk.jar

scripts/resources/Tachidesk Debug Launcher.bat

@@ -1 +1,7 @@
+:: cleaner output
+@echo off
+
 jre\bin\java -Dsuwayomi.tachidesk.config.server.debugLogsEnabled=true -jar  Tachidesk.jar
+
+:: Prevent cmd from closing when Tachidesk crashes
+pause

scripts/resources/Tachidesk Debug Launcher.command

@@ -0,0 +1,3 @@
+cd "`dirname "$0"`"
+
+./jre/Contents/Home/bin/java -Dsuwayomi.tachidesk.config.server.debugLogsEnabled=true -jar  Tachidesk.jar

scripts/resources/Tachidesk Electron Launcher.command

@@ -0,0 +1,3 @@
+cd "`dirname "$0"`"
+
+./jre/Contents/Home/bin/java "-Dsuwayomi.tachidesk.config.server.webUIInterface=electron" "-Dsuwayomi.tachidesk.config.server.electronPath=electron/Electron.app/Contents/MacOS/Electron" -jar Tachidesk.jar

scripts/resources/tachidesk-browser-launcher.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+./jre/bin/java -jar Tachidesk.jar

scripts/resources/tachidesk-debug-launcher.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+./jre/bin/java -Dsuwayomi.tachidesk.config.server.debugLogsEnabled=true -jar  Tachidesk.jar

scripts/resources/tachidesk-electron-launcher.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+./jre/bin/java "-Dsuwayomi.tachidesk.config.server.webUIInterface=electron" "-Dsuwayomi.tachidesk.config.server.electronPath=electron/electron" -jar Tachidesk.jar

scripts/unix-bundler.sh

@@ -0,0 +1,89 @@
+#!/bin/bash
+
+# Copyright (C) Contributors to the Suwayomi project
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+electron_version="v14.0.0"
+
+if [ $1 = "linux-x64" ]; then
+  jre="OpenJDK8U-jre_x64_linux_hotspot_8u302b08.tar.gz"
+  jre_release="jdk8u302-b08"
+  jre_url="https://github.com/adoptium/temurin8-binaries/releases/download/$jre_release/$jre"
+  jre_dir="$jre_release-jre"
+  electron="electron-$electron_version-linux-x64.zip"
+elif [ $1 = "macOS-x64" ]; then
+  jre="OpenJDK8U-jre_x64_mac_hotspot_8u302b08.tar.gz"
+  jre_release="jdk8u302-b08"
+  jre_url="https://github.com/adoptium/temurin8-binaries/releases/download/$jre_release/$jre"
+  jre_dir="$jre_release-jre"
+  electron="electron-$electron_version-darwin-x64.zip"
+elif [ $1 = "macOS-arm64" ]; then
+  jre="zulu8.56.0.23-ca-jre8.0.302-macosx_aarch64.tar.gz"
+  jre_release="zulu8.56.0.23-ca-jre8.0.302-macosx_aarch64"
+  jre_url="https://cdn.azul.com/zulu/bin/$jre"
+  jre_dir="$jre_release"
+  electron="electron-$electron_version-darwin-arm64.zip"
+else
+  echo "Unsupported arch value: $1"
+  exit 1
+fi
+
+arch="$1"
+os=$(echo $arch | cut -d '-' -f1)
+
+echo "creating $arch bundle"
+
+jar=$(ls ../server/build/*.jar | tail -n1)
+jar_name=$(echo $jar | cut -d'/' -f4)
+release_name=$(echo $jar_name | sed 's/.jar//')-$arch
+
+
+# make release dir
+mkdir $release_name
+
+
+echo "Dealing with jre..."
+if [ ! -f $jre ]; then
+  curl -L $jre_url -o $jre
+fi
+tar xvf $jre
+mv $jre_dir $release_name/jre
+
+echo "Dealing with electron"
+if [ ! -f $electron ]; then
+  curl -L "https://github.com/electron/electron/releases/download/$electron_version/$electron" -o $electron
+fi
+unzip $electron -d $release_name/electron
+
+# copy artifacts
+cp $jar $release_name/Tachidesk.jar
+if [ $os = linux ]; then
+  cp "resources/tachidesk-browser-launcher.sh" $release_name
+  cp "resources/tachidesk-debug-launcher.sh" $release_name
+  cp "resources/tachidesk-electron-launcher.sh" $release_name
+elif [ $os = macOS ]; then
+  cp "resources/Tachidesk Browser Launcher.command" $release_name
+  cp "resources/Tachidesk Debug Launcher.command" $release_name
+  cp "resources/Tachidesk Electron Launcher.command" $release_name
+fi
+
+archive_name=""
+if [ $os = linux ]; then
+  archive_name=$release_name.tar.gz
+  GZIP=-9 tar cvzf $archive_name $release_name
+elif [ $os = macOS ]; then
+  archive_name=$release_name.zip
+  zip -9 -r $archive_name $release_name
+fi
+
+rm -rf $release_name
+
+# clean up from possible previous runs
+if [ -f ../server/build/$archive_name ]; then
+  rm ../server/build/$archive_name
+fi
+
+mv $archive_name ../server/build/

scripts/windows-bundler.sh

@@ -6,19 +6,23 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
-electron_version="v12.0.9"
+electron_version="v14.0.0"
 
 if [ $1 = "win32" ]; then
   jre="OpenJDK8U-jre_x86-32_windows_hotspot_8u292b10.zip"
+  jre_release="jdk8u292-b10"
+  jre_url="https://github.com/AdoptOpenJDK/openjdk8-binaries/releases/download/$jre_release/$jre"
   arch="win32"
   electron="electron-$electron_version-win32-ia32.zip"
 else
-  jre="OpenJDK8U-jre_x64_windows_hotspot_8u292b10.zip"
+  jre="OpenJDK8U-jre_x64_windows_hotspot_8u302b08.zip"
+  jre_release="jdk8u302-b08"
+  jre_url="https://github.com/adoptium/temurin8-binaries/releases/download/$jre_release/$jre"
   arch="win64"
   electron="electron-$electron_version-win32-x64.zip"
 fi
 
-jre_dir="jdk8u292-b10-jre"
+jre_dir="$jre_release-jre"
 
 echo "creating windows bundle"
 
@@ -33,7 +37,7 @@ mkdir $release_name
 
 echo "Dealing with jre..."
 if [ ! -f $jre ]; then
-  curl -L "https://github.com/AdoptOpenJDK/openjdk8-binaries/releases/download/jdk8u292-b10/$jre" -o $jre
+  curl -L $jre_url -o $jre
 fi
 unzip $jre
 mv $jre_dir $release_name/jre

server/build.gradle.kts

@@ -1,24 +1,11 @@
-
 import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar
-import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
+import de.undercouch.gradle.tasks.download.Download
-import org.jmailen.gradle.kotlinter.tasks.FormatTask
-import org.jmailen.gradle.kotlinter.tasks.LintTask
 import java.time.Instant
 
 plugins {
     application
     id("com.github.johnrengelman.shadow") version "7.0.0"
-    id("org.jmailen.kotlinter") version "3.4.3"
+    id("com.github.gmazzo.buildconfig") version "3.0.3"
-    id("com.github.gmazzo.buildconfig") version "3.0.2"
-}
-
-repositories {
-    maven {
-        url = uri("https://repo1.maven.org/maven2/")
-    }
-    maven {
-        url = uri("https://jitpack.io")
-    }
 }
 
 dependencies {
@@ -30,13 +17,14 @@ dependencies {
     implementation("com.squareup.okio:okio:2.10.0")
 
     // Javalin api
-    implementation("io.javalin:javalin:3.13.6")
+    implementation("io.javalin:javalin:4.0.0")
-    // jackson version is tied to javalin, ref: `io.javalin.core.util.OptionalDependency`
+    // jackson version locked by javalin, ref: `io.javalin.core.util.OptionalDependency`
-    implementation("com.fasterxml.jackson.core:jackson-databind:2.10.3")
+    val jacksonVersion = "2.12.4"
-    implementation("com.fasterxml.jackson.module:jackson-module-kotlin:2.10.3")
+    implementation("com.fasterxml.jackson.core:jackson-databind:$jacksonVersion")
+    implementation("com.fasterxml.jackson.module:jackson-module-kotlin:$jacksonVersion")
 
     // Exposed ORM
-    val exposedVersion = "0.31.1"
+    val exposedVersion = "0.34.1"
     implementation("org.jetbrains.exposed:exposed-core:$exposedVersion")
     implementation("org.jetbrains.exposed:exposed-dao:$exposedVersion")
     implementation("org.jetbrains.exposed:exposed-jdbc:$exposedVersion")
@@ -45,28 +33,32 @@ dependencies {
     implementation("com.h2database:h2:1.4.200")
 
     // Exposed Migrations
-    val exposedMigrationsVersion = "3.1.0"
+    implementation("com.github.Suwayomi:exposed-migrations:3.1.2")
-    implementation("com.github.Suwayomi:exposed-migrations:$exposedMigrationsVersion")
 
     // tray icon
     implementation("com.dorkbox:SystemTray:4.1")
-    implementation("com.dorkbox:Utilities:1.9")
+    implementation("com.dorkbox:Utilities:1.9") // version locked by SystemTray
-
 
     // dependencies of Tachiyomi extensions, some are duplicate, keeping it here for reference
     implementation("com.github.inorichi.injekt:injekt-core:65b0440")
     implementation("com.squareup.okhttp3:okhttp:4.9.1")
     implementation("io.reactivex:rxjava:1.3.8")
-    implementation("org.jsoup:jsoup:1.13.1")
+    implementation("org.jsoup:jsoup:1.14.2")
-    implementation("com.google.code.gson:gson:2.8.6")
+    implementation("com.google.code.gson:gson:2.8.8")
     implementation("com.github.salomonbrys.kotson:kotson:2.5.0")
 
+    // Sort
+    implementation("com.github.gpanther:java-nat-sort:natural-comparator-1.1")
 
-    // asm for fixing SimpleDateFormat (must match Dex2Jar version)
+    // asm for ByteCodeEditor(fixing SimpleDateFormat) (must match Dex2Jar version)
-    implementation("org.ow2.asm:asm-debug-all:5.0.3")
+    implementation("org.ow2.asm:asm:9.2")
 
-    // extracting zip files
+    // Disk & File
     implementation("net.lingala.zip4j:zip4j:2.9.0")
+    implementation("com.github.junrar:junrar:7.4.0")
+
+    // CloudflareInterceptor
+    implementation("net.sourceforge.htmlunit:htmlunit:2.52.0")
 
     // Source models and interfaces from Tachiyomi 1.x
     // using source class from tachiyomi commit 9493577de27c40ce8b2b6122cc447d025e34c477 to not depend on tachiyomi.sourceapi
@@ -125,28 +117,17 @@ tasks {
     shadowJar {
         manifest {
             attributes(
-                mapOf(
+                "Main-Class" to MainClass,
-                    "Main-Class" to MainClass,
+                "Implementation-Title" to rootProject.name,
-                    "Implementation-Title" to rootProject.name,
+                "Implementation-Vendor" to "The Suwayomi Project",
-                    "Implementation-Vendor" to "The Suwayomi Project",
+                "Specification-Version" to tachideskVersion,
-                    "Specification-Version" to tachideskVersion,
+                "Implementation-Version" to tachideskRevision
-                    "Implementation-Version" to tachideskRevision
-                )
             )
         }
         archiveBaseName.set(rootProject.name)
         archiveVersion.set(tachideskVersion)
         archiveClassifier.set(tachideskRevision)
     }
-    withType<KotlinCompile> {
-        kotlinOptions {
-            freeCompilerArgs = listOf(
-                "-Xopt-in=kotlin.RequiresOptIn",
-                "-Xopt-in=kotlinx.coroutines.ExperimentalCoroutinesApi",
-                "-Xopt-in=kotlinx.coroutines.InternalCoroutinesApi"
-            )
-        }
-    }
 
     test {
         useJUnit()
@@ -157,7 +138,7 @@ tasks {
     }
 
     named("run") {
-        dependsOn("formatKotlin", "lintKotlin", "downloadWebUI")
+        dependsOn(":formatKotlin", ":lintKotlin")
     }
 
     named<Copy>("processResources") {
@@ -165,7 +146,7 @@ tasks {
         mustRunAfter("downloadWebUI")
     }
 
-    register<de.undercouch.gradle.tasks.download.Download>("downloadWebUI") {
+    register<Download>("downloadWebUI") {
         src("https://github.com/Suwayomi/Tachidesk-WebUI-preview/releases/download/$webUIRevisionTag/Tachidesk-WebUI-$webUIRevisionTag.zip")
         dest("src/main/resources/WebUI.zip")
 
@@ -180,8 +161,9 @@ tasks {
                     it.readText().trim()
                 }
 
-                if (zipRevision == webUIRevisionTag)
+                if (zipRevision == webUIRevisionTag) {
                     shouldOverwrite = false
+                }
             }
 
             return shouldOverwrite
@@ -189,12 +171,4 @@ tasks {
 
         overwrite(shouldOverwrite())
     }
-
-    withType<LintTask> {
-        source(files("src/kotlin"))
-    }
-
-    withType<FormatTask> {
-        source(files("src/kotlin"))
-    }
 }

server/src/main/kotlin/eu/kanade/tachiyomi/network/CloudflareInterceptor.kt

@@ -1,177 +0,0 @@
-package eu.kanade.tachiyomi.network
-
-/*
- * Copyright (C) Contributors to the Suwayomi project
- *
- * This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
-
-// import android.annotation.SuppressLint
-// import android.content.Context
-// import android.os.Build
-// import android.os.Handler
-// import android.os.Looper
-// import android.webkit.WebSettings
-// import android.webkit.WebView
-// import android.widget.Toast
-// import eu.kanade.tachiyomi.R
-// import eu.kanade.tachiyomi.util.lang.launchUI
-// import eu.kanade.tachiyomi.util.system.WebViewClientCompat
-// import eu.kanade.tachiyomi.util.system.WebViewUtil
-// import eu.kanade.tachiyomi.util.system.isOutdated
-// import eu.kanade.tachiyomi.util.system.setDefaultSettings
-// import eu.kanade.tachiyomi.util.system.toast
-import okhttp3.Interceptor
-import okhttp3.Response
-// import uy.kohesive.injekt.injectLazy
-
-class CloudflareInterceptor() : Interceptor {
-
-//    private val handler = Handler(Looper.getMainLooper())
-
-//    private val networkHelper = NetworkHelper()
-
-    /**
-     * When this is called, it initializes the WebView if it wasn't already. We use this to avoid
-     * blocking the main thread too much. If used too often we could consider moving it to the
-     * Application class.
-     */
-//    private val initWebView by lazy {
-//        WebSettings.getDefaultUserAgent(context)
-//    }
-
-    @Synchronized
-    override fun intercept(chain: Interceptor.Chain): Response {
-        val originalRequest = chain.request()
-        return chain.proceed(originalRequest)
-
-//        if (!WebViewUtil.supportsWebView(context)) {
-//            launchUI {
-//                context.toast(R.string.information_webview_required, Toast.LENGTH_LONG)
-//            }
-//            return chain.proceed(originalRequest)
-//        }
-//
-//        initWebView
-//
-//        val response = chain.proceed(originalRequest)
-//
-//        // Check if Cloudflare anti-bot is on
-//        if (response.code != 503 || response.header("Server") !in SERVER_CHECK) {
-//            return response
-//        }
-//
-//        try {
-//            response.close()
-//            networkHelper.cookieManager.remove(originalRequest.url, COOKIE_NAMES, 0)
-//            val oldCookie = networkHelper.cookieManager.get(originalRequest.url)
-//                .firstOrNull { it.name == "cf_clearance" }
-//            resolveWithWebView(originalRequest, oldCookie)
-//
-//            return chain.proceed(originalRequest)
-//        } catch (e: Exception) {
-//            // Because OkHttp's enqueue only handles IOExceptions, wrap the exception so that
-//            // we don't crash the entire app
-//            throw IOException(e)
-//        }
-    }
-//
-// //    @SuppressLint("SetJavaScriptEnabled")
-//    private fun resolveWithWebView(request: Request, oldCookie: Cookie?) {
-//        // We need to lock this thread until the WebView finds the challenge solution url, because
-//        // OkHttp doesn't support asynchronous interceptors.
-//        val latch = CountDownLatch(1)
-//
-//        var webView: WebView? = null
-//
-//        var challengeFound = false
-//        var cloudflareBypassed = false
-//        var isWebViewOutdated = false
-//
-//        val origRequestUrl = request.url.toString()
-//        val headers = request.headers.toMultimap().mapValues { it.value.getOrNull(0) ?: "" }.toMutableMap()
-//        headers["X-Requested-With"] = WebViewUtil.REQUESTED_WITH
-//
-//        handler.post {
-//            val webview = WebView(context)
-//            webView = webview
-//            webview.setDefaultSettings()
-//
-//            // Avoid sending empty User-Agent, Chromium WebView will reset to default if empty
-//            webview.settings.userAgentString = request.header("User-Agent")
-//                ?: HttpSource.DEFAULT_USERAGENT
-//
-//            webview.webViewClient = object : WebViewClientCompat() {
-//                override fun onPageFinished(view: WebView, url: String) {
-//                    fun isCloudFlareBypassed(): Boolean {
-//                        return networkHelper.cookieManager.get(origRequestUrl.toHttpUrl())
-//                            .firstOrNull { it.name == "cf_clearance" }
-//                            .let { it != null && it != oldCookie }
-//                    }
-//
-//                    if (isCloudFlareBypassed()) {
-//                        cloudflareBypassed = true
-//                        latch.countDown()
-//                    }
-//
-//                    // HTTP error codes are only received since M
-//                    if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M &&
-//                        url == origRequestUrl && !challengeFound
-//                    ) {
-//                        // The first request didn't return the challenge, abort.
-//                        latch.countDown()
-//                    }
-//                }
-//
-//                override fun onReceivedErrorCompat(
-//                    view: WebView,
-//                    errorCode: Int,
-//                    description: String?,
-//                    failingUrl: String,
-//                    isMainFrame: Boolean
-//                ) {
-//                    if (isMainFrame) {
-//                        if (errorCode == 503) {
-//                            // Found the Cloudflare challenge page.
-//                            challengeFound = true
-//                        } else {
-//                            // Unlock thread, the challenge wasn't found.
-//                            latch.countDown()
-//                        }
-//                    }
-//                }
-//            }
-//
-//            webView?.loadUrl(origRequestUrl, headers)
-//        }
-//
-//        // Wait a reasonable amount of time to retrieve the solution. The minimum should be
-//        // around 4 seconds but it can take more due to slow networks or server issues.
-//        latch.await(12, TimeUnit.SECONDS)
-//
-//        handler.post {
-//            if (!cloudflareBypassed) {
-//                isWebViewOutdated = webView?.isOutdated() == true
-//            }
-//
-//            webView?.stopLoading()
-//            webView?.destroy()
-//        }
-//
-//        // Throw exception if we failed to bypass Cloudflare
-//        if (!cloudflareBypassed) {
-//            // Prompt user to update WebView if it seems too outdated
-//            if (isWebViewOutdated) {
-//                context.toast(R.string.information_webview_outdated, Toast.LENGTH_LONG)
-//            }
-//
-//            throw Exception(context.getString(R.string.information_cloudflare_bypass_failure))
-//        }
-//    }
-//
-//    companion object {
-//        private val SERVER_CHECK = arrayOf("cloudflare-nginx", "cloudflare")
-//        private val COOKIE_NAMES = listOf("__cfduid", "cf_clearance")
-//    }
-}

server/src/main/kotlin/eu/kanade/tachiyomi/network/NetworkHelper.kt

@@ -10,12 +10,16 @@ package eu.kanade.tachiyomi.network
 // import android.content.Context
 // import eu.kanade.tachiyomi.BuildConfig
 // import eu.kanade.tachiyomi.data.preference.PreferencesHelper
-import android.content.Context
 // import okhttp3.HttpUrl.Companion.toHttpUrl
-import okhttp3.OkHttpClient
 // import okhttp3.dnsoverhttps.DnsOverHttps
 // import okhttp3.logging.HttpLoggingInterceptor
 // import uy.kohesive.injekt.injectLazy
+import android.content.Context
+import eu.kanade.tachiyomi.network.interceptor.CloudflareInterceptor
+import eu.kanade.tachiyomi.network.interceptor.UserAgentInterceptor
+import okhttp3.OkHttpClient
+import okhttp3.logging.HttpLoggingInterceptor
+import suwayomi.tachidesk.server.serverConfig
 import java.util.concurrent.TimeUnit
 
 @Suppress("UNUSED_PARAMETER")
@@ -25,55 +29,44 @@ class NetworkHelper(context: Context) {
 
 //    private val cacheDir = File(context.cacheDir, "network_cache")
 
-    private val cacheSize = 5L * 1024 * 1024 // 5 MiB
+//    private val cacheSize = 5L * 1024 * 1024 // 5 MiB
 
-    val cookieManager = MemoryCookieJar()
+    val cookieManager = PersistentCookieJar(context)
 
-    val client by lazy {
+    private val baseClientBuilder: OkHttpClient.Builder
-        val builder = OkHttpClient.Builder()
+        get() {
-            .cookieJar(cookieManager)
+            val builder = OkHttpClient.Builder()
-//            .cache(Cache(cacheDir, cacheSize))
+                .cookieJar(cookieManager)
-            .connectTimeout(30, TimeUnit.SECONDS)
+                .connectTimeout(30, TimeUnit.SECONDS)
-            .readTimeout(5, TimeUnit.MINUTES)
+                .readTimeout(30, TimeUnit.SECONDS)
-            .writeTimeout(5, TimeUnit.MINUTES)
+                .addInterceptor(UserAgentInterceptor())
-//            .dispatcher(Dispatcher(Executors.newFixedThreadPool(1)))
 
-//            .addInterceptor(UserAgentInterceptor())
+            if (serverConfig.debugLogsEnabled) {
+                val httpLoggingInterceptor = HttpLoggingInterceptor().apply {
+                    level = HttpLoggingInterceptor.Level.BASIC
+                }
+                builder.addInterceptor(httpLoggingInterceptor)
+            }
 
-//        if (BuildConfig.DEBUG) {
+//            when (preferences.dohProvider()) {
-//            val httpLoggingInterceptor = HttpLoggingInterceptor().apply {
+//                PREF_DOH_CLOUDFLARE -> builder.dohCloudflare()
-//                level = HttpLoggingInterceptor.Level.HEADERS
+//                PREF_DOH_GOOGLE -> builder.dohGoogle()
 //            }
-//            builder.addInterceptor(httpLoggingInterceptor)
-//        }
 
-//        if (preferences.enableDoh()) {
+            return builder
-//            builder.dns(
+        }
-//                DnsOverHttps.Builder().client(builder.build())
-//                    .url("https://cloudflare-dns.com/dns-query".toHttpUrl())
-//                    .bootstrapDnsHosts(
-//                        listOf(
-//                            InetAddress.getByName("162.159.36.1"),
-//                            InetAddress.getByName("162.159.46.1"),
-//                            InetAddress.getByName("1.1.1.1"),
-//                            InetAddress.getByName("1.0.0.1"),
-//                            InetAddress.getByName("162.159.132.53"),
-//                            InetAddress.getByName("2606:4700:4700::1111"),
-//                            InetAddress.getByName("2606:4700:4700::1001"),
-//                            InetAddress.getByName("2606:4700:4700::0064"),
-//                            InetAddress.getByName("2606:4700:4700::6400")
-//                        )
-//                    )
-//                    .build()
-//            )
-//        }
 
-        builder.build()
+//    val client by lazy { baseClientBuilder.cache(Cache(cacheDir, cacheSize)).build() }
-    }
+    val client by lazy { baseClientBuilder.build() }
 
     val cloudflareClient by lazy {
         client.newBuilder()
             .addInterceptor(CloudflareInterceptor())
             .build()
     }
+
+    // Tachidesk -->
+    val cookies: PersistentCookieStore
+        get() = cookieManager.store
+    // Tachidesk <--
 }

server/src/main/kotlin/eu/kanade/tachiyomi/network/PersistentCookieJar.kt

@@ -0,0 +1,20 @@
+package eu.kanade.tachiyomi.network
+
+import android.content.Context
+import okhttp3.Cookie
+import okhttp3.CookieJar
+import okhttp3.HttpUrl
+
+// from TachiWeb-Server
+class PersistentCookieJar(context: Context) : CookieJar {
+
+    val store = PersistentCookieStore(context)
+
+    override fun saveFromResponse(url: HttpUrl, cookies: List<Cookie>) {
+        store.addAll(url, cookies)
+    }
+
+    override fun loadForRequest(url: HttpUrl): List<Cookie> {
+        return store.get(url)
+    }
+}

server/src/main/kotlin/eu/kanade/tachiyomi/network/PersistentCookieStore.kt

@@ -0,0 +1,79 @@
+package eu.kanade.tachiyomi.network
+
+import android.content.Context
+import okhttp3.Cookie
+import okhttp3.HttpUrl
+import okhttp3.HttpUrl.Companion.toHttpUrlOrNull
+import java.net.URI
+import java.util.concurrent.ConcurrentHashMap
+
+// from TachiWeb-Server
+class PersistentCookieStore(context: Context) {
+
+    private val cookieMap = ConcurrentHashMap<String, List<Cookie>>()
+    private val prefs = context.getSharedPreferences("cookie_store", Context.MODE_PRIVATE)
+
+    init {
+        for ((key, value) in prefs.all) {
+            @Suppress("UNCHECKED_CAST")
+            val cookies = value as? Set<String>
+            if (cookies != null) {
+                try {
+                    val url = "http://$key".toHttpUrlOrNull() ?: continue
+                    val nonExpiredCookies = cookies.mapNotNull { Cookie.parse(url, it) }
+                        .filter { !it.hasExpired() }
+                    cookieMap.put(key, nonExpiredCookies)
+                } catch (e: Exception) {
+                    // Ignore
+                }
+            }
+        }
+    }
+
+    @Synchronized
+    fun addAll(url: HttpUrl, cookies: List<Cookie>) {
+        val key = url.toUri().host
+
+        // Append or replace the cookies for this domain.
+        val cookiesForDomain = cookieMap[key].orEmpty().toMutableList()
+        for (cookie in cookies) {
+            // Find a cookie with the same name. Replace it if found, otherwise add a new one.
+            val pos = cookiesForDomain.indexOfFirst { it.name == cookie.name }
+            if (pos == -1) {
+                cookiesForDomain.add(cookie)
+            } else {
+                cookiesForDomain[pos] = cookie
+            }
+        }
+        cookieMap.put(key, cookiesForDomain)
+
+        // Get cookies to be stored in disk
+        val newValues = cookiesForDomain.asSequence()
+            .filter { it.persistent && !it.hasExpired() }
+            .map(Cookie::toString)
+            .toSet()
+
+        prefs.edit().putStringSet(key, newValues).apply()
+    }
+
+    @Synchronized
+    fun removeAll() {
+        prefs.edit().clear().apply()
+        cookieMap.clear()
+    }
+
+    fun remove(uri: URI) {
+        prefs.edit().remove(uri.host).apply()
+        cookieMap.remove(uri.host)
+    }
+
+    fun get(url: HttpUrl) = get(url.toUri().host)
+
+    fun get(uri: URI) = get(uri.host)
+
+    private fun get(url: String): List<Cookie> {
+        return cookieMap[url].orEmpty().filter { !it.hasExpired() }
+    }
+
+    private fun Cookie.hasExpired() = System.currentTimeMillis() >= expiresAt
+}

server/src/main/kotlin/eu/kanade/tachiyomi/network/interceptor/CloudflareInterceptor.kt

@@ -0,0 +1,110 @@
+package eu.kanade.tachiyomi.network.interceptor
+
+import com.gargoylesoftware.htmlunit.BrowserVersion
+import com.gargoylesoftware.htmlunit.WebClient
+import com.gargoylesoftware.htmlunit.html.HtmlPage
+import eu.kanade.tachiyomi.network.NetworkHelper
+import mu.KotlinLogging
+import okhttp3.Cookie
+import okhttp3.HttpUrl
+import okhttp3.Interceptor
+import okhttp3.Request
+import okhttp3.Response
+import uy.kohesive.injekt.injectLazy
+import java.io.IOException
+
+// from TachiWeb-Server
+class CloudflareInterceptor : Interceptor {
+    private val logger = KotlinLogging.logger {}
+
+    private val network: NetworkHelper by injectLazy()
+
+    @Synchronized
+    override fun intercept(chain: Interceptor.Chain): Response {
+        val originalRequest = chain.request()
+
+        logger.trace { "CloudflareInterceptor is being used." }
+
+        val response = chain.proceed(originalRequest)
+
+        // Check if Cloudflare anti-bot is on
+        if (response.code != 503 || response.header("Server") !in SERVER_CHECK) {
+            return response
+        }
+
+        logger.debug { "Cloudflare anti-bot is on, CloudflareInterceptor is kicking in..." }
+
+        return try {
+            response.close()
+            network.cookies.remove(originalRequest.url.toUri())
+
+            chain.proceed(resolveChallenge(response))
+        } catch (e: Exception) {
+            // Because OkHttp's enqueue only handles IOExceptions, wrap the exception so that
+            // we don't crash the entire app
+            throw IOException(e)
+        }
+    }
+
+    private fun resolveChallenge(response: Response): Request {
+        val browserVersion = BrowserVersion.BrowserVersionBuilder(BrowserVersion.BEST_SUPPORTED)
+            .setUserAgent(response.request.header("User-Agent") ?: BrowserVersion.BEST_SUPPORTED.userAgent)
+            .build()
+        val convertedCookies = WebClient(browserVersion).use { webClient ->
+            webClient.options.isThrowExceptionOnFailingStatusCode = false
+            webClient.options.isThrowExceptionOnScriptError = false
+            webClient.getPage<HtmlPage>(response.request.url.toString())
+            webClient.waitForBackgroundJavaScript(10000)
+            // Challenge solved, process cookies
+            webClient.cookieManager.cookies.filter {
+                // Only include Cloudflare cookies
+                it.name.startsWith("__cf") || it.name.startsWith("cf_")
+            }.map {
+                // Convert cookies -> OkHttp format
+                Cookie.Builder()
+                    .domain(it.domain.removePrefix("."))
+                    .expiresAt(it.expires?.time ?: Long.MAX_VALUE)
+                    .name(it.name)
+                    .path(it.path)
+                    .value(it.value).apply {
+                        if (it.isHttpOnly) httpOnly()
+                        if (it.isSecure) secure()
+                    }.build()
+            }
+        }
+
+        // Copy cookies to cookie store
+        convertedCookies.forEach {
+            network.cookies.addAll(
+                HttpUrl.Builder()
+                    .scheme("http")
+                    .host(it.domain)
+                    .build(),
+                listOf(it)
+            )
+        }
+        // Merge new and existing cookies for this request
+        // Find the cookies that we need to merge into this request
+        val convertedForThisRequest = convertedCookies.filter {
+            it.matches(response.request.url)
+        }
+        // Extract cookies from current request
+        val existingCookies = Cookie.parseAll(
+            response.request.url,
+            response.request.headers
+        )
+        // Filter out existing values of cookies that we are about to merge in
+        val filteredExisting = existingCookies.filter { existing ->
+            convertedForThisRequest.none { converted -> converted.name == existing.name }
+        }
+        val newCookies = filteredExisting + convertedForThisRequest
+        return response.request.newBuilder()
+            .header("Cookie", newCookies.map { it.toString() }.joinToString("; "))
+            .build()
+    }
+
+    companion object {
+        private val SERVER_CHECK = arrayOf("cloudflare-nginx", "cloudflare")
+        private val COOKIE_NAMES = listOf("cf_clearance")
+    }
+}

server/src/main/kotlin/eu/kanade/tachiyomi/network/interceptor/UserAgentInterceptor.kt

@@ -0,0 +1,22 @@
+package eu.kanade.tachiyomi.network.interceptor
+
+import eu.kanade.tachiyomi.source.online.HttpSource
+import okhttp3.Interceptor
+import okhttp3.Response
+
+class UserAgentInterceptor : Interceptor {
+    override fun intercept(chain: Interceptor.Chain): Response {
+        val originalRequest = chain.request()
+
+        return if (originalRequest.header("User-Agent").isNullOrEmpty()) {
+            val newRequest = originalRequest
+                .newBuilder()
+                .removeHeader("User-Agent")
+                .addHeader("User-Agent", HttpSource.DEFAULT_USER_AGENT)
+                .build()
+            chain.proceed(newRequest)
+        } else {
+            chain.proceed(originalRequest)
+        }
+    }
+}

server/src/main/kotlin/eu/kanade/tachiyomi/source/LocalSource.kt

@@ -1,358 +0,0 @@
-package eu.kanade.tachiyomi.source
-
-import android.content.Context
-import eu.kanade.tachiyomi.source.model.FilterList
-import eu.kanade.tachiyomi.source.model.MangasPage
-import eu.kanade.tachiyomi.source.model.Page
-import eu.kanade.tachiyomi.source.model.SChapter
-import eu.kanade.tachiyomi.source.model.SManga
-import rx.Observable
-
-// import com.github.junrar.Archive
-// import com.google.gson.JsonParser
-// import eu.kanade.tachiyomi.R
-// import eu.kanade.tachiyomi.source.model.Filter
-// import eu.kanade.tachiyomi.source.model.FilterList
-// import eu.kanade.tachiyomi.source.model.MangasPage
-// import eu.kanade.tachiyomi.source.model.Page
-// import eu.kanade.tachiyomi.source.model.SChapter
-// import eu.kanade.tachiyomi.source.model.SManga
-// import eu.kanade.tachiyomi.util.chapter.ChapterRecognition
-// import eu.kanade.tachiyomi.util.lang.compareToCaseInsensitiveNaturalOrder
-// import eu.kanade.tachiyomi.util.storage.DiskUtil
-// import eu.kanade.tachiyomi.util.storage.EpubFile
-// import eu.kanade.tachiyomi.util.system.ImageUtil
-// import rx.Observable
-// import timber.log.Timber
-// import java.io.File
-// import java.io.FileInputStream
-// import java.io.InputStream
-// import java.util.Locale
-// import java.util.concurrent.TimeUnit
-// import java.util.zip.ZipFile
-
-class LocalSource(private val context: Context) : CatalogueSource {
-    companion object {
-        const val ID = 0L
-//        const val HELP_URL = "https://tachiyomi.org/help/guides/reading-local-manga/"
-//
-//        private const val COVER_NAME = "cover.jpg"
-//        private val SUPPORTED_ARCHIVE_TYPES = setOf("zip", "rar", "cbr", "cbz", "epub")
-//
-//        private val POPULAR_FILTERS = FilterList(OrderBy())
-//        private val LATEST_FILTERS = FilterList(OrderBy().apply { state = Filter.Sort.Selection(1, false) })
-//        private val LATEST_THRESHOLD = TimeUnit.MILLISECONDS.convert(7, TimeUnit.DAYS)
-//
-//        fun updateCover(context: Context, manga: SManga, input: InputStream): File? {
-//            val dir = getBaseDirectories(context).firstOrNull()
-//            if (dir == null) {
-//                input.close()
-//                return null
-//            }
-//            val cover = File("${dir.absolutePath}/${manga.url}", COVER_NAME)
-//
-//            // It might not exist if using the external SD card
-//            cover.parentFile?.mkdirs()
-//            input.use {
-//                cover.outputStream().use {
-//                    input.copyTo(it)
-//                }
-//            }
-//            return cover
-//        }
-//
-//        private fun getBaseDirectories(context: Context): List<File> {
-//            val c = context.getString(R.string.app_name) + File.separator + "local"
-//            return DiskUtil.getExternalStorages(context).map { File(it.absolutePath, c) }
-//        }
-    }
-
-    override val id = ID
-    override val name = "Local source"
-    override val lang = ""
-    override val supportsLatest = true
-
-    override fun fetchMangaDetails(manga: SManga): Observable<SManga> {
-        TODO("Not yet implemented")
-    }
-
-    override fun fetchChapterList(manga: SManga): Observable<List<SChapter>> {
-        TODO("Not yet implemented")
-    }
-
-    override fun fetchPageList(chapter: SChapter): Observable<List<Page>> {
-        TODO("Not yet implemented")
-    }
-    override fun fetchPopularManga(page: Int): Observable<MangasPage> {
-        TODO("Not yet implemented")
-    }
-
-    override fun fetchSearchManga(page: Int, query: String, filters: FilterList): Observable<MangasPage> {
-        TODO("Not yet implemented")
-    }
-
-    override fun fetchLatestUpdates(page: Int): Observable<MangasPage> {
-        TODO("Not yet implemented")
-    }
-
-    override fun getFilterList(): FilterList {
-        TODO("Not yet implemented")
-    }
-//
-//    override fun toString() = context.getString(R.string.local_source)
-//
-//    override fun fetchPopularManga(page: Int) = fetchSearchManga(page, "", POPULAR_FILTERS)
-//
-//    override fun fetchSearchManga(page: Int, query: String, filters: FilterList): Observable<MangasPage> {
-//        val baseDirs = getBaseDirectories(context)
-//
-//        val time = if (filters === LATEST_FILTERS) System.currentTimeMillis() - LATEST_THRESHOLD else 0L
-//        var mangaDirs = baseDirs
-//            .asSequence()
-//            .mapNotNull { it.listFiles()?.toList() }
-//            .flatten()
-//            .filter { it.isDirectory }
-//            .filterNot { it.name.startsWith('.') }
-//            .filter { if (time == 0L) it.name.contains(query, ignoreCase = true) else it.lastModified() >= time }
-//            .distinctBy { it.name }
-//
-//        val state = ((if (filters.isEmpty()) POPULAR_FILTERS else filters)[0] as OrderBy).state
-//        when (state?.index) {
-//            0 -> {
-//                mangaDirs = if (state.ascending) {
-//                    mangaDirs.sortedBy { it.name.toLowerCase(Locale.ENGLISH) }
-//                } else {
-//                    mangaDirs.sortedByDescending { it.name.toLowerCase(Locale.ENGLISH) }
-//                }
-//            }
-//            1 -> {
-//                mangaDirs = if (state.ascending) {
-//                    mangaDirs.sortedBy(File::lastModified)
-//                } else {
-//                    mangaDirs.sortedByDescending(File::lastModified)
-//                }
-//            }
-//        }
-//
-//        val mangas = mangaDirs.map { mangaDir ->
-//            SManga.create().apply {
-//                title = mangaDir.name
-//                url = mangaDir.name
-//
-//                // Try to find the cover
-//                for (dir in baseDirs) {
-//                    val cover = File("${dir.absolutePath}/$url", COVER_NAME)
-//                    if (cover.exists()) {
-//                        thumbnail_url = cover.absolutePath
-//                        break
-//                    }
-//                }
-//
-//                val chapters = fetchChapterList(this).toBlocking().first()
-//                if (chapters.isNotEmpty()) {
-//                    val chapter = chapters.last()
-//                    val format = getFormat(chapter)
-//                    if (format is Format.Epub) {
-//                        EpubFile(format.file).use { epub ->
-//                            epub.fillMangaMetadata(this)
-//                        }
-//                    }
-//
-//                    // Copy the cover from the first chapter found.
-//                    if (thumbnail_url == null) {
-//                        try {
-//                            val dest = updateCover(chapter, this)
-//                            thumbnail_url = dest?.absolutePath
-//                        } catch (e: Exception) {
-//                            Timber.e(e)
-//                        }
-//                    }
-//                }
-//            }
-//        }
-//
-//        return Observable.just(MangasPage(mangas.toList(), false))
-//    }
-//
-//    override fun fetchLatestUpdates(page: Int) = fetchSearchManga(page, "", LATEST_FILTERS)
-//
-//    override fun fetchMangaDetails(manga: SManga): Observable<SManga> {
-//        getBaseDirectories(context)
-//            .asSequence()
-//            .mapNotNull { File(it, manga.url).listFiles()?.toList() }
-//            .flatten()
-//            .firstOrNull { it.extension == "json" }
-//            ?.apply {
-//                val reader = this.inputStream().bufferedReader()
-//                val json = JsonParser.parseReader(reader).asJsonObject
-//
-//                manga.title = json["title"]?.asString ?: manga.title
-//                manga.author = json["author"]?.asString ?: manga.author
-//                manga.artist = json["artist"]?.asString ?: manga.artist
-//                manga.description = json["description"]?.asString ?: manga.description
-//                manga.genre = json["genre"]?.asJsonArray?.joinToString(", ") { it.asString }
-//                    ?: manga.genre
-//                manga.status = json["status"]?.asInt ?: manga.status
-//            }
-//
-//        return Observable.just(manga)
-//    }
-//
-//    override fun fetchChapterList(manga: SManga): Observable<List<SChapter>> {
-//        val chapters = getBaseDirectories(context)
-//            .asSequence()
-//            .mapNotNull { File(it, manga.url).listFiles()?.toList() }
-//            .flatten()
-//            .filter { it.isDirectory || isSupportedFile(it.extension) }
-//            .map { chapterFile ->
-//                SChapter.create().apply {
-//                    url = "${manga.url}/${chapterFile.name}"
-//                    name = if (chapterFile.isDirectory) {
-//                        chapterFile.name
-//                    } else {
-//                        chapterFile.nameWithoutExtension
-//                    }
-//                    date_upload = chapterFile.lastModified()
-//
-//                    val format = getFormat(this)
-//                    if (format is Format.Epub) {
-//                        EpubFile(format.file).use { epub ->
-//                            epub.fillChapterMetadata(this)
-//                        }
-//                    }
-//
-//                    val chapNameCut = stripMangaTitle(name, manga.title)
-//                    if (chapNameCut.isNotEmpty()) name = chapNameCut
-//                    ChapterRecognition.parseChapterNumber(this, manga)
-//                }
-//            }
-//            .sortedWith(
-//                Comparator { c1, c2 ->
-//                    val c = c2.chapter_number.compareTo(c1.chapter_number)
-//                    if (c == 0) c2.name.compareToCaseInsensitiveNaturalOrder(c1.name) else c
-//                }
-//            )
-//            .toList()
-//
-//        return Observable.just(chapters)
-//    }
-//
-//    /**
-//     * Strips the manga title from a chapter name, matching only based on alphanumeric and whitespace
-//     * characters.
-//     */
-//    private fun stripMangaTitle(chapterName: String, mangaTitle: String): String {
-//        var chapterNameIndex = 0
-//        var mangaTitleIndex = 0
-//        while (chapterNameIndex < chapterName.length && mangaTitleIndex < mangaTitle.length) {
-//            val chapterChar = chapterName[chapterNameIndex]
-//            val mangaChar = mangaTitle[mangaTitleIndex]
-//            if (!chapterChar.equals(mangaChar, true)) {
-//                val invalidChapterChar = !chapterChar.isLetterOrDigit() && !chapterChar.isWhitespace()
-//                val invalidMangaChar = !mangaChar.isLetterOrDigit() && !mangaChar.isWhitespace()
-//
-//                if (!invalidChapterChar && !invalidMangaChar) {
-//                    return chapterName
-//                }
-//
-//                if (invalidChapterChar) {
-//                    chapterNameIndex++
-//                }
-//
-//                if (invalidMangaChar) {
-//                    mangaTitleIndex++
-//                }
-//            } else {
-//                chapterNameIndex++
-//                mangaTitleIndex++
-//            }
-//        }
-//
-//        return chapterName.substring(chapterNameIndex).trimStart(' ', '-', '_', ',', ':')
-//    }
-//
-//    override fun fetchPageList(chapter: SChapter): Observable<List<Page>> {
-//        return Observable.error(Exception("Unused"))
-//    }
-//
-//    private fun isSupportedFile(extension: String): Boolean {
-//        return extension.toLowerCase() in SUPPORTED_ARCHIVE_TYPES
-//    }
-//
-//    fun getFormat(chapter: SChapter): Format {
-//        val baseDirs = getBaseDirectories(context)
-//
-//        for (dir in baseDirs) {
-//            val chapFile = File(dir, chapter.url)
-//            if (!chapFile.exists()) continue
-//
-//            return getFormat(chapFile)
-//        }
-//        throw Exception("Chapter not found")
-//    }
-//
-//    private fun getFormat(file: File): Format {
-//        val extension = file.extension
-//        return if (file.isDirectory) {
-//            Format.Directory(file)
-//        } else if (extension.equals("zip", true) || extension.equals("cbz", true)) {
-//            Format.Zip(file)
-//        } else if (extension.equals("rar", true) || extension.equals("cbr", true)) {
-//            Format.Rar(file)
-//        } else if (extension.equals("epub", true)) {
-//            Format.Epub(file)
-//        } else {
-//            throw Exception("Invalid chapter format")
-//        }
-//    }
-//
-//    private fun updateCover(chapter: SChapter, manga: SManga): File? {
-//        return when (val format = getFormat(chapter)) {
-//            is Format.Directory -> {
-//                val entry = format.file.listFiles()
-//                    ?.sortedWith { f1, f2 -> f1.name.compareToCaseInsensitiveNaturalOrder(f2.name) }
-//                    ?.find { !it.isDirectory && ImageUtil.isImage(it.name) { FileInputStream(it) } }
-//
-//                entry?.let { updateCover(context, manga, it.inputStream()) }
-//            }
-//            is Format.Zip -> {
-//                ZipFile(format.file).use { zip ->
-//                    val entry = zip.entries().toList()
-//                        .sortedWith { f1, f2 -> f1.name.compareToCaseInsensitiveNaturalOrder(f2.name) }
-//                        .find { !it.isDirectory && ImageUtil.isImage(it.name) { zip.getInputStream(it) } }
-//
-//                    entry?.let { updateCover(context, manga, zip.getInputStream(it)) }
-//                }
-//            }
-//            is Format.Rar -> {
-//                Archive(format.file).use { archive ->
-//                    val entry = archive.fileHeaders
-//                        .sortedWith { f1, f2 -> f1.fileName.compareToCaseInsensitiveNaturalOrder(f2.fileName) }
-//                        .find { !it.isDirectory && ImageUtil.isImage(it.fileName) { archive.getInputStream(it) } }
-//
-//                    entry?.let { updateCover(context, manga, archive.getInputStream(it)) }
-//                }
-//            }
-//            is Format.Epub -> {
-//                EpubFile(format.file).use { epub ->
-//                    val entry = epub.getImagesFromPages()
-//                        .firstOrNull()
-//                        ?.let { epub.getEntry(it) }
-//
-//                    entry?.let { updateCover(context, manga, epub.getInputStream(it)) }
-//                }
-//            }
-//        }
-//    }
-//
-//    private class OrderBy : Filter.Sort("Order by", arrayOf("Title", "Date"), Selection(0, true))
-//
-//    override fun getFilterList() = FilterList(OrderBy())
-//
-//    sealed class Format {
-//        data class Directory(val file: File) : Format()
-//        data class Zip(val file: File) : Format()
-//        data class Rar(val file: File) : Format()
-//        data class Epub(val file: File) : Format()
-//    }
-}

server/src/main/kotlin/eu/kanade/tachiyomi/source/local/LocalSource.kt

@@ -0,0 +1,478 @@
+package eu.kanade.tachiyomi.source.local
+
+import com.github.junrar.Archive
+import eu.kanade.tachiyomi.source.local.FileSystemInterceptor.fakeUrlFrom
+import eu.kanade.tachiyomi.source.local.LocalSource.Format.Directory
+import eu.kanade.tachiyomi.source.local.LocalSource.Format.Epub
+import eu.kanade.tachiyomi.source.local.LocalSource.Format.Rar
+import eu.kanade.tachiyomi.source.local.LocalSource.Format.Zip
+import eu.kanade.tachiyomi.source.local.loader.EpubPageLoader
+import eu.kanade.tachiyomi.source.local.loader.RarPageLoader
+import eu.kanade.tachiyomi.source.local.loader.ZipPageLoader
+import eu.kanade.tachiyomi.source.model.Filter
+import eu.kanade.tachiyomi.source.model.FilterList
+import eu.kanade.tachiyomi.source.model.MangasPage
+import eu.kanade.tachiyomi.source.model.Page
+import eu.kanade.tachiyomi.source.model.SChapter
+import eu.kanade.tachiyomi.source.model.SManga
+import eu.kanade.tachiyomi.source.online.HttpSource
+import eu.kanade.tachiyomi.util.chapter.ChapterRecognition
+import eu.kanade.tachiyomi.util.lang.compareToCaseInsensitiveNaturalOrder
+import eu.kanade.tachiyomi.util.storage.EpubFile
+import kotlinx.serialization.json.Json
+import kotlinx.serialization.json.JsonObject
+import kotlinx.serialization.json.contentOrNull
+import kotlinx.serialization.json.decodeFromStream
+import kotlinx.serialization.json.intOrNull
+import kotlinx.serialization.json.jsonArray
+import kotlinx.serialization.json.jsonPrimitive
+import mu.KotlinLogging
+import okhttp3.Interceptor
+import okhttp3.OkHttpClient
+import okhttp3.Protocol
+import okhttp3.Request
+import okhttp3.Response
+import okhttp3.ResponseBody.Companion.asResponseBody
+import okhttp3.ResponseBody.Companion.toResponseBody
+import okio.buffer
+import okio.source
+import org.jetbrains.exposed.sql.insert
+import org.jetbrains.exposed.sql.insertAndGetId
+import org.jetbrains.exposed.sql.select
+import org.jetbrains.exposed.sql.transactions.transaction
+import org.kodein.di.DI
+import org.kodein.di.conf.global
+import org.kodein.di.instance
+import rx.Observable
+import suwayomi.tachidesk.manga.impl.util.storage.ImageUtil
+import suwayomi.tachidesk.manga.model.table.ExtensionTable
+import suwayomi.tachidesk.manga.model.table.SourceTable
+import suwayomi.tachidesk.server.ApplicationDirs
+import uy.kohesive.injekt.injectLazy
+import java.io.File
+import java.io.FileInputStream
+import java.io.FileNotFoundException
+import java.io.InputStream
+import java.net.URLDecoder
+import java.util.Locale
+import java.util.concurrent.TimeUnit
+import java.util.zip.ZipFile
+
+class LocalSource : HttpSource() {
+    companion object {
+        const val ID = 0L
+        const val LANG = "localsourcelang"
+        const val NAME = "Local source"
+
+        const val EXTENSION_NAME = "Local Source fake extension"
+
+        const val HELP_URL = "https://tachiyomi.org/help/guides/local-manga/"
+
+        private val SUPPORTED_ARCHIVE_TYPES = setOf("zip", "rar", "cbr", "cbz", "epub")
+
+        private val LATEST_THRESHOLD = TimeUnit.MILLISECONDS.convert(7, TimeUnit.DAYS)
+
+        private val logger = KotlinLogging.logger {}
+
+        private val applicationDirs by DI.global.instance<ApplicationDirs>()
+
+        val pageCache: MutableMap<String, List<() -> InputStream>> = mutableMapOf()
+
+        fun updateCover(manga: SManga, input: InputStream): File? {
+            val cover = getCoverFile(File("${applicationDirs.localMangaRoot}/${manga.url}"))
+                ?: File("${applicationDirs.localMangaRoot}/${manga.url}/cover.jpg")
+
+            cover.parentFile?.mkdirs()
+            input.use {
+                cover.outputStream().use {
+                    input.copyTo(it)
+                }
+            }
+
+            return cover
+        }
+
+        /**
+         * Returns valid cover file inside [parent] directory.
+         */
+        private fun getCoverFile(parent: File): File? {
+            return parent.listFiles()?.find { it.nameWithoutExtension == "cover" }?.takeIf {
+                it.isFile && ImageUtil.isImage(it.name) { it.inputStream() }
+            }
+        }
+
+        fun addDbRecords() {
+            transaction {
+                val sourceRecord = SourceTable.select { SourceTable.id eq ID }.firstOrNull()
+
+                if (sourceRecord == null) {
+                    // must do this to avoid database integrity errors
+                    val extensionId = ExtensionTable.insertAndGetId {
+                        it[apkName] = "localSource"
+                        it[name] = EXTENSION_NAME
+                        it[pkgName] = LocalSource::class.java.`package`.name
+                        it[versionName] = "1.2"
+                        it[versionCode] = 0
+                        it[lang] = LANG
+                        it[isNsfw] = false
+                        it[isInstalled] = true
+                    }
+
+                    SourceTable.insert {
+                        it[id] = ID
+                        it[name] = NAME
+                        it[lang] = LANG
+                        it[extension] = extensionId
+                        it[isNsfw] = false
+                    }
+                }
+            }
+        }
+    }
+
+    override val id = ID
+    override val name = NAME
+    override val lang = LANG
+    override val baseUrl: String = ""
+    override val supportsLatest = true
+
+    override val client: OkHttpClient = super.client.newBuilder()
+        .addInterceptor(FileSystemInterceptor)
+        .build()
+
+    private val json: Json by injectLazy()
+
+    override fun toString() = name
+
+    override fun fetchPopularManga(page: Int) = fetchSearchManga(page, "", POPULAR_FILTERS)
+
+    override fun fetchSearchManga(page: Int, query: String, filters: FilterList): Observable<MangasPage> {
+        val time = if (filters === LATEST_FILTERS) System.currentTimeMillis() - LATEST_THRESHOLD else 0L
+
+        var mangaDirs = File(applicationDirs.localMangaRoot).listFiles().orEmpty().toList()
+            .filter { it.isDirectory }
+            .filterNot { it.name.startsWith('.') }
+            .filter { if (time == 0L) it.name.contains(query, ignoreCase = true) else it.lastModified() >= time }
+            .distinctBy { it.name }
+
+        val state = ((if (filters.isEmpty()) POPULAR_FILTERS else filters)[0] as OrderBy).state
+        when (state?.index) {
+            0 -> {
+                mangaDirs = if (state.ascending) {
+                    mangaDirs.sortedBy { it.name.lowercase(Locale.ENGLISH) }
+                } else {
+                    mangaDirs.sortedByDescending { it.name.lowercase(Locale.ENGLISH) }
+                }
+            }
+            1 -> {
+                mangaDirs = if (state.ascending) {
+                    mangaDirs.sortedBy(File::lastModified)
+                } else {
+                    mangaDirs.sortedByDescending(File::lastModified)
+                }
+            }
+        }
+
+        val mangas = mangaDirs.map { mangaDir ->
+            SManga.create().apply {
+                title = mangaDir.name
+                url = mangaDir.name
+
+                // Try to find the cover
+                val cover = getCoverFile(File("${applicationDirs.localMangaRoot}/$url"))
+                if (cover != null && cover.exists()) {
+                    thumbnail_url = fakeUrlFrom(cover.absolutePath)
+                }
+
+                val chapters = fetchChapterList(this).toBlocking().first()
+                if (chapters.isNotEmpty()) {
+                    val chapter = chapters.last()
+                    val format = getFormat(chapter)
+                    if (format is Format.Epub) {
+                        EpubFile(format.file).use { epub ->
+                            epub.fillMangaMetadata(this)
+                        }
+                    }
+
+                    // Copy the cover from the first chapter found.
+                    if (thumbnail_url == null) {
+                        try {
+                            val dest = updateCover(chapter, this)
+                            thumbnail_url = dest?.absolutePath?.let { fakeUrlFrom(it) }
+                        } catch (e: Exception) {
+                            logger.error { e }
+                        }
+                    }
+                }
+            }
+        }
+
+        return Observable.just(MangasPage(mangas.toList(), false))
+    }
+
+    override fun fetchLatestUpdates(page: Int) = fetchSearchManga(page, "", LATEST_FILTERS)
+
+    override fun fetchMangaDetails(manga: SManga): Observable<SManga> {
+        File(applicationDirs.localMangaRoot, manga.url).listFiles().orEmpty().toList()
+            .firstOrNull { it.extension == "json" }
+            ?.apply {
+                val obj = json.decodeFromStream<JsonObject>(inputStream())
+
+                manga.title = obj["title"]?.jsonPrimitive?.contentOrNull ?: manga.title
+                manga.author = obj["author"]?.jsonPrimitive?.contentOrNull ?: manga.author
+                manga.artist = obj["artist"]?.jsonPrimitive?.contentOrNull ?: manga.artist
+                manga.description = obj["description"]?.jsonPrimitive?.contentOrNull ?: manga.description
+                manga.genre = obj["genre"]?.jsonArray?.joinToString(", ") { it.jsonPrimitive.content }
+                    ?: manga.genre
+                manga.status = obj["status"]?.jsonPrimitive?.intOrNull ?: manga.status
+            }
+
+        return Observable.just(manga)
+    }
+
+    override fun fetchChapterList(manga: SManga): Observable<List<SChapter>> {
+        val chapters = File(applicationDirs.localMangaRoot, manga.url).listFiles().orEmpty().toList()
+            .filter { it.isDirectory || isSupportedFile(it.extension) }
+            .map { chapterFile ->
+                SChapter.create().apply {
+                    url = "${manga.url}/${chapterFile.name}"
+                    name = if (chapterFile.isDirectory) {
+                        chapterFile.name
+                    } else {
+                        chapterFile.nameWithoutExtension
+                    }
+                    date_upload = chapterFile.lastModified()
+
+                    val format = getFormat(this)
+                    if (format is Format.Epub) {
+                        EpubFile(format.file).use { epub ->
+                            epub.fillChapterMetadata(this)
+                        }
+                    }
+
+                    val chapNameCut = stripMangaTitle(name, manga.title)
+                    if (chapNameCut.isNotEmpty()) name = chapNameCut
+                    ChapterRecognition.parseChapterNumber(this, manga)
+                }
+            }
+            .sortedWith { c1, c2 ->
+                val c = c2.chapter_number.compareTo(c1.chapter_number)
+                if (c == 0) c2.name.compareToCaseInsensitiveNaturalOrder(c1.name) else c
+            }
+            .toList()
+
+        return Observable.just(chapters)
+    }
+
+    /**
+     * Strips the manga title from a chapter name, matching only based on alphanumeric and whitespace
+     * characters.
+     */
+    private fun stripMangaTitle(chapterName: String, mangaTitle: String): String {
+        var chapterNameIndex = 0
+        var mangaTitleIndex = 0
+        while (chapterNameIndex < chapterName.length && mangaTitleIndex < mangaTitle.length) {
+            val chapterChar = chapterName[chapterNameIndex]
+            val mangaChar = mangaTitle[mangaTitleIndex]
+            if (!chapterChar.equals(mangaChar, true)) {
+                val invalidChapterChar = !chapterChar.isLetterOrDigit() && !chapterChar.isWhitespace()
+                val invalidMangaChar = !mangaChar.isLetterOrDigit() && !mangaChar.isWhitespace()
+
+                if (!invalidChapterChar && !invalidMangaChar) {
+                    return chapterName
+                }
+
+                if (invalidChapterChar) {
+                    chapterNameIndex++
+                }
+
+                if (invalidMangaChar) {
+                    mangaTitleIndex++
+                }
+            } else {
+                chapterNameIndex++
+                mangaTitleIndex++
+            }
+        }
+
+        return chapterName.substring(chapterNameIndex).trimStart(' ', '-', '_', ',', ':')
+    }
+
+    private fun isSupportedFile(extension: String): Boolean {
+        return extension.lowercase() in SUPPORTED_ARCHIVE_TYPES
+    }
+
+    override fun fetchPageList(chapter: SChapter): Observable<List<Page>> {
+        val chapterFile = File(applicationDirs.localMangaRoot + "/" + chapter.url)
+
+        return when (getFormat(chapterFile)) {
+            is Directory -> {
+                Observable.just(
+                    chapterFile.listFiles().orEmpty().sortedBy { it.name }.mapIndexed { index, page ->
+                        Page(
+                            index,
+                            imageUrl = fakeUrlFrom(applicationDirs.localMangaRoot + "/" + chapter.url + "/" + page.name)
+                        )
+                    }
+                )
+            }
+            is Zip -> {
+                val pages = ZipPageLoader(chapterFile).getPages()
+                pageCache[chapter.url] = pages.map { it.stream!! }
+
+                Observable.just(pages)
+            }
+            is Rar -> {
+                val pages = RarPageLoader(chapterFile).getPages()
+                pageCache[chapter.url] = pages.map { it.stream!! }
+
+                Observable.just(pages)
+            }
+            is Epub -> {
+                val pages = EpubPageLoader(chapterFile).getPages()
+                pageCache[chapter.url] = pages.map { it.stream!! }
+
+                Observable.just(pages)
+            }
+        }
+    }
+
+    fun getFormat(chapter: SChapter): Format {
+        val chapFile = File(applicationDirs.localMangaRoot, chapter.url)
+        if (chapFile.exists())
+            return getFormat(chapFile)
+
+        throw Exception("Chapter not found")
+    }
+
+    private fun getFormat(file: File): Format = with(file) {
+        when {
+            isDirectory -> Format.Directory(this)
+            extension.equals("zip", true) || extension.equals("cbz", true) -> Format.Zip(this)
+            extension.equals("rar", true) || extension.equals("cbr", true) -> Format.Rar(this)
+            extension.equals("epub", true) -> Format.Epub(this)
+
+            else -> throw Exception("Invalid chapter format")
+        }
+    }
+
+    private fun updateCover(chapter: SChapter, manga: SManga): File? {
+        return when (val format = getFormat(chapter)) {
+            is Format.Directory -> {
+                val entry = format.file.listFiles()
+                    ?.sortedWith { f1, f2 -> f1.name.compareToCaseInsensitiveNaturalOrder(f2.name) }
+                    ?.find { !it.isDirectory && ImageUtil.isImage(it.name) { FileInputStream(it) } }
+
+                entry?.let { updateCover(manga, it.inputStream()) }
+            }
+            is Format.Zip -> {
+                ZipFile(format.file).use { zip ->
+                    val entry = zip.entries().toList()
+                        .sortedWith { f1, f2 -> f1.name.compareToCaseInsensitiveNaturalOrder(f2.name) }
+                        .find { !it.isDirectory && ImageUtil.isImage(it.name) { zip.getInputStream(it) } }
+
+                    entry?.let { updateCover(manga, zip.getInputStream(it)) }
+                }
+            }
+            is Format.Rar -> {
+                Archive(format.file).use { archive ->
+                    val entry = archive.fileHeaders
+                        .sortedWith { f1, f2 -> f1.fileName.compareToCaseInsensitiveNaturalOrder(f2.fileName) }
+                        .find { !it.isDirectory && ImageUtil.isImage(it.fileName) { archive.getInputStream(it) } }
+
+                    entry?.let { updateCover(manga, archive.getInputStream(it)) }
+                }
+            }
+            is Format.Epub -> {
+                EpubFile(format.file).use { epub ->
+                    val entry = epub.getImagesFromPages()
+                        .firstOrNull()
+                        ?.let { epub.getEntry(it) }
+
+                    entry?.let { updateCover(manga, epub.getInputStream(it)) }
+                }
+            }
+        }
+    }
+
+    override fun getFilterList() = POPULAR_FILTERS
+
+    private val POPULAR_FILTERS = FilterList(OrderBy())
+    private val LATEST_FILTERS = FilterList(OrderBy().apply { state = Filter.Sort.Selection(1, false) })
+
+    private class OrderBy : Filter.Sort(
+        "Order by",
+        arrayOf("Title", "Date"),
+        Selection(0, true)
+    )
+
+    sealed class Format {
+        data class Directory(val file: File) : Format()
+        data class Zip(val file: File) : Format()
+        data class Rar(val file: File) : Format()
+        data class Epub(val file: File) : Format()
+    }
+
+    // ///////////////////// Not used ///////////////////// //
+
+    override fun mangaDetailsParse(response: Response): SManga = throw Exception("Not used")
+
+    override fun chapterListParse(response: Response): List<SChapter> = throw Exception("Not used")
+
+    override fun pageListParse(response: Response): List<Page> = throw Exception("Not used")
+
+    override fun imageUrlParse(response: Response): String = throw Exception("Not used")
+
+    override fun popularMangaRequest(page: Int): Request = throw Exception("Not used")
+
+    override fun popularMangaParse(response: Response): MangasPage = throw Exception("Not used")
+
+    override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request =
+        throw Exception("Not used")
+
+    override fun searchMangaParse(response: Response): MangasPage = throw Exception("Not used")
+
+    override fun latestUpdatesRequest(page: Int): Request = throw Exception("Not used")
+
+    override fun latestUpdatesParse(response: Response): MangasPage = throw Exception("Not used")
+}
+
+private object FileSystemInterceptor : Interceptor {
+    fun fakeUrlFrom(path: String): String = "http://$path"
+
+
+    private fun restoreFilePath(url: String): String {
+        val path = URLDecoder.decode(url.replaceFirst("http://", ""), "UTF-8")
+
+        // Windows
+        if (System.getProperty("os.name").lowercase().startsWith("win")) {
+            // convert paths like "c/Users/..." to "c:/Users/..."
+            return StringBuilder(path).insert(1, ":").toString()
+        }
+
+        return "/$path"
+    }
+
+    override fun intercept(chain: Interceptor.Chain): Response {
+        val request = chain.request()
+        val url = request.url
+        val filePath = restoreFilePath(url.toString())
+        return try {
+            Response.Builder()
+                .body(File(filePath).source().buffer().asResponseBody())
+                .code(200)
+                .message("Some file")
+                .protocol(Protocol.HTTP_1_0)
+                .request(request)
+                .build()
+        } catch (e: FileNotFoundException) {
+            Response.Builder()
+                .body("".toResponseBody())
+                .code(404)
+                .message(e.message ?: "File not found ($filePath)")
+                .protocol(Protocol.HTTP_1_0)
+                .request(request)
+                .build()
+        }
+    }
+}

server/src/main/kotlin/eu/kanade/tachiyomi/source/local/loader/EpubPageLoader.kt

@@ -0,0 +1,31 @@
+package eu.kanade.tachiyomi.source.local.loader
+
+import eu.kanade.tachiyomi.source.model.Page
+import eu.kanade.tachiyomi.util.storage.EpubFile
+import java.io.File
+
+/**
+ * Loader used to load a chapter from a .epub file.
+ */
+class EpubPageLoader(file: File) : PageLoader {
+
+    /**
+     * The epub file.
+     */
+    private val epub = EpubFile(file)
+
+    /**
+     * Returns an observable containing the pages found on this zip archive ordered with a natural
+     * comparator.
+     */
+    override fun getPages(): List<ReaderPage> {
+        return epub.getImagesFromPages()
+            .mapIndexed { i, path ->
+                val streamFn = { epub.getInputStream(epub.getEntry(path)!!) }
+                ReaderPage(i).apply {
+                    stream = streamFn
+                    status = Page.READY
+                }
+            }
+    }
+}

server/src/main/kotlin/eu/kanade/tachiyomi/source/local/loader/PageLoader.kt

@@ -0,0 +1,10 @@
+package eu.kanade.tachiyomi.source.local.loader
+
+// adapted from eu.kanade.tachiyomi.ui.reader.loader.PageLoader
+interface PageLoader {
+    /**
+     * Returns an observable containing the list of pages of a chapter. Only the first emission
+     * will be used.
+     */
+    fun getPages(): List<ReaderPage>
+}

server/src/main/kotlin/eu/kanade/tachiyomi/source/local/loader/RarPageLoader.kt

@@ -0,0 +1,63 @@
+package eu.kanade.tachiyomi.source.local.loader
+
+import com.github.junrar.Archive
+import com.github.junrar.rarfile.FileHeader
+import eu.kanade.tachiyomi.source.model.Page
+import eu.kanade.tachiyomi.util.lang.compareToCaseInsensitiveNaturalOrder
+import suwayomi.tachidesk.manga.impl.util.storage.ImageUtil
+import java.io.File
+import java.io.InputStream
+import java.io.PipedInputStream
+import java.io.PipedOutputStream
+import java.util.concurrent.Executors
+
+/**
+ * Loader used to load a chapter from a .rar or .cbr file.
+ */
+class RarPageLoader(file: File) : PageLoader {
+
+    /**
+     * The rar archive to load pages from.
+     */
+    private val archive = Archive(file)
+
+    /**
+     * Pool for copying compressed files to an input stream.
+     */
+    private val pool = Executors.newFixedThreadPool(1)
+
+    /**
+     * Returns an observable containing the pages found on this rar archive ordered with a natural
+     * comparator.
+     */
+    override fun getPages(): List<ReaderPage> {
+        return archive.fileHeaders
+            .filter { !it.isDirectory && ImageUtil.isImage(it.fileName) { archive.getInputStream(it) } }
+            .sortedWith { f1, f2 -> f1.fileName.compareToCaseInsensitiveNaturalOrder(f2.fileName) }
+            .mapIndexed { i, header ->
+                val streamFn = { getStream(header) }
+
+                ReaderPage(i).apply {
+                    stream = streamFn
+                    status = Page.READY
+                }
+            }
+    }
+
+    /**
+     * Returns an input stream for the given [header].
+     */
+    private fun getStream(header: FileHeader): InputStream {
+        val pipeIn = PipedInputStream()
+        val pipeOut = PipedOutputStream(pipeIn)
+        pool.execute {
+            try {
+                pipeOut.use {
+                    archive.extractFile(header, it)
+                }
+            } catch (e: Exception) {
+            }
+        }
+        return pipeIn
+    }
+}

server/src/main/kotlin/eu/kanade/tachiyomi/source/local/loader/ReaderPage.kt

@@ -0,0 +1,11 @@
+package eu.kanade.tachiyomi.source.local.loader
+
+import eu.kanade.tachiyomi.source.model.Page
+import java.io.InputStream
+
+class ReaderPage(
+    index: Int,
+    url: String = "",
+    imageUrl: String? = null,
+    var stream: (() -> InputStream)? = null
+) : Page(index, url, imageUrl, null)

server/src/main/kotlin/eu/kanade/tachiyomi/source/local/loader/ZipPageLoader.kt

@@ -0,0 +1,31 @@
+package eu.kanade.tachiyomi.source.local.loader
+
+import eu.kanade.tachiyomi.source.model.Page
+import eu.kanade.tachiyomi.util.lang.compareToCaseInsensitiveNaturalOrder
+import suwayomi.tachidesk.manga.impl.util.storage.ImageUtil
+import java.io.File
+import java.util.zip.ZipFile
+
+class ZipPageLoader(file: File) : PageLoader {
+    /**
+     * The zip file to load pages from.
+     */
+    private val zip = ZipFile(file)
+
+    /**
+     * Returns an observable containing the pages found on this zip archive ordered with a natural
+     * comparator.
+     */
+    override fun getPages(): List<ReaderPage> {
+        return zip.entries().toList()
+            .filter { !it.isDirectory && ImageUtil.isImage(it.name) { zip.getInputStream(it) } }
+            .sortedWith { f1, f2 -> f1.name.compareToCaseInsensitiveNaturalOrder(f2.name) }
+            .mapIndexed { i, entry ->
+                val streamFn = { zip.getInputStream(entry) }
+                ReaderPage(i).apply {
+                    stream = streamFn
+                    status = Page.READY
+                }
+            }
+    }
+}

server/src/main/kotlin/eu/kanade/tachiyomi/source/model/SManga.kt

@@ -61,3 +61,30 @@ interface SManga : Serializable {
         }
     }
 }
+
+// fun SManga.toMangaInfo(): MangaInfo {
+//    return MangaInfo(
+//        key = this.url,
+//        title = this.title,
+//        artist = this.artist ?: "",
+//        author = this.author ?: "",
+//        description = this.description ?: "",
+//        genres = this.genre?.split(", ") ?: emptyList(),
+//        status = this.status,
+//        cover = this.thumbnail_url ?: ""
+//    )
+// }
+//
+// fun MangaInfo.toSManga(): SManga {
+//    val mangaInfo = this
+//    return SManga.create().apply {
+//        url = mangaInfo.key
+//        title = mangaInfo.title
+//        artist = mangaInfo.artist
+//        author = mangaInfo.author
+//        description = mangaInfo.description
+//        genre = mangaInfo.genres.joinToString(", ")
+//        status = mangaInfo.status
+//        thumbnail_url = mangaInfo.cover
+//    }
+// }

server/src/main/kotlin/eu/kanade/tachiyomi/source/online/HttpSource.kt

@@ -75,7 +75,7 @@ abstract class HttpSource : CatalogueSource {
      * Headers builder for requests. Implementations can override this method for custom headers.
      */
     protected open fun headersBuilder() = Headers.Builder().apply {
-        add("User-Agent", DEFAULT_USERAGENT)
+        add("User-Agent", DEFAULT_USER_AGENT)
     }
 
     /**
@@ -372,6 +372,6 @@ abstract class HttpSource : CatalogueSource {
     override fun getFilterList() = FilterList()
 
     companion object {
-        const val DEFAULT_USERAGENT = "Mozilla/5.0 (Windows NT 6.3; WOW64)"
+        const val DEFAULT_USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.150 Safari/537.36 Edg/88.0.705.63"
     }
 }

server/src/main/kotlin/eu/kanade/tachiyomi/util/chapter/ChapterRecognition.kt

@@ -0,0 +1,152 @@
+package eu.kanade.tachiyomi.util.chapter
+
+import eu.kanade.tachiyomi.source.model.SChapter
+import eu.kanade.tachiyomi.source.model.SManga
+
+/**
+ * -R> = regex conversion.
+ */
+object ChapterRecognition {
+    /**
+     * All cases with Ch.xx
+     * Mokushiroku Alice Vol.1 Ch. 4: Misrepresentation -R> 4
+     */
+    private val basic = Regex("""(?<=ch\.) *([0-9]+)(\.[0-9]+)?(\.?[a-z]+)?""")
+
+    /**
+     * Regex used when only one number occurrence
+     * Example: Bleach 567: Down With Snowwhite -R> 567
+     */
+    private val occurrence = Regex("""([0-9]+)(\.[0-9]+)?(\.?[a-z]+)?""")
+
+    /**
+     * Regex used when manga title removed
+     * Example: Solanin 028 Vol. 2 -> 028 Vol.2 -> 028Vol.2 -R> 028
+     */
+    private val withoutManga = Regex("""^([0-9]+)(\.[0-9]+)?(\.?[a-z]+)?""")
+
+    /**
+     * Regex used to remove unwanted tags
+     * Example Prison School 12 v.1 vol004 version1243 volume64 -R> Prison School 12
+     */
+    private val unwanted = Regex("""(?<![a-z])(v|ver|vol|version|volume|season|s).?[0-9]+""")
+
+    /**
+     * Regex used to remove unwanted whitespace
+     * Example One Piece 12 special -R> One Piece 12special
+     */
+    private val unwantedWhiteSpace = Regex("""(\s)(extra|special|omake)""")
+
+    fun parseChapterNumber(chapter: SChapter, manga: SManga) {
+        // If chapter number is known return.
+        if (chapter.chapter_number == -2f || chapter.chapter_number > -1f) {
+            return
+        }
+
+        // Get chapter title with lower case
+        var name = chapter.name.lowercase()
+
+        // Remove comma's from chapter.
+        name = name.replace(',', '.')
+
+        // Remove unwanted white spaces.
+        unwantedWhiteSpace.findAll(name).let {
+            it.forEach { occurrence -> name = name.replace(occurrence.value, occurrence.value.trim()) }
+        }
+
+        // Remove unwanted tags.
+        unwanted.findAll(name).let {
+            it.forEach { occurrence -> name = name.replace(occurrence.value, "") }
+        }
+
+        // Check base case ch.xx
+        if (updateChapter(basic.find(name), chapter)) {
+            return
+        }
+
+        // Check one number occurrence.
+        val occurrences: MutableList<MatchResult> = arrayListOf()
+        occurrence.findAll(name).let {
+            it.forEach { occurrence -> occurrences.add(occurrence) }
+        }
+
+        if (occurrences.size == 1) {
+            if (updateChapter(occurrences[0], chapter)) {
+                return
+            }
+        }
+
+        // Remove manga title from chapter title.
+        val nameWithoutManga = name.replace(manga.title.lowercase(), "").trim()
+
+        // Check if first value is number after title remove.
+        if (updateChapter(withoutManga.find(nameWithoutManga), chapter)) {
+            return
+        }
+
+        // Take the first number encountered.
+        if (updateChapter(occurrence.find(nameWithoutManga), chapter)) {
+            return
+        }
+    }
+
+    /**
+     * Check if volume is found and update chapter
+     * @param match result of regex
+     * @param chapter chapter object
+     * @return true if volume is found
+     */
+    private fun updateChapter(match: MatchResult?, chapter: SChapter): Boolean {
+        match?.let {
+            val initial = it.groups[1]?.value?.toFloat()!!
+            val subChapterDecimal = it.groups[2]?.value
+            val subChapterAlpha = it.groups[3]?.value
+            val addition = checkForDecimal(subChapterDecimal, subChapterAlpha)
+            chapter.chapter_number = initial.plus(addition)
+            return true
+        }
+        return false
+    }
+
+    /**
+     * Check for decimal in received strings
+     * @param decimal decimal value of regex
+     * @param alpha alpha value of regex
+     * @return decimal/alpha float value
+     */
+    private fun checkForDecimal(decimal: String?, alpha: String?): Float {
+        if (!decimal.isNullOrEmpty()) {
+            return decimal.toFloat()
+        }
+
+        if (!alpha.isNullOrEmpty()) {
+            if (alpha.contains("extra")) {
+                return .99f
+            }
+
+            if (alpha.contains("omake")) {
+                return .98f
+            }
+
+            if (alpha.contains("special")) {
+                return .97f
+            }
+
+            return if (alpha[0] == '.') {
+                // Take value after (.)
+                parseAlphaPostFix(alpha[1])
+            } else {
+                parseAlphaPostFix(alpha[0])
+            }
+        }
+
+        return .0f
+    }
+
+    /**
+     * x.a -> x.1, x.b -> x.2, etc
+     */
+    private fun parseAlphaPostFix(alpha: Char): Float {
+        return ("0." + (alpha.code - 96).toString()).toFloat()
+    }
+}

server/src/main/kotlin/eu/kanade/tachiyomi/util/lang/StringExtensions.kt

@@ -0,0 +1,58 @@
+package eu.kanade.tachiyomi.util.lang
+
+import net.greypanther.natsort.CaseInsensitiveSimpleNaturalComparator
+import kotlin.math.floor
+
+/**
+ * Replaces the given string to have at most [count] characters using [replacement] at its end.
+ * If [replacement] is longer than [count] an exception will be thrown when `length > count`.
+ */
+fun String.chop(count: Int, replacement: String = "…"): String {
+    return if (length > count) {
+        take(count - replacement.length) + replacement
+    } else {
+        this
+    }
+}
+
+/**
+ * Replaces the given string to have at most [count] characters using [replacement] near the center.
+ * If [replacement] is longer than [count] an exception will be thrown when `length > count`.
+ */
+fun String.truncateCenter(count: Int, replacement: String = "..."): String {
+    if (length <= count) {
+        return this
+    }
+
+    val pieceLength: Int = floor((count - replacement.length).div(2.0)).toInt()
+
+    return "${take(pieceLength)}$replacement${takeLast(pieceLength)}"
+}
+
+/**
+ * Case-insensitive natural comparator for strings.
+ */
+fun String.compareToCaseInsensitiveNaturalOrder(other: String): Int {
+    val comparator = CaseInsensitiveSimpleNaturalComparator.getInstance<String>()
+    return comparator.compare(this, other)
+}
+
+/**
+ * Returns the size of the string as the number of bytes.
+ */
+fun String.byteSize(): Int {
+    return toByteArray(Charsets.UTF_8).size
+}
+
+/**
+ * Returns a string containing the first [n] bytes from this string, or the entire string if this
+ * string is shorter.
+ */
+fun String.takeBytes(n: Int): String {
+    val bytes = toByteArray(Charsets.UTF_8)
+    return if (bytes.size <= n) {
+        this
+    } else {
+        bytes.decodeToString(endIndex = n).replace("\uFFFD", "")
+    }
+}

server/src/main/kotlin/eu/kanade/tachiyomi/util/storage/EpubFile.kt

@@ -0,0 +1,215 @@
+package eu.kanade.tachiyomi.util.storage
+
+import eu.kanade.tachiyomi.source.model.SChapter
+import eu.kanade.tachiyomi.source.model.SManga
+import org.jsoup.Jsoup
+import org.jsoup.nodes.Document
+import java.io.Closeable
+import java.io.File
+import java.io.InputStream
+import java.text.ParseException
+import java.text.SimpleDateFormat
+import java.util.Locale
+import java.util.zip.ZipEntry
+import java.util.zip.ZipFile
+
+/**
+ * Wrapper over ZipFile to load files in epub format.
+ */
+class EpubFile(file: File) : Closeable {
+
+    /**
+     * Zip file of this epub.
+     */
+    private val zip = ZipFile(file)
+
+    /**
+     * Path separator used by this epub.
+     */
+    private val pathSeparator = getPathSeparator()
+
+    /**
+     * Closes the underlying zip file.
+     */
+    override fun close() {
+        zip.close()
+    }
+
+    /**
+     * Returns an input stream for reading the contents of the specified zip file entry.
+     */
+    fun getInputStream(entry: ZipEntry): InputStream {
+        return zip.getInputStream(entry)
+    }
+
+    /**
+     * Returns the zip file entry for the specified name, or null if not found.
+     */
+    fun getEntry(name: String): ZipEntry? {
+        return zip.getEntry(name)
+    }
+
+    /**
+     * Fills manga metadata using this epub file's metadata.
+     */
+    fun fillMangaMetadata(manga: SManga) {
+        val ref = getPackageHref()
+        val doc = getPackageDocument(ref)
+
+        val creator = doc.getElementsByTag("dc:creator").first()
+        val description = doc.getElementsByTag("dc:description").first()
+
+        manga.author = creator?.text()
+        manga.description = description?.text()
+    }
+
+    /**
+     * Fills chapter metadata using this epub file's metadata.
+     */
+    fun fillChapterMetadata(chapter: SChapter) {
+        val ref = getPackageHref()
+        val doc = getPackageDocument(ref)
+
+        val title = doc.getElementsByTag("dc:title").first()
+        val publisher = doc.getElementsByTag("dc:publisher").first()
+        val creator = doc.getElementsByTag("dc:creator").first()
+        var date = doc.getElementsByTag("dc:date").first()
+        if (date == null) {
+            date = doc.select("meta[property=dcterms:modified]").first()
+        }
+
+        if (title != null) {
+            chapter.name = title.text()
+        }
+
+        if (publisher != null) {
+            chapter.scanlator = publisher.text()
+        } else if (creator != null) {
+            chapter.scanlator = creator.text()
+        }
+
+        if (date != null) {
+            val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ", Locale.getDefault())
+            try {
+                val parsedDate = dateFormat.parse(date.text())
+                if (parsedDate != null) {
+                    chapter.date_upload = parsedDate.time
+                }
+            } catch (e: ParseException) {
+                // Empty
+            }
+        }
+    }
+
+    /**
+     * Returns the path of all the images found in the epub file.
+     */
+    fun getImagesFromPages(): List<String> {
+        val ref = getPackageHref()
+        val doc = getPackageDocument(ref)
+        val pages = getPagesFromDocument(doc)
+        return getImagesFromPages(pages, ref)
+    }
+
+    /**
+     * Returns the path to the package document.
+     */
+    private fun getPackageHref(): String {
+        val meta = zip.getEntry(resolveZipPath("META-INF", "container.xml"))
+        if (meta != null) {
+            val metaDoc = zip.getInputStream(meta).use { Jsoup.parse(it, null, "") }
+            val path = metaDoc.getElementsByTag("rootfile").first()?.attr("full-path")
+            if (path != null) {
+                return path
+            }
+        }
+        return resolveZipPath("OEBPS", "content.opf")
+    }
+
+    /**
+     * Returns the package document where all the files are listed.
+     */
+    private fun getPackageDocument(ref: String): Document {
+        val entry = zip.getEntry(ref)
+        return zip.getInputStream(entry).use { Jsoup.parse(it, null, "") }
+    }
+
+    /**
+     * Returns all the pages from the epub.
+     */
+    private fun getPagesFromDocument(document: Document): List<String> {
+        val pages = document.select("manifest > item")
+            .filter { "application/xhtml+xml" == it.attr("media-type") }
+            .associateBy { it.attr("id") }
+
+        val spine = document.select("spine > itemref").map { it.attr("idref") }
+        return spine.mapNotNull { pages[it] }.map { it.attr("href") }
+    }
+
+    /**
+     * Returns all the images contained in every page from the epub.
+     */
+    private fun getImagesFromPages(pages: List<String>, packageHref: String): List<String> {
+        val result = mutableListOf<String>()
+        val basePath = getParentDirectory(packageHref)
+        pages.forEach { page ->
+            val entryPath = resolveZipPath(basePath, page)
+            val entry = zip.getEntry(entryPath)
+            val document = zip.getInputStream(entry).use { Jsoup.parse(it, null, "") }
+            val imageBasePath = getParentDirectory(entryPath)
+
+            document.allElements.forEach {
+                if (it.tagName() == "img") {
+                    result.add(resolveZipPath(imageBasePath, it.attr("src")))
+                } else if (it.tagName() == "image") {
+                    result.add(resolveZipPath(imageBasePath, it.attr("xlink:href")))
+                }
+            }
+        }
+
+        return result
+    }
+
+    /**
+     * Returns the path separator used by the epub file.
+     */
+    private fun getPathSeparator(): String {
+        val meta = zip.getEntry("META-INF\\container.xml")
+        return if (meta != null) {
+            "\\"
+        } else {
+            "/"
+        }
+    }
+
+    /**
+     * Resolves a zip path from base and relative components and a path separator.
+     */
+    private fun resolveZipPath(basePath: String, relativePath: String): String {
+        if (relativePath.startsWith(pathSeparator)) {
+            // Path is absolute, so return as-is.
+            return relativePath
+        }
+
+        var fixedBasePath = basePath.replace(pathSeparator, File.separator)
+        if (!fixedBasePath.startsWith(File.separator)) {
+            fixedBasePath = "${File.separator}$fixedBasePath"
+        }
+
+        val fixedRelativePath = relativePath.replace(pathSeparator, File.separator)
+        val resolvedPath = File(fixedBasePath, fixedRelativePath).canonicalPath
+        return resolvedPath.replace(File.separator, pathSeparator).substring(1)
+    }
+
+    /**
+     * Gets the parent directory of a path.
+     */
+    private fun getParentDirectory(path: String): String {
+        val separatorIndex = path.lastIndexOf(pathSeparator)
+        return if (separatorIndex >= 0) {
+            path.substring(0, separatorIndex)
+        } else {
+            ""
+        }
+    }
+}

server/src/main/kotlin/suwayomi/tachidesk/anime/AnimeAPI.kt

@@ -28,7 +28,7 @@ object AnimeAPI {
     fun defineEndpoints(app: Javalin) {
         // list all extensions
         app.get("/api/v1/anime/extension/list") { ctx ->
-            ctx.json(
+            ctx.future(
                 future {
                     getExtensionList()
                 }
@@ -36,10 +36,10 @@ object AnimeAPI {
         }
 
         // install extension identified with "pkgName"
-        app.get("/api/v1/anime/extension/install/:pkgName") { ctx ->
+        app.get("/api/v1/anime/extension/install/{pkgName}") { ctx ->
             val pkgName = ctx.pathParam("pkgName")
 
-            ctx.json(
+            ctx.future(
                 future {
                     installExtension(pkgName)
                 }
@@ -47,10 +47,10 @@ object AnimeAPI {
         }
 
         // update extension identified with "pkgName"
-        app.get("/api/v1/anime/extension/update/:pkgName") { ctx ->
+        app.get("/api/v1/anime/extension/update/{pkgName}") { ctx ->
             val pkgName = ctx.pathParam("pkgName")
 
-            ctx.json(
+            ctx.future(
                 future {
                     updateExtension(pkgName)
                 }
@@ -58,7 +58,7 @@ object AnimeAPI {
         }
 
         // uninstall extension identified with "pkgName"
-        app.get("/api/v1/anime/extension/uninstall/:pkgName") { ctx ->
+        app.get("/api/v1/anime/extension/uninstall/{pkgName}") { ctx ->
             val pkgName = ctx.pathParam("pkgName")
 
             uninstallExtension(pkgName)
@@ -66,10 +66,10 @@ object AnimeAPI {
         }
 
         // icon for extension named `apkName`
-        app.get("/api/v1/anime/extension/icon/:apkName") { ctx -> // TODO: move to pkgName
+        app.get("/api/v1/anime/extension/icon/{apkName}") { ctx -> // TODO: move to pkgName
             val apkName = ctx.pathParam("apkName")
 
-            ctx.result(
+            ctx.future(
                 future { getExtensionIcon(apkName) }
                     .thenApply {
                         ctx.header("content-type", it.second)
@@ -84,16 +84,16 @@ object AnimeAPI {
         }
 
         // fetch source with id `sourceId`
-        app.get("/api/v1/anime/source/:sourceId") { ctx ->
+        app.get("/api/v1/anime/source/{sourceId}") { ctx ->
             val sourceId = ctx.pathParam("sourceId").toLong()
             ctx.json(getAnimeSource(sourceId))
         }
 
         // popular animes from source with id `sourceId`
-        app.get("/api/v1/anime/source/:sourceId/popular/:pageNum") { ctx ->
+        app.get("/api/v1/anime/source/{sourceId}/popular/{pageNum}") { ctx ->
             val sourceId = ctx.pathParam("sourceId").toLong()
             val pageNum = ctx.pathParam("pageNum").toInt()
-            ctx.json(
+            ctx.future(
                 future {
                     getAnimeList(sourceId, pageNum, popular = true)
                 }
@@ -101,10 +101,10 @@ object AnimeAPI {
         }
 
         // latest animes from source with id `sourceId`
-        app.get("/api/v1/anime/source/:sourceId/latest/:pageNum") { ctx ->
+        app.get("/api/v1/anime/source/{sourceId}/latest/{pageNum}") { ctx ->
             val sourceId = ctx.pathParam("sourceId").toLong()
             val pageNum = ctx.pathParam("pageNum").toInt()
-            ctx.json(
+            ctx.future(
                 future {
                     getAnimeList(sourceId, pageNum, popular = false)
                 }
@@ -112,11 +112,11 @@ object AnimeAPI {
         }
 
         // get anime info
-        app.get("/api/v1/anime/anime/:animeId/") { ctx ->
+        app.get("/api/v1/anime/anime/{animeId}/") { ctx ->
             val animeId = ctx.pathParam("animeId").toInt()
-            val onlineFetch = ctx.queryParam("onlineFetch", "false").toBoolean()
+            val onlineFetch = ctx.queryParam("onlineFetch")?.toBoolean() ?: false
 
-            ctx.json(
+            ctx.future(
                 future {
                     getAnime(animeId, onlineFetch)
                 }
@@ -124,10 +124,10 @@ object AnimeAPI {
         }
 
         // anime thumbnail
-        app.get("api/v1/anime/anime/:animeId/thumbnail") { ctx ->
+        app.get("api/v1/anime/anime/{animeId}/thumbnail") { ctx ->
             val animeId = ctx.pathParam("animeId").toInt()
 
-            ctx.result(
+            ctx.future(
                 future { getAnimeThumbnail(animeId) }
                     .thenApply {
                         ctx.header("content-type", it.second)
@@ -137,13 +137,13 @@ object AnimeAPI {
         }
 //
 //        // list manga's categories
-//        app.get("api/v1/manga/:mangaId/category/") { ctx ->
+//        app.get("api/v1/manga/{mangaId}/category/") { ctx ->
 //            val mangaId = ctx.pathParam("mangaId").toInt()
 //            ctx.json(getMangaCategories(mangaId))
 //        }
 //
 //        // adds the manga to category
-//        app.get("api/v1/manga/:mangaId/category/:categoryId") { ctx ->
+//        app.get("api/v1/manga/{mangaId}/category/{categoryId}") { ctx ->
 //            val mangaId = ctx.pathParam("mangaId").toInt()
 //            val categoryId = ctx.pathParam("categoryId").toInt()
 //            addMangaToCategory(mangaId, categoryId)
@@ -151,7 +151,7 @@ object AnimeAPI {
 //        }
 //
 //        // removes the manga from the category
-//        app.delete("api/v1/manga/:mangaId/category/:categoryId") { ctx ->
+//        app.delete("api/v1/manga/{mangaId}/category/{categoryId}") { ctx ->
 //            val mangaId = ctx.pathParam("mangaId").toInt()
 //            val categoryId = ctx.pathParam("categoryId").toInt()
 //            removeMangaFromCategory(mangaId, categoryId)
@@ -159,23 +159,23 @@ object AnimeAPI {
 //        }
 //
         // get episode list when showing a anime
-        app.get("/api/v1/anime/anime/:animeId/episodes") { ctx ->
+        app.get("/api/v1/anime/anime/{animeId}/episodes") { ctx ->
             val animeId = ctx.pathParam("animeId").toInt()
 
             val onlineFetch = ctx.queryParam("onlineFetch")?.toBoolean()
 
-            ctx.json(future { getEpisodeList(animeId, onlineFetch) })
+            ctx.future(future { getEpisodeList(animeId, onlineFetch) })
         }
 
         // used to display a episode, get a episode in order to show it's <Quality pending>
-        app.get("/api/v1/anime/anime/:animeId/episode/:episodeIndex") { ctx ->
+        app.get("/api/v1/anime/anime/{animeId}/episode/{episodeIndex}") { ctx ->
             val episodeIndex = ctx.pathParam("episodeIndex").toInt()
             val animeId = ctx.pathParam("animeId").toInt()
-            ctx.json(future { getEpisode(episodeIndex, animeId) })
+            ctx.future(future { getEpisode(episodeIndex, animeId) })
         }
 
         // used to modify a episode's parameters
-        app.patch("/api/v1/anime/anime/:animeId/episode/:episodeIndex") { ctx ->
+        app.patch("/api/v1/anime/anime/{animeId}/episode/{episodeIndex}") { ctx ->
             val episodeIndex = ctx.pathParam("episodeIndex").toInt()
             val animeId = ctx.pathParam("animeId").toInt()
 
@@ -190,7 +190,7 @@ object AnimeAPI {
         }
 //
 //        // get page at index "index"
-//        app.get("/api/v1/manga/:mangaId/chapter/:chapterIndex/page/:index") { ctx ->
+//        app.get("/api/v1/manga/{mangaId}/chapter/{chapterIndex}/page/{index}") { ctx ->
 //            val mangaId = ctx.pathParam("mangaId").toInt()
 //            val chapterIndex = ctx.pathParam("chapterIndex").toInt()
 //            val index = ctx.pathParam("index").toInt()
@@ -205,49 +205,49 @@ object AnimeAPI {
 //        }
 //
 //        // submit a chapter for download
-//        app.put("/api/v1/manga/:mangaId/chapter/:chapterIndex/download") { ctx ->
+//        app.put("/api/v1/manga/{mangaId}/chapter/{chapterIndex}/download") { ctx ->
 //            // TODO
 //        }
 //
 //        // cancel a chapter download
-//        app.delete("/api/v1/manga/:mangaId/chapter/:chapterIndex/download") { ctx ->
+//        app.delete("/api/v1/manga/{mangaId}/chapter/{chapterIndex}/download") { ctx ->
 //            // TODO
 //        }
 //
 //        // global search, Not implemented yet
-//        app.get("/api/v1/search/:searchTerm") { ctx ->
+//        app.get("/api/v1/search/{searchTerm}") { ctx ->
 //            val searchTerm = ctx.pathParam("searchTerm")
 //            ctx.json(sourceGlobalSearch(searchTerm))
 //        }
 //
         // single source search
-        app.get("/api/v1/anime/source/:sourceId/search/:searchTerm/:pageNum") { ctx ->
+        app.get("/api/v1/anime/source/{sourceId}/search/{searchTerm}/{pageNum}") { ctx ->
             val sourceId = ctx.pathParam("sourceId").toLong()
             val searchTerm = ctx.pathParam("searchTerm")
             val pageNum = ctx.pathParam("pageNum").toInt()
-            ctx.json(future { sourceSearch(sourceId, searchTerm, pageNum) })
+            ctx.future(future { sourceSearch(sourceId, searchTerm, pageNum) })
         }
 //
 //        // source filter list
-//        app.get("/api/v1/source/:sourceId/filters/") { ctx ->
+//        app.get("/api/v1/source/{sourceId}/filters/") { ctx ->
 //            val sourceId = ctx.pathParam("sourceId").toLong()
 //            ctx.json(sourceFilters(sourceId))
 //        }
 //
 //        // adds the manga to library
-//        app.get("api/v1/manga/:mangaId/library") { ctx ->
+//        app.get("api/v1/manga/{mangaId}/library") { ctx ->
 //            val mangaId = ctx.pathParam("mangaId").toInt()
 //
-//            ctx.result(
+//            ctx.future(
 //                JavalinSetup.future { addMangaToLibrary(mangaId) }
 //            )
 //        }
 //
 //        // removes the manga from the library
-//        app.delete("api/v1/manga/:mangaId/library") { ctx ->
+//        app.delete("api/v1/manga/{mangaId}/library") { ctx ->
 //            val mangaId = ctx.pathParam("mangaId").toInt()
 //
-//            ctx.result(
+//            ctx.future(
 //                JavalinSetup.future { removeMangaFromLibrary(mangaId) }
 //            )
 //        }
@@ -275,7 +275,7 @@ object AnimeAPI {
 //        }
 //
 //        // category modification
-//        app.patch("/api/v1/category/:categoryId") { ctx ->
+//        app.patch("/api/v1/category/{categoryId}") { ctx ->
 //            val categoryId = ctx.pathParam("categoryId").toInt()
 //            val name = ctx.formParam("name")
 //            val isDefault = ctx.formParam("default")?.toBoolean()
@@ -284,7 +284,7 @@ object AnimeAPI {
 //        }
 //
 //        // category re-ordering
-//        app.patch("/api/v1/category/:categoryId/reorder") { ctx ->
+//        app.patch("/api/v1/category/{categoryId}/reorder") { ctx ->
 //            val categoryId = ctx.pathParam("categoryId").toInt()
 //            val from = ctx.formParam("from")!!.toInt()
 //            val to = ctx.formParam("to")!!.toInt()
@@ -293,21 +293,21 @@ object AnimeAPI {
 //        }
 //
 //        // category delete
-//        app.delete("/api/v1/category/:categoryId") { ctx ->
+//        app.delete("/api/v1/category/{categoryId}") { ctx ->
 //            val categoryId = ctx.pathParam("categoryId").toInt()
 //            Category.removeCategory(categoryId)
 //            ctx.status(200)
 //        }
 //
 //        // returns the manga list associated with a category
-//        app.get("/api/v1/category/:categoryId") { ctx ->
+//        app.get("/api/v1/category/{categoryId}") { ctx ->
 //            val categoryId = ctx.pathParam("categoryId").toInt()
 //            ctx.json(getCategoryMangaList(categoryId))
 //        }
 //
 //        // expects a Tachiyomi legacy backup json in the body
 //        app.post("/api/v1/backup/legacy/import") { ctx ->
-//            ctx.result(
+//            ctx.future(
 //                future {
 //                    restoreLegacyBackup(ctx.bodyAsInputStream())
 //                }
@@ -316,7 +316,7 @@ object AnimeAPI {
 //
 //        // expects a Tachiyomi legacy backup json as a file upload, the file must be named "backup.json"
 //        app.post("/api/v1/backup/legacy/import/file") { ctx ->
-//            ctx.result(
+//            ctx.future(
 //                JavalinSetup.future {
 //                    restoreLegacyBackup(ctx.uploadedFile("backup.json")!!.content)
 //                }
@@ -326,7 +326,7 @@ object AnimeAPI {
 //        // returns a Tachiyomi legacy backup json created from the current database as a json body
 //        app.get("/api/v1/backup/legacy/export") { ctx ->
 //            ctx.contentType("application/json")
-//            ctx.result(
+//            ctx.future(
 //                JavalinSetup.future {
 //                    createLegacyBackup(
 //                        BackupFlags(
@@ -348,7 +348,7 @@ object AnimeAPI {
 //            val currentDate = sdf.format(Date())
 //
 //            ctx.header("Content-Disposition", "attachment; filename=\"tachidesk_$currentDate.json\"")
-//            ctx.result(
+//            ctx.future(
 //                JavalinSetup.future {
 //                    createLegacyBackup(
 //                        BackupFlags(

server/src/main/kotlin/suwayomi/tachidesk/anime/impl/Anime.kt

@@ -23,8 +23,8 @@ import suwayomi.tachidesk.anime.model.table.AnimeStatus
 import suwayomi.tachidesk.anime.model.table.AnimeTable
 import suwayomi.tachidesk.manga.impl.util.lang.awaitSingle
 import suwayomi.tachidesk.manga.impl.util.network.await
-import suwayomi.tachidesk.manga.impl.util.storage.CachedImageResponse.clearCachedImage
+import suwayomi.tachidesk.manga.impl.util.storage.ImageResponse.clearCachedImage
-import suwayomi.tachidesk.manga.impl.util.storage.CachedImageResponse.getCachedImageResponse
+import suwayomi.tachidesk.manga.impl.util.storage.ImageResponse.getCachedImageResponse
 import suwayomi.tachidesk.server.ApplicationDirs
 import java.io.InputStream
 

server/src/main/kotlin/suwayomi/tachidesk/anime/impl/extension/Extension.kt

@@ -40,7 +40,7 @@ import suwayomi.tachidesk.anime.impl.util.PackageTools.trustedSignatures
 import suwayomi.tachidesk.anime.model.table.AnimeExtensionTable
 import suwayomi.tachidesk.anime.model.table.AnimeSourceTable
 import suwayomi.tachidesk.manga.impl.util.network.await
-import suwayomi.tachidesk.manga.impl.util.storage.CachedImageResponse.getCachedImageResponse
+import suwayomi.tachidesk.manga.impl.util.storage.ImageResponse.getCachedImageResponse
 import suwayomi.tachidesk.server.ApplicationDirs
 import uy.kohesive.injekt.injectLazy
 import java.io.File

server/src/main/kotlin/suwayomi/tachidesk/anime/impl/util/PackageTools.kt

@@ -82,7 +82,7 @@ object PackageTools {
             )
             handler.dump(errorFile, emptyArray<String>())
         } else {
-            BytecodeEditor.fixAndroidClasses(jarFilePath.toFile())
+            BytecodeEditor.fixAndroidClasses(jarFilePath)
         }
     }
 

server/src/main/kotlin/suwayomi/tachidesk/anime/model/table/AnimeTable.kt

@@ -12,6 +12,7 @@ import org.jetbrains.exposed.dao.id.IntIdTable
 import org.jetbrains.exposed.sql.ResultRow
 import suwayomi.tachidesk.manga.impl.MangaList.proxyThumbnailUrl
 import suwayomi.tachidesk.manga.model.dataclass.MangaDataClass
+import suwayomi.tachidesk.manga.model.dataclass.toGenreList
 import suwayomi.tachidesk.manga.model.table.MangaStatus.Companion
 
 object AnimeTable : IntIdTable() {
@@ -48,7 +49,7 @@ fun AnimeTable.toDataClass(mangaEntry: ResultRow) =
         mangaEntry[artist],
         mangaEntry[author],
         mangaEntry[description],
-        mangaEntry[genre],
+        mangaEntry[genre].toGenreList(),
         Companion.valueOf(mangaEntry[status]).name,
         mangaEntry[inLibrary]
     )

server/src/main/kotlin/suwayomi/tachidesk/manga/MangaAPI.kt

@@ -14,9 +14,9 @@ import io.javalin.apibuilder.ApiBuilder.path
 import io.javalin.apibuilder.ApiBuilder.post
 import io.javalin.apibuilder.ApiBuilder.ws
 import suwayomi.tachidesk.manga.controller.BackupController
+import suwayomi.tachidesk.manga.controller.CategoryController
 import suwayomi.tachidesk.manga.controller.DownloadController
 import suwayomi.tachidesk.manga.controller.ExtensionController
-import suwayomi.tachidesk.manga.controller.LibraryController
 import suwayomi.tachidesk.manga.controller.MangaController
 import suwayomi.tachidesk.manga.controller.SourceController
 
@@ -25,72 +25,73 @@ object MangaAPI {
         path("extension") {
             get("list", ExtensionController::list)
 
-            get("install/:pkgName", ExtensionController::install)
+            get("install/{pkgName}", ExtensionController::install)
-            get("update/:pkgName", ExtensionController::update)
+            post("install", ExtensionController::installFile)
-            get("uninstall/:pkgName", ExtensionController::uninstall)
+            get("update/{pkgName}", ExtensionController::update)
+            get("uninstall/{pkgName}", ExtensionController::uninstall)
 
-            get("icon/:apkName", ExtensionController::icon)
+            get("icon/{apkName}", ExtensionController::icon)
         }
 
         path("source") {
             get("list", SourceController::list)
-            get(":sourceId", SourceController::retrieve)
+            get("{sourceId}", SourceController::retrieve)
 
-            get(":sourceId/popular/:pageNum", SourceController::popular)
+            get("{sourceId}/popular/{pageNum}", SourceController::popular)
-            get(":sourceId/latest/:pageNum", SourceController::latest)
+            get("{sourceId}/latest/{pageNum}", SourceController::latest)
 
-            get(":sourceId/preferences", SourceController::getPreferences)
+            get("{sourceId}/preferences", SourceController::getPreferences)
-            post(":sourceId/preferences", SourceController::setPreference)
+            post("{sourceId}/preferences", SourceController::setPreference)
 
-            post(":sourceId/filters", SourceController::filters) // TODO
+            get("{sourceId}/filters", SourceController::filters)
 
-            get(":sourceId/search/:searchTerm/:pageNum", SourceController::searchSingle)
+            get("{sourceId}/search/{searchTerm}/{pageNum}", SourceController::searchSingle)
-            get("search/:searchTerm/:pageNum", SourceController::searchSingle) // TODO
+//            get("search/{searchTerm}/{pageNum}", SourceController::searchGlobal)
         }
 
         path("manga") {
-            get(":mangaId", MangaController::retrieve)
+            get("{mangaId}", MangaController::retrieve)
-            get(":mangaId/thumbnail", MangaController::thumbnail)
+            get("{mangaId}/thumbnail", MangaController::thumbnail)
 
-            get(":mangaId/category", MangaController::categoryList)
+            get("{mangaId}/category", MangaController::categoryList)
-            get(":mangaId/category/:categoryId", MangaController::addToCategory)
+            get("{mangaId}/category/{categoryId}", MangaController::addToCategory)
-            delete(":mangaId/category/:categoryId", MangaController::removeFromCategory)
+            delete("{mangaId}/category/{categoryId}", MangaController::removeFromCategory)
 
-            get(":mangaId/library", MangaController::addToLibrary)
+            get("{mangaId}/library", MangaController::addToLibrary)
-            delete(":mangaId/library", MangaController::removeFromLibrary)
+            delete("{mangaId}/library", MangaController::removeFromLibrary)
 
-            patch(":mangaId/meta", MangaController::meta)
+            patch("{mangaId}/meta", MangaController::meta)
 
-            get(":mangaId/chapters", MangaController::chapterList)
+            get("{mangaId}/chapters", MangaController::chapterList)
-            get(":mangaId/chapter/:chapterIndex", MangaController::chapterRetrieve)
+            get("{mangaId}/chapter/{chapterIndex}", MangaController::chapterRetrieve)
-            patch(":mangaId/chapter/:chapterIndex", MangaController::chapterModify)
+            patch("{mangaId}/chapter/{chapterIndex}", MangaController::chapterModify)
+            delete("{mangaId}/chapter/{chapterIndex}", MangaController::chapterDelete)
 
-            patch(":mangaId/chapter/:chapterIndex/meta", MangaController::chapterMeta)
+            patch("{mangaId}/chapter/{chapterIndex}/meta", MangaController::chapterMeta)
 
-            get(":mangaId/chapter/:chapterIndex/page/:index", MangaController::pageRetrieve)
+            get("{mangaId}/chapter/{chapterIndex}/page/{index}", MangaController::pageRetrieve)
         }
 
-        path("") {
+        path("category") {
-            get("library", LibraryController::list)
+            get("", CategoryController::categoryList)
+            post("", CategoryController::categoryCreate)
 
-            path("category") {
+            get("{categoryId}", CategoryController::categoryMangas)
-                get("", LibraryController::categoryList)
+            patch("{categoryId}", CategoryController::categoryModify)
-                post("", LibraryController::categoryCreate)
+            delete("{categoryId}", CategoryController::categoryDelete)
 
-                get(":categoryId", LibraryController::categoryMangas)
+            patch("reorder", CategoryController::categoryReorder)
-                patch(":categoryId", LibraryController::categoryModify)
-                delete(":categoryId", LibraryController::categoryDelete)
-
-                patch(":categoryId/reorder", LibraryController::categoryReorder) // TODO: the underlying code doesn't need `:categoryId`, remove it
-            }
         }
 
         path("backup") {
-            post("legacy/import", BackupController::legacyImport)
+            post("import", BackupController::protobufImport)
-            post("legacy/import/file", BackupController::legacyImportFile)
+            post("import/file", BackupController::protobufImportFile)
 
-            get("legacy/export", BackupController::legacyExport)
+            post("validate", BackupController::protobufValidate)
-            get("legacy/export/file", BackupController::legacyExportFile)
+            post("validate/file", BackupController::protobufValidateFile)
+
+            get("export", BackupController::protobufExport)
+            get("export/file", BackupController::protobufExportFile)
         }
 
         path("downloads") {
@@ -102,8 +103,8 @@ object MangaAPI {
         }
 
         path("download") {
-            get(":mangaId/chapter/:chapterIndex", DownloadController::queueChapter)
+            get("{mangaId}/chapter/{chapterIndex}", DownloadController::queueChapter)
-            delete(":mangaId/chapter/:chapterIndex", DownloadController::unqueueChapter)
+            delete("{mangaId}/chapter/{chapterIndex}", DownloadController::unqueueChapter)
         }
     }
 }

server/src/main/kotlin/suwayomi/tachidesk/manga/controller/BackupController.kt

@@ -2,9 +2,10 @@ package suwayomi.tachidesk.manga.controller
 
 import io.javalin.http.Context
 import suwayomi.tachidesk.manga.impl.backup.BackupFlags
-import suwayomi.tachidesk.manga.impl.backup.legacy.LegacyBackupExport
+import suwayomi.tachidesk.manga.impl.backup.proto.ProtoBackupExport
-import suwayomi.tachidesk.manga.impl.backup.legacy.LegacyBackupImport
+import suwayomi.tachidesk.manga.impl.backup.proto.ProtoBackupImport
-import suwayomi.tachidesk.server.JavalinSetup
+import suwayomi.tachidesk.manga.impl.backup.proto.ProtoBackupValidator
+import suwayomi.tachidesk.server.JavalinSetup.future
 import java.text.SimpleDateFormat
 import java.util.Date
 
@@ -16,30 +17,32 @@ import java.util.Date
  * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
 
 object BackupController {
-    /** expects a Tachiyomi legacy backup json in the body */
+
-    fun legacyImport(ctx: Context) {
+    /** expects a Tachiyomi protobuf backup in the body */
-        ctx.result(
+    fun protobufImport(ctx: Context) {
-            JavalinSetup.future {
+        ctx.future(
-                LegacyBackupImport.restoreLegacyBackup(ctx.bodyAsInputStream())
+            future {
+                ProtoBackupImport.performRestore(ctx.bodyAsInputStream())
             }
         )
     }
 
-    /** expects a Tachiyomi legacy backup json as a file upload, the file must be named "backup.json" */
+    /** expects a Tachiyomi protobuf backup as a file upload, the file must be named "backup.proto.gz" */
-    fun legacyImportFile(ctx: Context) {
+    fun protobufImportFile(ctx: Context) {
-        ctx.result(
+        // TODO: rewrite this with ctx.uploadedFiles(), don't call the multipart field "backup.proto.gz"
-            JavalinSetup.future {
+        ctx.future(
-                LegacyBackupImport.restoreLegacyBackup(ctx.uploadedFile("backup.json")!!.content)
+            future {
+                ProtoBackupImport.performRestore(ctx.uploadedFile("backup.proto.gz")!!.content)
             }
         )
     }
 
-    /** returns a Tachiyomi legacy backup json created from the current database as a json body */
+    /** returns a Tachiyomi protobuf backup created from the current database as a body */
-    fun legacyExport(ctx: Context) {
+    fun protobufExport(ctx: Context) {
-        ctx.contentType("application/json")
+        ctx.contentType("application/octet-stream")
-        ctx.result(
+        ctx.future(
-            JavalinSetup.future {
+            future {
-                LegacyBackupExport.createLegacyBackup(
+                ProtoBackupExport.createBackup(
                     BackupFlags(
                         includeManga = true,
                         includeCategories = true,
@@ -52,16 +55,15 @@ object BackupController {
         )
     }
 
-    /** returns a Tachiyomi legacy backup json created from the current database as a file */
+    /** returns a Tachiyomi protobuf backup created from the current database as a file */
-    fun legacyExportFile(ctx: Context) {
+    fun protobufExportFile(ctx: Context) {
-        ctx.contentType("application/json")
+        ctx.contentType("application/octet-stream")
-        val sdf = SimpleDateFormat("yyyy-MM-dd_HH-mm")
+        val currentDate = SimpleDateFormat("yyyy-MM-dd_HH-mm").format(Date())
-        val currentDate = sdf.format(Date())
 
-        ctx.header("Content-Disposition", "attachment; filename=\"tachidesk_$currentDate.json\"")
+        ctx.header("Content-Disposition", """attachment; filename="tachidesk_$currentDate.proto.gz"""")
-        ctx.result(
+        ctx.future(
-            JavalinSetup.future {
+            future {
-                LegacyBackupExport.createLegacyBackup(
+                ProtoBackupExport.createBackup(
                     BackupFlags(
                         includeManga = true,
                         includeCategories = true,
@@ -73,4 +75,22 @@ object BackupController {
             }
         )
     }
+
+    /** Reports missing sources and trackers, expects a Tachiyomi protobuf backup in the body */
+    fun protobufValidate(ctx: Context) {
+        ctx.future(
+            future {
+                ProtoBackupValidator.validate(ctx.bodyAsInputStream())
+            }
+        )
+    }
+
+    /** Reports missing sources and trackers, expects a Tachiyomi protobuf backup as a file upload, the file must be named "backup.proto.gz" */
+    fun protobufValidateFile(ctx: Context) {
+        ctx.future(
+            future {
+                ProtoBackupValidator.validate(ctx.uploadedFile("backup.proto.gz")!!.content)
+            }
+        )
+    }
 }

server/src/main/kotlin/suwayomi/tachidesk/manga/controller/CategoryController.kt

@@ -10,14 +10,8 @@ package suwayomi.tachidesk.manga.controller
 import io.javalin.http.Context
 import suwayomi.tachidesk.manga.impl.Category
 import suwayomi.tachidesk.manga.impl.CategoryManga
-import suwayomi.tachidesk.manga.impl.Library
-
-object LibraryController {
-    /** lists mangas that have no category assigned */
-    fun list(ctx: Context) {
-        ctx.json(Library.getLibraryMangas())
-    }
 
+object CategoryController {
     /** category list */
     fun categoryList(ctx: Context) {
         ctx.json(Category.getCategoryList())

server/src/main/kotlin/suwayomi/tachidesk/manga/controller/DownloadController.kt

@@ -8,12 +8,12 @@ package suwayomi.tachidesk.manga.controller
  * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
 
 import io.javalin.http.Context
-import io.javalin.websocket.WsHandler
+import io.javalin.websocket.WsConfig
 import suwayomi.tachidesk.manga.impl.download.DownloadManager
 
 object DownloadController {
     /** Download queue stats */
-    fun downloadsWS(ws: WsHandler) {
+    fun downloadsWS(ws: WsConfig) {
         ws.onConnect { ctx ->
             DownloadManager.addClient(ctx)
             DownloadManager.notifyClient(ctx)

server/src/main/kotlin/suwayomi/tachidesk/manga/controller/ExtensionController.kt

@@ -8,14 +8,17 @@ package suwayomi.tachidesk.manga.controller
  * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
 
 import io.javalin.http.Context
+import mu.KotlinLogging
 import suwayomi.tachidesk.manga.impl.extension.Extension
 import suwayomi.tachidesk.manga.impl.extension.ExtensionsList
 import suwayomi.tachidesk.server.JavalinSetup.future
 
 object ExtensionController {
+    private val logger = KotlinLogging.logger {}
+
     /** list all extensions */
     fun list(ctx: Context) {
-        ctx.json(
+        ctx.future(
             future {
                 ExtensionsList.getExtensionList()
             }
@@ -26,18 +29,31 @@ object ExtensionController {
     fun install(ctx: Context) {
         val pkgName = ctx.pathParam("pkgName")
 
-        ctx.json(
+        ctx.future(
             future {
                 Extension.installExtension(pkgName)
             }
         )
     }
 
+    /** install the uploaded apk file */
+    fun installFile(ctx: Context) {
+
+        val uploadedFile = ctx.uploadedFile("file")!!
+        logger.debug { "Uploaded extension file name: " + uploadedFile.filename }
+
+        ctx.future(
+            future {
+                Extension.installExternalExtension(uploadedFile.content, uploadedFile.filename)
+            }
+        )
+    }
+
     /** update extension identified with "pkgName" */
     fun update(ctx: Context) {
         val pkgName = ctx.pathParam("pkgName")
 
-        ctx.json(
+        ctx.future(
             future {
                 Extension.updateExtension(pkgName)
             }
@@ -55,9 +71,10 @@ object ExtensionController {
     /** icon for extension named `apkName` */
     fun icon(ctx: Context) {
         val apkName = ctx.pathParam("apkName")
+        val useCache = ctx.queryParam("useCache")?.toBoolean() ?: true
 
-        ctx.result(
+        ctx.future(
-            future { Extension.getExtensionIcon(apkName) }
+            future { Extension.getExtensionIcon(apkName, useCache) }
                 .thenApply {
                     ctx.header("content-type", it.second)
                     it.first

server/src/main/kotlin/suwayomi/tachidesk/manga/controller/MangaController.kt

@@ -19,9 +19,9 @@ object MangaController {
     /** get manga info */
     fun retrieve(ctx: Context) {
         val mangaId = ctx.pathParam("mangaId").toInt()
-        val onlineFetch = ctx.queryParam("onlineFetch", "false").toBoolean()
+        val onlineFetch = ctx.queryParam("onlineFetch")?.toBoolean() ?: false
 
-        ctx.json(
+        ctx.future(
             future {
                 Manga.getManga(mangaId, onlineFetch)
             }
@@ -31,9 +31,10 @@ object MangaController {
     /** manga thumbnail */
     fun thumbnail(ctx: Context) {
         val mangaId = ctx.pathParam("mangaId").toInt()
+        val useCache = ctx.queryParam("useCache")?.toBoolean() ?: true
 
-        ctx.result(
+        ctx.future(
-            future { Manga.getMangaThumbnail(mangaId) }
+            future { Manga.getMangaThumbnail(mangaId, useCache) }
                 .thenApply {
                     ctx.header("content-type", it.second)
                     it.first
@@ -45,7 +46,7 @@ object MangaController {
     fun addToLibrary(ctx: Context) {
         val mangaId = ctx.pathParam("mangaId").toInt()
 
-        ctx.result(
+        ctx.future(
             future { Library.addMangaToLibrary(mangaId) }
         )
     }
@@ -54,7 +55,7 @@ object MangaController {
     fun removeFromLibrary(ctx: Context) {
         val mangaId = ctx.pathParam("mangaId").toInt()
 
-        ctx.result(
+        ctx.future(
             future { Library.removeMangaFromLibrary(mangaId) }
         )
     }
@@ -97,16 +98,16 @@ object MangaController {
     fun chapterList(ctx: Context) {
         val mangaId = ctx.pathParam("mangaId").toInt()
 
-        val onlineFetch = ctx.queryParam("onlineFetch", "false").toBoolean()
+        val onlineFetch = ctx.queryParam("onlineFetch")?.toBoolean() ?: false
 
-        ctx.json(future { Chapter.getChapterList(mangaId, onlineFetch) })
+        ctx.future(future { Chapter.getChapterList(mangaId, onlineFetch) })
     }
 
     /** used to display a chapter, get a chapter in order to show its pages */
     fun chapterRetrieve(ctx: Context) {
         val chapterIndex = ctx.pathParam("chapterIndex").toInt()
         val mangaId = ctx.pathParam("mangaId").toInt()
-        ctx.json(future { Chapter.getChapter(chapterIndex, mangaId) })
+        ctx.future(future { Chapter.getChapter(chapterIndex, mangaId) })
     }
 
     /** used to modify a chapter's parameters */
@@ -124,6 +125,16 @@ object MangaController {
         ctx.status(200)
     }
 
+    /** delete a downloaded chapter */
+    fun chapterDelete(ctx: Context) {
+        val chapterIndex = ctx.pathParam("chapterIndex").toInt()
+        val mangaId = ctx.pathParam("mangaId").toInt()
+
+        Chapter.deleteChapter(mangaId, chapterIndex)
+
+        ctx.status(200)
+    }
+
     /** used to modify a chapter's meta parameters */
     fun chapterMeta(ctx: Context) {
         val chapterIndex = ctx.pathParam("chapterIndex").toInt()
@@ -142,9 +153,10 @@ object MangaController {
         val mangaId = ctx.pathParam("mangaId").toInt()
         val chapterIndex = ctx.pathParam("chapterIndex").toInt()
         val index = ctx.pathParam("index").toInt()
+        val useCache = ctx.queryParam("useCache")?.toBoolean() ?: true
 
-        ctx.result(
+        ctx.future(
-            future { Page.getPageImage(mangaId, chapterIndex, index) }
+            future { Page.getPageImage(mangaId, chapterIndex, index, useCache) }
                 .thenApply {
                     ctx.header("content-type", it.second)
                     it.first

server/src/main/kotlin/suwayomi/tachidesk/manga/controller/SourceController.kt

@@ -12,7 +12,6 @@ import suwayomi.tachidesk.manga.impl.MangaList
 import suwayomi.tachidesk.manga.impl.Search
 import suwayomi.tachidesk.manga.impl.Source
 import suwayomi.tachidesk.manga.impl.Source.SourcePreferenceChange
-import suwayomi.tachidesk.server.JavalinSetup
 import suwayomi.tachidesk.server.JavalinSetup.future
 
 object SourceController {
@@ -31,7 +30,7 @@ object SourceController {
     fun popular(ctx: Context) {
         val sourceId = ctx.pathParam("sourceId").toLong()
         val pageNum = ctx.pathParam("pageNum").toInt()
-        ctx.json(
+        ctx.future(
             future {
                 MangaList.getMangaList(sourceId, pageNum, popular = true)
             }
@@ -42,7 +41,7 @@ object SourceController {
     fun latest(ctx: Context) {
         val sourceId = ctx.pathParam("sourceId").toLong()
         val pageNum = ctx.pathParam("pageNum").toInt()
-        ctx.json(
+        ctx.future(
             future {
                 MangaList.getMangaList(sourceId, pageNum, popular = false)
             }
@@ -63,9 +62,11 @@ object SourceController {
     }
 
     /** fetch filters of source with id `sourceId` */
-    fun filters(ctx: Context) { // TODO
+    fun filters(ctx: Context) {
         val sourceId = ctx.pathParam("sourceId").toLong()
-        ctx.json(Search.sourceFilters(sourceId))
+        val reset = ctx.queryParam("reset")?.toBoolean() ?: false
+
+        ctx.json(Search.getInitialFilterList(sourceId, reset))
     }
 
     /** single source search */
@@ -73,7 +74,7 @@ object SourceController {
         val sourceId = ctx.pathParam("sourceId").toLong()
         val searchTerm = ctx.pathParam("searchTerm")
         val pageNum = ctx.pathParam("pageNum").toInt()
-        ctx.json(JavalinSetup.future { Search.sourceSearch(sourceId, searchTerm, pageNum) })
+        ctx.future(future { Search.sourceSearch(sourceId, searchTerm, pageNum) })
     }
 
     /** all source search */

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Category.kt

@@ -8,6 +8,7 @@ package suwayomi.tachidesk.manga.impl
  * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
 
 import org.jetbrains.exposed.sql.SortOrder
+import org.jetbrains.exposed.sql.and
 import org.jetbrains.exposed.sql.deleteWhere
 import org.jetbrains.exposed.sql.insert
 import org.jetbrains.exposed.sql.select
@@ -15,9 +16,11 @@ import org.jetbrains.exposed.sql.selectAll
 import org.jetbrains.exposed.sql.transactions.transaction
 import org.jetbrains.exposed.sql.update
 import suwayomi.tachidesk.manga.impl.CategoryManga.removeMangaFromCategory
+import suwayomi.tachidesk.manga.impl.util.lang.isNotEmpty
 import suwayomi.tachidesk.manga.model.dataclass.CategoryDataClass
 import suwayomi.tachidesk.manga.model.table.CategoryMangaTable
 import suwayomi.tachidesk.manga.model.table.CategoryTable
+import suwayomi.tachidesk.manga.model.table.MangaTable
 import suwayomi.tachidesk.manga.model.table.toDataClass
 
 object Category {
@@ -25,27 +28,31 @@ object Category {
      * The new category will be placed at the end of the list
      */
     fun createCategory(name: String) {
+        // creating a category named Default is illegal
+        if (name.equals(DEFAULT_CATEGORY_NAME, ignoreCase = true)) return
+
         transaction {
-            val count = CategoryTable.selectAll().count()
+            if (CategoryTable.select { CategoryTable.name eq name }.firstOrNull() == null) {
-            if (CategoryTable.select { CategoryTable.name eq name }.firstOrNull() == null)
                 CategoryTable.insert {
                     it[CategoryTable.name] = name
-                    it[CategoryTable.order] = count.toInt() + 1
+                    it[CategoryTable.order] = Int.MAX_VALUE
                 }
+                normalizeCategories()
+            }
         }
     }
 
     fun updateCategory(categoryId: Int, name: String?, isDefault: Boolean?) {
         transaction {
             CategoryTable.update({ CategoryTable.id eq categoryId }) {
-                if (name != null) it[CategoryTable.name] = name
+                if (name != null && !name.equals(DEFAULT_CATEGORY_NAME, ignoreCase = true)) it[CategoryTable.name] = name
                 if (isDefault != null) it[CategoryTable.isDefault] = isDefault
             }
         }
     }
 
     /**
-     * Move the category from position `from` to `to`
+     * Move the category from order number `from` to `to`
      */
     fun reorderCategory(from: Int, to: Int) {
         transaction {
@@ -65,14 +72,38 @@ object Category {
                 removeMangaFromCategory(it[CategoryMangaTable.manga].value, categoryId)
             }
             CategoryTable.deleteWhere { CategoryTable.id eq categoryId }
+            normalizeCategories()
         }
     }
 
-    fun getCategoryList(): List<CategoryDataClass> {
+    /** make sure category order numbers starts from 1 and is consecutive */
-        return transaction {
+    private fun normalizeCategories() {
-            CategoryTable.selectAll().orderBy(CategoryTable.order to SortOrder.ASC).map {
+        transaction {
-                CategoryTable.toDataClass(it)
+            val categories = CategoryTable.selectAll().orderBy(CategoryTable.order to SortOrder.ASC)
+            categories.forEachIndexed { index, cat ->
+                CategoryTable.update({ CategoryTable.id eq cat[CategoryTable.id].value }) {
+                    it[CategoryTable.order] = index + 1
+                }
             }
         }
     }
+
+    const val DEFAULT_CATEGORY_ID = 0
+    const val DEFAULT_CATEGORY_NAME = "Default"
+    private fun addDefaultIfNecessary(categories: List<CategoryDataClass>): List<CategoryDataClass> =
+        if (MangaTable.select { (MangaTable.inLibrary eq true) and (MangaTable.defaultCategory eq true) }.isNotEmpty()) {
+            listOf(CategoryDataClass(DEFAULT_CATEGORY_ID, 0, DEFAULT_CATEGORY_NAME, true)) + categories
+        } else {
+            categories
+        }
+
+    fun getCategoryList(): List<CategoryDataClass> {
+        return transaction {
+            val categories = CategoryTable.selectAll().orderBy(CategoryTable.order to SortOrder.ASC).map {
+                CategoryTable.toDataClass(it)
+            }
+
+            addDefaultIfNecessary(categories)
+        }
+    }
 }

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/CategoryManga.kt

@@ -14,6 +14,8 @@ import org.jetbrains.exposed.sql.insert
 import org.jetbrains.exposed.sql.select
 import org.jetbrains.exposed.sql.transactions.transaction
 import org.jetbrains.exposed.sql.update
+import suwayomi.tachidesk.manga.impl.Category.DEFAULT_CATEGORY_ID
+import suwayomi.tachidesk.manga.impl.util.lang.isEmpty
 import suwayomi.tachidesk.manga.model.dataclass.CategoryDataClass
 import suwayomi.tachidesk.manga.model.dataclass.MangaDataClass
 import suwayomi.tachidesk.manga.model.table.CategoryMangaTable
@@ -23,8 +25,10 @@ import suwayomi.tachidesk.manga.model.table.toDataClass
 
 object CategoryManga {
     fun addMangaToCategory(mangaId: Int, categoryId: Int) {
+        fun notAlreadyInCategory() = CategoryMangaTable.select { (CategoryMangaTable.category eq categoryId) and (CategoryMangaTable.manga eq mangaId) }.isEmpty()
+
         transaction {
-            if (CategoryMangaTable.select { (CategoryMangaTable.category eq categoryId) and (CategoryMangaTable.manga eq mangaId) }.firstOrNull() == null) {
+            if (notAlreadyInCategory()) {
                 CategoryMangaTable.insert {
                     it[CategoryMangaTable.category] = categoryId
                     it[CategoryMangaTable.manga] = mangaId
@@ -52,6 +56,13 @@ object CategoryManga {
      * list of mangas that belong to a category
      */
     fun getCategoryMangaList(categoryId: Int): List<MangaDataClass> {
+        if (categoryId == DEFAULT_CATEGORY_ID)
+            return transaction {
+                MangaTable.select { (MangaTable.inLibrary eq true) and (MangaTable.defaultCategory eq true) }.map {
+                    MangaTable.toDataClass(it)
+                }
+            }
+
         return transaction {
             CategoryMangaTable.innerJoin(MangaTable).select { CategoryMangaTable.category eq categoryId }.map {
                 MangaTable.toDataClass(it)

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Chapter.kt

@@ -9,6 +9,7 @@ package suwayomi.tachidesk.manga.impl
 
 import eu.kanade.tachiyomi.source.model.SChapter
 import eu.kanade.tachiyomi.source.model.SManga
+import eu.kanade.tachiyomi.util.chapter.ChapterRecognition
 import org.jetbrains.exposed.dao.id.EntityID
 import org.jetbrains.exposed.sql.SortOrder.DESC
 import org.jetbrains.exposed.sql.and
@@ -18,14 +19,18 @@ import org.jetbrains.exposed.sql.select
 import org.jetbrains.exposed.sql.transactions.transaction
 import org.jetbrains.exposed.sql.update
 import suwayomi.tachidesk.manga.impl.Manga.getManga
+import suwayomi.tachidesk.manga.impl.Page.getPageName
 import suwayomi.tachidesk.manga.impl.util.GetHttpSource.getHttpSource
+import suwayomi.tachidesk.manga.impl.util.getChapterDir
 import suwayomi.tachidesk.manga.impl.util.lang.awaitSingle
+import suwayomi.tachidesk.manga.impl.util.storage.ImageResponse
 import suwayomi.tachidesk.manga.model.dataclass.ChapterDataClass
 import suwayomi.tachidesk.manga.model.table.ChapterMetaTable
 import suwayomi.tachidesk.manga.model.table.ChapterTable
 import suwayomi.tachidesk.manga.model.table.MangaTable
 import suwayomi.tachidesk.manga.model.table.PageTable
 import suwayomi.tachidesk.manga.model.table.toDataClass
+import java.io.File
 import java.time.Instant
 
 object Chapter {
@@ -46,14 +51,21 @@ object Chapter {
     }
 
     private suspend fun getSourceChapters(mangaId: Int): List<ChapterDataClass> {
-        val mangaDetails = getManga(mangaId)
+        val manga = getManga(mangaId)
-        val source = getHttpSource(mangaDetails.sourceId.toLong())
+        val source = getHttpSource(manga.sourceId.toLong())
-        val chapterList = source.fetchChapterList(
+
-            SManga.create().apply {
+        val sManga = SManga.create().apply {
-                title = mangaDetails.title
+            title = manga.title
-                url = mangaDetails.url
+            url = manga.url
-            }
+        }
-        ).awaitSingle()
+
+        val chapterList = source.fetchChapterList(sManga).awaitSingle()
+
+        // Recognize number for new chapters.
+        chapterList.forEach {
+            source.prepareNewChapter(it, sManga)
+            ChapterRecognition.parseChapterNumber(it, sManga)
+        }
 
         val chapterCount = chapterList.count()
 
@@ -69,7 +81,7 @@ object Chapter {
                         it[scanlator] = fetchedChapter.scanlator
 
                         it[chapterIndex] = index + 1
-                        it[manga] = mangaId
+                        it[ChapterTable.manga] = mangaId
                     }
                 } else {
                     ChapterTable.update({ ChapterTable.url eq fetchedChapter.url }) {
@@ -79,7 +91,7 @@ object Chapter {
                         it[scanlator] = fetchedChapter.scanlator
 
                         it[chapterIndex] = index + 1
-                        it[manga] = mangaId
+                        it[ChapterTable.manga] = mangaId
                     }
                 }
             }
@@ -143,7 +155,15 @@ object Chapter {
             }.first()
         }
 
-        return if (!chapterEntry[ChapterTable.isDownloaded]) {
+        val isReallyDownloaded =
+            chapterEntry[ChapterTable.isDownloaded] && firstPageExists(mangaId, chapterEntry[ChapterTable.id].value)
+        return if (!isReallyDownloaded) {
+            transaction {
+                ChapterTable.update({ (ChapterTable.chapterIndex eq chapterIndex) and (ChapterTable.manga eq mangaId) }) {
+                    it[isDownloaded] = false
+                }
+            }
+
             val mangaEntry = transaction { MangaTable.select { MangaTable.id eq mangaId }.first() }
             val source = getHttpSource(mangaEntry[MangaTable.sourceReference])
 
@@ -210,6 +230,15 @@ object Chapter {
         }
     }
 
+    private fun firstPageExists(mangaId: Int, chapterId: Int): Boolean {
+        val chapterDir = getChapterDir(mangaId, chapterId)
+
+        return ImageResponse.findFileNameStartingWith(
+            chapterDir,
+            getPageName(0, chapterDir)
+        ) != null
+    }
+
     fun modifyChapter(
         mangaId: Int,
         chapterIndex: Int,
@@ -251,16 +280,16 @@ object Chapter {
 
     fun modifyChapterMeta(mangaId: Int, chapterIndex: Int, key: String, value: String) {
         transaction {
-            val chapter =
+            val chapterId =
                 ChapterTable.select { (ChapterTable.manga eq mangaId) and (ChapterTable.chapterIndex eq chapterIndex) }
-                    .first()[ChapterTable.id]
+                    .first()[ChapterTable.id].value
             val meta =
-                transaction { ChapterMetaTable.select { (ChapterMetaTable.ref eq chapter) and (ChapterMetaTable.key eq key) } }.firstOrNull()
+                transaction { ChapterMetaTable.select { (ChapterMetaTable.ref eq chapterId) and (ChapterMetaTable.key eq key) } }.firstOrNull()
             if (meta == null) {
                 ChapterMetaTable.insert {
                     it[ChapterMetaTable.key] = key
                     it[ChapterMetaTable.value] = value
-                    it[ChapterMetaTable.ref] = chapter
+                    it[ChapterMetaTable.ref] = chapterId
                 }
             } else {
                 ChapterMetaTable.update {
@@ -269,4 +298,20 @@ object Chapter {
             }
         }
     }
+
+    fun deleteChapter(mangaId: Int, chapterIndex: Int) {
+        transaction {
+            val chapterId =
+                ChapterTable.select { (ChapterTable.manga eq mangaId) and (ChapterTable.chapterIndex eq chapterIndex) }
+                    .first()[ChapterTable.id].value
+
+            val chapterDir = getChapterDir(mangaId, chapterId)
+
+            File(chapterDir).deleteRecursively()
+
+            ChapterTable.update({ (ChapterTable.manga eq mangaId) and (ChapterTable.chapterIndex eq chapterIndex) }) {
+                it[isDownloaded] = false
+            }
+        }
+    }
 }

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Library.kt

@@ -7,23 +7,17 @@ package suwayomi.tachidesk.manga.impl
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
 
-import org.jetbrains.exposed.sql.and
 import org.jetbrains.exposed.sql.deleteWhere
 import org.jetbrains.exposed.sql.insert
 import org.jetbrains.exposed.sql.select
 import org.jetbrains.exposed.sql.transactions.transaction
 import org.jetbrains.exposed.sql.update
 import suwayomi.tachidesk.manga.impl.Manga.getManga
-import suwayomi.tachidesk.manga.model.dataclass.MangaDataClass
 import suwayomi.tachidesk.manga.model.table.CategoryMangaTable
 import suwayomi.tachidesk.manga.model.table.CategoryTable
 import suwayomi.tachidesk.manga.model.table.MangaTable
-import suwayomi.tachidesk.manga.model.table.toDataClass
 
 object Library {
-    // TODO: `Category.isLanding` is to handle the default categories a new library manga gets,
-// ..implement that shit at some time...
-// ..also Consider to rename it to `isDefault`
     suspend fun addMangaToLibrary(mangaId: Int) {
         val manga = getManga(mangaId)
         if (!manga.inLibrary) {
@@ -57,12 +51,4 @@ object Library {
             }
         }
     }
-
-    fun getLibraryMangas(): List<MangaDataClass> {
-        return transaction {
-            MangaTable.select { (MangaTable.inLibrary eq true) and (MangaTable.defaultCategory eq true) }.map {
-                MangaTable.toDataClass(it)
-            }
-        }
-    }
 }

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Manga.kt

@@ -9,7 +9,6 @@ package suwayomi.tachidesk.manga.impl
 
 import eu.kanade.tachiyomi.network.GET
 import eu.kanade.tachiyomi.source.model.SManga
-import org.jetbrains.exposed.dao.id.EntityID
 import org.jetbrains.exposed.sql.and
 import org.jetbrains.exposed.sql.insert
 import org.jetbrains.exposed.sql.select
@@ -23,9 +22,11 @@ import suwayomi.tachidesk.manga.impl.Source.getSource
 import suwayomi.tachidesk.manga.impl.util.GetHttpSource.getHttpSource
 import suwayomi.tachidesk.manga.impl.util.lang.awaitSingle
 import suwayomi.tachidesk.manga.impl.util.network.await
-import suwayomi.tachidesk.manga.impl.util.storage.CachedImageResponse.clearCachedImage
+import suwayomi.tachidesk.manga.impl.util.storage.ImageResponse.clearCachedImage
-import suwayomi.tachidesk.manga.impl.util.storage.CachedImageResponse.getCachedImageResponse
+import suwayomi.tachidesk.manga.impl.util.storage.ImageResponse.getImageResponse
+import suwayomi.tachidesk.manga.impl.util.updateMangaDownloadDir
 import suwayomi.tachidesk.manga.model.dataclass.MangaDataClass
+import suwayomi.tachidesk.manga.model.dataclass.toGenreList
 import suwayomi.tachidesk.manga.model.table.MangaMetaTable
 import suwayomi.tachidesk.manga.model.table.MangaStatus
 import suwayomi.tachidesk.manga.model.table.MangaTable
@@ -57,25 +58,31 @@ object Manga {
                 mangaEntry[MangaTable.artist],
                 mangaEntry[MangaTable.author],
                 mangaEntry[MangaTable.description],
-                mangaEntry[MangaTable.genre],
+                mangaEntry[MangaTable.genre].toGenreList(),
                 MangaStatus.valueOf(mangaEntry[MangaTable.status]).name,
                 mangaEntry[MangaTable.inLibrary],
                 getSource(mangaEntry[MangaTable.sourceReference]),
-                getMangaMetaMap(mangaEntry[MangaTable.id]),
+                getMangaMetaMap(mangaId),
+                mangaEntry[MangaTable.realUrl],
                 false
             )
         } else { // initialize manga
             val source = getHttpSource(mangaEntry[MangaTable.sourceReference])
-            val fetchedManga = source.fetchMangaDetails(
+            val sManga = SManga.create().apply {
-                SManga.create().apply {
+                url = mangaEntry[MangaTable.url]
-                    url = mangaEntry[MangaTable.url]
+                title = mangaEntry[MangaTable.title]
-                    title = mangaEntry[MangaTable.title]
+            }
-                }
+            val fetchedManga = source.fetchMangaDetails(sManga).awaitSingle()
-            ).awaitSingle()
 
             transaction {
                 MangaTable.update({ MangaTable.id eq mangaId }) {
 
+                    if (fetchedManga.title != mangaEntry[MangaTable.title]) {
+                        val canUpdateTitle = updateMangaDownloadDir(mangaId, fetchedManga.title)
+
+                        if (canUpdateTitle)
+                            it[MangaTable.title] = fetchedManga.title
+                    }
                     it[MangaTable.initialized] = true
 
                     it[MangaTable.artist] = fetchedManga.artist
@@ -85,6 +92,12 @@ object Manga {
                     it[MangaTable.status] = fetchedManga.status
                     if (fetchedManga.thumbnail_url != null && fetchedManga.thumbnail_url.orEmpty().isNotEmpty())
                         it[MangaTable.thumbnail_url] = fetchedManga.thumbnail_url
+
+                    it[MangaTable.realUrl] = try {
+                        source.mangaDetailsRequest(sManga).url.toString()
+                    } catch (e: Exception) {
+                        null
+                    }
                 }
             }
 
@@ -105,17 +118,18 @@ object Manga {
                 fetchedManga.artist,
                 fetchedManga.author,
                 fetchedManga.description,
-                fetchedManga.genre,
+                fetchedManga.genre.toGenreList(),
                 MangaStatus.valueOf(fetchedManga.status).name,
                 mangaEntry[MangaTable.inLibrary],
                 getSource(mangaEntry[MangaTable.sourceReference]),
-                getMangaMetaMap(mangaEntry[MangaTable.id]),
+                getMangaMetaMap(mangaId),
+                mangaEntry[MangaTable.realUrl],
                 true
             )
         }
     }
 
-    fun getMangaMetaMap(manga: EntityID<Int>): Map<String, String> {
+    fun getMangaMetaMap(manga: Int): Map<String, String> {
         return transaction {
             MangaMetaTable.select { MangaMetaTable.ref eq manga }
                 .associate { it[MangaMetaTable.key] to it[MangaMetaTable.value] }
@@ -126,7 +140,8 @@ object Manga {
         transaction {
             val manga = MangaMetaTable.select { (MangaTable.id eq mangaId) }
                 .first()[MangaTable.id]
-            val meta = transaction { MangaMetaTable.select { (MangaMetaTable.ref eq manga) and (MangaMetaTable.key eq key) } }.firstOrNull()
+            val meta =
+                transaction { MangaMetaTable.select { (MangaMetaTable.ref eq manga) and (MangaMetaTable.key eq key) } }.firstOrNull()
             if (meta == null) {
                 MangaMetaTable.insert {
                     it[MangaMetaTable.key] = key
@@ -142,19 +157,25 @@ object Manga {
     }
 
     private val applicationDirs by DI.global.instance<ApplicationDirs>()
-    suspend fun getMangaThumbnail(mangaId: Int): Pair<InputStream, String> {
+    suspend fun getMangaThumbnail(mangaId: Int, useCache: Boolean): Pair<InputStream, String> {
         val saveDir = applicationDirs.mangaThumbnailsRoot
         val fileName = mangaId.toString()
 
-        return getCachedImageResponse(saveDir, fileName) {
+        return getImageResponse(saveDir, fileName, useCache) {
-            getManga(mangaId) // make sure is initialized
-
             val mangaEntry = transaction { MangaTable.select { MangaTable.id eq mangaId }.first() }
 
             val sourceId = mangaEntry[MangaTable.sourceReference]
             val source = getHttpSource(sourceId)
 
-            val thumbnailUrl = mangaEntry[MangaTable.thumbnail_url]!!
+            val thumbnailUrl = mangaEntry[MangaTable.thumbnail_url]
+                ?: if (!mangaEntry[MangaTable.initialized]) {
+                    // initialize then try again
+                    getManga(mangaId)
+                    transaction { MangaTable.select { MangaTable.id eq mangaId }.first() }[MangaTable.thumbnail_url]!!
+                } else {
+                    // source provides no thumbnail url for this manga
+                    throw NullPointerException()
+                }
 
             source.client.newCall(
                 GET(thumbnailUrl, source.headers)

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/MangaList.kt

@@ -8,6 +8,7 @@ package suwayomi.tachidesk.manga.impl
  * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
 
 import eu.kanade.tachiyomi.source.model.MangasPage
+import org.jetbrains.exposed.sql.and
 import org.jetbrains.exposed.sql.insertAndGetId
 import org.jetbrains.exposed.sql.select
 import org.jetbrains.exposed.sql.transactions.transaction
@@ -16,6 +17,7 @@ import suwayomi.tachidesk.manga.impl.util.GetHttpSource.getHttpSource
 import suwayomi.tachidesk.manga.impl.util.lang.awaitSingle
 import suwayomi.tachidesk.manga.model.dataclass.MangaDataClass
 import suwayomi.tachidesk.manga.model.dataclass.PagedMangaListDataClass
+import suwayomi.tachidesk.manga.model.dataclass.toGenreList
 import suwayomi.tachidesk.manga.model.table.MangaStatus
 import suwayomi.tachidesk.manga.model.table.MangaTable
 
@@ -41,7 +43,9 @@ object MangaList {
         val mangasPage = this
         val mangaList = transaction {
             return@transaction mangasPage.mangas.map { manga ->
-                val mangaEntry = MangaTable.select { MangaTable.url eq manga.url }.firstOrNull()
+                var mangaEntry = MangaTable.select {
+                    (MangaTable.url eq manga.url) and (MangaTable.sourceReference eq sourceId)
+                }.firstOrNull()
                 if (mangaEntry == null) { // create manga entry
                     val mangaId = MangaTable.insertAndGetId {
                         it[url] = manga.url
@@ -57,6 +61,10 @@ object MangaList {
                         it[sourceReference] = sourceId
                     }.value
 
+                    mangaEntry = MangaTable.select {
+                        (MangaTable.url eq manga.url) and (MangaTable.sourceReference eq sourceId)
+                    }.first()
+
                     MangaDataClass(
                         mangaId,
                         sourceId.toString(),
@@ -70,8 +78,12 @@ object MangaList {
                         manga.artist,
                         manga.author,
                         manga.description,
-                        manga.genre,
+                        manga.genre.toGenreList(),
-                        MangaStatus.valueOf(manga.status).name
+                        MangaStatus.valueOf(manga.status).name,
+                        false, // It's a new manga entry
+                        meta = getMangaMetaMap(mangaId),
+                        realUrl = mangaEntry[MangaTable.realUrl],
+                        freshData = true
                     )
                 } else {
                     val mangaId = mangaEntry[MangaTable.id].value
@@ -88,10 +100,12 @@ object MangaList {
                         mangaEntry[MangaTable.artist],
                         mangaEntry[MangaTable.author],
                         mangaEntry[MangaTable.description],
-                        mangaEntry[MangaTable.genre],
+                        mangaEntry[MangaTable.genre].toGenreList(),
                         MangaStatus.valueOf(mangaEntry[MangaTable.status]).name,
                         mangaEntry[MangaTable.inLibrary],
-                        meta = getMangaMetaMap(mangaEntry[MangaTable.id])
+                        meta = getMangaMetaMap(mangaId),
+                        realUrl = mangaEntry[MangaTable.realUrl],
+                        freshData = false
                     )
                 }
             }

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Page.kt

@@ -7,6 +7,7 @@ package suwayomi.tachidesk.manga.impl
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
 
+import eu.kanade.tachiyomi.source.local.LocalSource
 import eu.kanade.tachiyomi.source.model.Page
 import eu.kanade.tachiyomi.source.online.HttpSource
 import org.jetbrains.exposed.sql.and
@@ -17,9 +18,10 @@ import org.kodein.di.DI
 import org.kodein.di.conf.global
 import org.kodein.di.instance
 import suwayomi.tachidesk.manga.impl.util.GetHttpSource.getHttpSource
+import suwayomi.tachidesk.manga.impl.util.getChapterDir
 import suwayomi.tachidesk.manga.impl.util.lang.awaitSingle
-import suwayomi.tachidesk.manga.impl.util.storage.CachedImageResponse.getCachedImageResponse
+import suwayomi.tachidesk.manga.impl.util.storage.ImageResponse
-import suwayomi.tachidesk.manga.impl.util.storage.SafePath
+import suwayomi.tachidesk.manga.impl.util.storage.ImageResponse.getImageResponse
 import suwayomi.tachidesk.manga.model.table.ChapterTable
 import suwayomi.tachidesk.manga.model.table.MangaTable
 import suwayomi.tachidesk.manga.model.table.PageTable
@@ -39,7 +41,7 @@ object Page {
         return page.imageUrl!!
     }
 
-    suspend fun getPageImage(mangaId: Int, chapterIndex: Int, index: Int): Pair<InputStream, String> {
+    suspend fun getPageImage(mangaId: Int, chapterIndex: Int, index: Int, useCache: Boolean = true): Pair<InputStream, String> {
         val mangaEntry = transaction { MangaTable.select { MangaTable.id eq mangaId }.first() }
         val source = getHttpSource(mangaEntry[MangaTable.sourceReference])
         val chapterEntry = transaction {
@@ -49,16 +51,31 @@ object Page {
         }
         val chapterId = chapterEntry[ChapterTable.id].value
 
-        val pageEntry = transaction { PageTable.select { (PageTable.chapter eq chapterId) and (PageTable.index eq index) }.first() }
+        val pageEntry =
+            transaction { PageTable.select { (PageTable.chapter eq chapterId) and (PageTable.index eq index) }.first() }
 
-        val tachiPage = Page(
+        val tachiyomiPage = Page(
             pageEntry[PageTable.index],
             pageEntry[PageTable.url],
             pageEntry[PageTable.imageUrl]
         )
 
+        // we treat Local source differently
+        if (mangaEntry[MangaTable.sourceReference] == LocalSource.ID) {
+            // is of archive format
+            if (LocalSource.pageCache.containsKey(chapterEntry[ChapterTable.url])) {
+                val pageStream = LocalSource.pageCache[chapterEntry[ChapterTable.url]]!![index]()
+                return pageStream to "image/jpeg"
+            }
+
+            // is of directory format
+            return ImageResponse.getNoCacheImageResponse {
+                source.fetchImage(tachiyomiPage).awaitSingle()
+            }
+        }
+
         if (pageEntry[PageTable.imageUrl] == null) {
-            val trueImageUrl = getTrueImageUrl(tachiPage, source)
+            val trueImageUrl = getTrueImageUrl(tachiyomiPage, source)
             transaction {
                 PageTable.update({ (PageTable.chapter eq chapterId) and (PageTable.index eq index) }) {
                     it[imageUrl] = trueImageUrl
@@ -66,30 +83,28 @@ object Page {
             }
         }
 
-        val saveDir = getChapterDir(mangaId, chapterId)
+        val chapterDir = getChapterDir(mangaId, chapterId)
-        File(saveDir).mkdirs()
+        File(chapterDir).mkdirs()
-        val fileName = String.format("%03d", index) // e.g. 001.jpeg
+        val fileName = getPageName(index, chapterDir) // e.g. 001
 
-        return getCachedImageResponse(saveDir, fileName) {
+        return getImageResponse(chapterDir, fileName, useCache) {
-            source.fetchImage(tachiPage).awaitSingle()
+            source.fetchImage(tachiyomiPage).awaitSingle()
         }
     }
 
-    private val applicationDirs by DI.global.instance<ApplicationDirs>()
+    // TODO(v0.6.0) : zero based pages are deprecated
-    private fun getChapterDir(mangaId: Int, chapterId: Int): String {
+    fun getPageName(index: Int, chapterDir: String): String {
-        val mangaEntry = transaction { MangaTable.select { MangaTable.id eq mangaId }.first() }
+        val zeroBasedPageExists = ImageResponse.findFileNameStartingWith(
-        val source = getHttpSource(mangaEntry[MangaTable.sourceReference])
+            chapterDir,
-        val chapterEntry = transaction { ChapterTable.select { ChapterTable.id eq chapterId }.first() }
+            formatPageName(0)
+        ) != null
 
-        val sourceDir = source.toString()
+        if (zeroBasedPageExists) return formatPageName(index)
-        val mangaDir = SafePath.buildValidFilename(mangaEntry[MangaTable.title])
-        val chapterDir = SafePath.buildValidFilename(
-            when {
-                chapterEntry[ChapterTable.scanlator] != null -> "${chapterEntry[ChapterTable.scanlator]}_${chapterEntry[ChapterTable.name]}"
-                else -> chapterEntry[ChapterTable.name]
-            }
-        )
 
-        return "${applicationDirs.mangaRoot}/$sourceDir/$mangaDir/$chapterDir"
+        return formatPageName(index + 1)
     }
+
+    private fun formatPageName(index: Int) = String.format("%03d", index)
+
+    private val applicationDirs by DI.global.instance<ApplicationDirs>()
 }

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Search.kt

@@ -7,6 +7,8 @@ package suwayomi.tachidesk.manga.impl
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
 
+import eu.kanade.tachiyomi.source.model.Filter
+import eu.kanade.tachiyomi.source.model.FilterList
 import suwayomi.tachidesk.manga.impl.MangaList.processEntries
 import suwayomi.tachidesk.manga.impl.util.GetHttpSource.getHttpSource
 import suwayomi.tachidesk.manga.impl.util.lang.awaitSingle
@@ -15,28 +17,53 @@ import suwayomi.tachidesk.manga.model.dataclass.PagedMangaListDataClass
 object Search {
     suspend fun sourceSearch(sourceId: Long, searchTerm: String, pageNum: Int): PagedMangaListDataClass {
         val source = getHttpSource(sourceId)
-        val searchManga = source.fetchSearchManga(pageNum, searchTerm, source.getFilterList()).awaitSingle()
+        val searchManga = source.fetchSearchManga(pageNum, searchTerm, getFilterListOf(sourceId)).awaitSingle()
         return searchManga.processEntries(sourceId)
     }
 
-    // TODO
+    private val filterListCache = mutableMapOf<Long, FilterList>()
-    @Suppress("UNUSED_PARAMETER", "UNUSED_VARIABLE")
+
-    fun sourceFilters(sourceId: Long) {
+    private fun getFilterListOf(sourceId: Long, reset: Boolean = false): FilterList {
-        val source = getHttpSource(sourceId)
+        if (reset || !filterListCache.containsKey(sourceId)) {
-        // source.getFilterList().toItems()
+            filterListCache[sourceId] = getHttpSource(sourceId).getFilterList()
+        }
+        return filterListCache[sourceId]!!
     }
 
+    fun getInitialFilterList(sourceId: Long, reset: Boolean): List<FilterObject> {
+        return getFilterListOf(sourceId, reset).list.map {
+            FilterObject(
+                when (it) {
+                    is Filter.Header -> "Header"
+                    is Filter.Separator -> "Separator"
+                    is Filter.CheckBox -> "CheckBox"
+                    is Filter.TriState -> "TriState"
+                    is Filter.Text -> "Text"
+                    is Filter.Select<*> -> "Select"
+                    is Filter.Group<*> -> "Group"
+                    is Filter.Sort -> "Sort"
+                },
+//                when (it) {
+//                    is Filter.Select<*> -> it.getValuesType()
+//                    else -> null
+//                },
+                it
+            )
+        }
+    }
+
+//    private fun Filter.Select<*>.getValuesType(): String = values::class.java.componentType!!.simpleName
+
+    data class FilterObject(
+        val type: String,
+        val filter: Filter<*>
+    )
+
     @Suppress("UNUSED_PARAMETER")
     fun sourceGlobalSearch(searchTerm: String) {
         // TODO
     }
 
-    @Suppress("unused")
-    data class FilterWrapper(
-        val type: String,
-        val filter: Any
-    )
-
     /**
      * Note: Exhentai had a filter serializer (now in SY) that we might be able to steal
      */

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Source.kt

@@ -12,6 +12,7 @@ import android.content.Context
 import androidx.preference.PreferenceScreen
 import eu.kanade.tachiyomi.source.ConfigurableSource
 import eu.kanade.tachiyomi.source.getPreferenceKey
+import eu.kanade.tachiyomi.source.local.LocalSource
 import mu.KotlinLogging
 import org.jetbrains.exposed.sql.select
 import org.jetbrains.exposed.sql.selectAll
@@ -35,29 +36,49 @@ object Source {
     fun getSourceList(): List<SourceDataClass> {
         return transaction {
             SourceTable.selectAll().map {
+                val httpSource = getHttpSource(it[SourceTable.id].value)
+                val sourceExtension = ExtensionTable.select { ExtensionTable.id eq it[SourceTable.extension] }.first()
+
                 SourceDataClass(
                     it[SourceTable.id].value.toString(),
                     it[SourceTable.name],
                     it[SourceTable.lang],
-                    getExtensionIconUrl(ExtensionTable.select { ExtensionTable.id eq it[SourceTable.extension] }.first()[ExtensionTable.apkName]),
+                    getExtensionIconUrl(sourceExtension[ExtensionTable.apkName]),
-                    getHttpSource(it[SourceTable.id].value).supportsLatest,
+                    httpSource.supportsLatest,
-                    getHttpSource(it[SourceTable.id].value) is ConfigurableSource
+                    httpSource is ConfigurableSource,
+                    it[SourceTable.isNsfw],
+                    httpSource.toString(),
                 )
             }
         }
     }
 
-    fun getSource(sourceId: Long): SourceDataClass {
+    fun getSource(sourceId: Long): SourceDataClass { // all the data extracted fresh form the source instance
         return transaction {
+            if (sourceId == LocalSource.ID) {
+                // initialize local source
+                getHttpSource(sourceId)
+            }
+
             val source = SourceTable.select { SourceTable.id eq sourceId }.firstOrNull()
+            val httpSource = source?.let { getHttpSource(sourceId) }
+            val extension = source?.let {
+                ExtensionTable.select { ExtensionTable.id eq source[SourceTable.extension] }.first()
+            }
 
             SourceDataClass(
                 sourceId.toString(),
                 source?.get(SourceTable.name),
                 source?.get(SourceTable.lang),
-                source?.let { getExtensionIconUrl(ExtensionTable.select { ExtensionTable.id eq source[SourceTable.extension] }.first()[ExtensionTable.apkName]) },
+                source?.let {
-                source?.let { getHttpSource(sourceId).supportsLatest },
+                    getExtensionIconUrl(
-                source?.let { getHttpSource(sourceId) is ConfigurableSource },
+                        extension!![ExtensionTable.apkName]
+                    )
+                },
+                httpSource?.supportsLatest,
+                httpSource?.let { it is ConfigurableSource },
+                source?.get(SourceTable.isNsfw),
+                httpSource?.toString()
             )
         }
     }
@@ -65,7 +86,7 @@ object Source {
     private val context by DI.global.instance<CustomContext>()
 
     /**
-     * Clients should support these types for extensions to work properly (in order of importance)
+     * (2021-08) Clients should support these types for extensions to work properly
      * - EditTextPreference
      * - SwitchPreferenceCompat
      * - ListPreference
@@ -85,7 +106,8 @@ object Source {
         val source = getHttpSource(sourceId)
 
         if (source is ConfigurableSource) {
-            val sourceShardPreferences = Injekt.get<Application>().getSharedPreferences(source.getPreferenceKey(), Context.MODE_PRIVATE)
+            val sourceShardPreferences =
+                Injekt.get<Application>().getSharedPreferences(source.getPreferenceKey(), Context.MODE_PRIVATE)
 
             val screen = PreferenceScreen(context)
             screen.sharedPreferences = sourceShardPreferences

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/AbstractBackupValidator.kt

@@ -0,0 +1,12 @@
+package suwayomi.tachidesk.manga.impl.backup
+
+/*
+ * Copyright (C) Contributors to the Suwayomi project
+ * 
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
+
+abstract class AbstractBackupValidator {
+    data class ValidationResult(val missingSources: List<String>, val missingTrackers: List<String>)
+}

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/legacy/LegacyBackupBase.kt

@@ -1,45 +0,0 @@
-package suwayomi.tachidesk.manga.impl.backup.legacy
-
-/*
- * Copyright (C) Contributors to the Suwayomi project
- *
- * This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
-
-import com.github.salomonbrys.kotson.registerTypeAdapter
-import com.github.salomonbrys.kotson.registerTypeHierarchyAdapter
-import com.google.gson.Gson
-import com.google.gson.GsonBuilder
-import suwayomi.tachidesk.manga.impl.backup.legacy.models.DHistory
-import suwayomi.tachidesk.manga.impl.backup.legacy.serializer.CategoryTypeAdapter
-import suwayomi.tachidesk.manga.impl.backup.legacy.serializer.ChapterTypeAdapter
-import suwayomi.tachidesk.manga.impl.backup.legacy.serializer.HistoryTypeAdapter
-import suwayomi.tachidesk.manga.impl.backup.legacy.serializer.MangaTypeAdapter
-import suwayomi.tachidesk.manga.impl.backup.legacy.serializer.TrackTypeAdapter
-import suwayomi.tachidesk.manga.impl.backup.models.CategoryImpl
-import suwayomi.tachidesk.manga.impl.backup.models.ChapterImpl
-import suwayomi.tachidesk.manga.impl.backup.models.MangaImpl
-import suwayomi.tachidesk.manga.impl.backup.models.TrackImpl
-import java.util.Date
-
-open class LegacyBackupBase {
-    protected val parser: Gson = when (version) {
-        2 -> GsonBuilder()
-            .registerTypeAdapter<MangaImpl>(MangaTypeAdapter.build())
-            .registerTypeHierarchyAdapter<ChapterImpl>(ChapterTypeAdapter.build())
-            .registerTypeAdapter<CategoryImpl>(CategoryTypeAdapter.build())
-            .registerTypeAdapter<DHistory>(HistoryTypeAdapter.build())
-            .registerTypeHierarchyAdapter<TrackImpl>(TrackTypeAdapter.build())
-            .create()
-        else -> throw Exception("Unknown backup version")
-    }
-
-    protected var sourceMapping: Map<Long, String> = emptyMap()
-
-    protected val errors = mutableListOf<Pair<Date, String>>()
-
-    companion object {
-        internal const val version = 2
-    }
-}

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/legacy/LegacyBackupExport.kt

@@ -1,154 +0,0 @@
-package suwayomi.tachidesk.manga.impl.backup.legacy
-
-/*
- * Copyright (C) Contributors to the Suwayomi project
- *
- * This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
-
-import com.github.salomonbrys.kotson.set
-import com.google.gson.JsonArray
-import com.google.gson.JsonElement
-import com.google.gson.JsonObject
-import eu.kanade.tachiyomi.source.LocalSource
-import org.jetbrains.exposed.sql.select
-import org.jetbrains.exposed.sql.transactions.transaction
-import suwayomi.tachidesk.manga.impl.Category.getCategoryList
-import suwayomi.tachidesk.manga.impl.CategoryManga.getMangaCategories
-import suwayomi.tachidesk.manga.impl.backup.BackupFlags
-import suwayomi.tachidesk.manga.impl.backup.legacy.models.Backup
-import suwayomi.tachidesk.manga.impl.backup.legacy.models.Backup.CURRENT_VERSION
-import suwayomi.tachidesk.manga.impl.backup.models.CategoryImpl
-import suwayomi.tachidesk.manga.impl.backup.models.ChapterImpl
-import suwayomi.tachidesk.manga.impl.backup.models.Manga
-import suwayomi.tachidesk.manga.impl.backup.models.MangaImpl
-import suwayomi.tachidesk.manga.impl.util.GetHttpSource.getHttpSource
-import suwayomi.tachidesk.manga.model.table.ChapterTable
-import suwayomi.tachidesk.manga.model.table.MangaTable
-
-object LegacyBackupExport : LegacyBackupBase() {
-
-    suspend fun createLegacyBackup(flags: BackupFlags): String? {
-        // Create root object
-        val root = JsonObject()
-
-        // Create manga array
-        val mangaEntries = JsonArray()
-
-        // Create category array
-        val categoryEntries = JsonArray()
-
-        // Create extension ID/name mapping
-        val extensionEntries = JsonArray()
-
-        // Add values to root
-        root[Backup.VERSION] = CURRENT_VERSION
-        root[Backup.MANGAS] = mangaEntries
-        root[Backup.CATEGORIES] = categoryEntries
-        root[Backup.EXTENSIONS] = extensionEntries
-
-        transaction {
-            val mangas = MangaTable.select { (MangaTable.inLibrary eq true) }
-
-            val extensions: MutableSet<String> = mutableSetOf()
-
-            // Backup library manga and its dependencies
-            mangas.map {
-                MangaImpl.fromQuery(it)
-            }.forEach { manga ->
-
-                mangaEntries.add(backupMangaObject(manga, flags))
-
-                // Maintain set of extensions/sources used (excludes local source)
-                if (manga.source != LocalSource.ID) {
-                    getHttpSource(manga.source).let {
-                        extensions.add("${it.id}:${it.name}")
-                    }
-                }
-            }
-
-            // Backup categories
-            if (flags.includeCategories) {
-                backupCategories(categoryEntries)
-            }
-
-            // Backup extension ID/name mapping
-            backupExtensionInfo(extensionEntries, extensions)
-        }
-
-        return parser.toJson(root)
-    }
-
-    private fun backupMangaObject(manga: Manga, options: BackupFlags): JsonElement {
-        // Entry for this manga
-        val entry = JsonObject()
-
-        // Backup manga fields
-        entry[Backup.MANGA] = parser.toJsonTree(manga)
-        val mangaId = manga.id!!.toInt()
-
-        // Check if user wants chapter information in backup
-        if (options.includeChapters) {
-            // Backup all the chapters
-            val chapters = ChapterTable.select { ChapterTable.manga eq mangaId }.map { ChapterImpl.fromQuery(it) }
-            if (chapters.count() > 0) {
-                val chaptersJson = parser.toJsonTree(chapters)
-                if (chaptersJson.asJsonArray.size() > 0) {
-                    entry[Backup.CHAPTERS] = chaptersJson
-                }
-            }
-        }
-
-        // Check if user wants category information in backup
-        if (options.includeCategories) {
-            // Backup categories for this manga
-            val categoriesForManga = getMangaCategories(mangaId)
-            if (categoriesForManga.isNotEmpty()) {
-                val categoriesNames = categoriesForManga.map { it.name }
-                entry[Backup.CATEGORIES] = parser.toJsonTree(categoriesNames)
-            }
-        }
-
-        // Check if user wants track information in backup
-        if (options.includeTracking) { // TODO
-//            val tracks = databaseHelper.getTracks(manga).executeAsBlocking()
-//            if (tracks.isNotEmpty()) {
-//                entry[TRACK] = parser.toJsonTree(tracks)
-//            }
-        }
-//
-//        // Check if user wants history information in backup
-        if (options.includeHistory) { // TODO
-//            val historyForManga = databaseHelper.getHistoryByMangaId(manga.id!!).executeAsBlocking()
-//            if (historyForManga.isNotEmpty()) {
-//                val historyData = historyForManga.mapNotNull { history ->
-//                    val url = databaseHelper.getChapter(history.chapter_id).executeAsBlocking()?.url
-//                    url?.let { DHistory(url, history.last_read) }
-//                }
-//                val historyJson = parser.toJsonTree(historyData)
-//                if (historyJson.asJsonArray.size() > 0) {
-//                    entry[HISTORY] = historyJson
-//                }
-//            }
-        }
-
-        return entry
-    }
-
-    private fun backupCategories(root: JsonArray) {
-        val categories = getCategoryList().map {
-            CategoryImpl().apply {
-                name = it.name
-                order = it.order
-            }
-        }
-        categories.forEach { root.add(parser.toJsonTree(it)) }
-    }
-
-    private fun backupExtensionInfo(root: JsonArray, extensions: Set<String>) {
-        extensions.sorted().forEach {
-            root.add(it)
-        }
-    }
-}

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/legacy/LegacyBackupImport.kt

@@ -1,212 +0,0 @@
-package suwayomi.tachidesk.manga.impl.backup.legacy
-
-import com.github.salomonbrys.kotson.fromJson
-import com.google.gson.JsonArray
-import com.google.gson.JsonElement
-import com.google.gson.JsonObject
-import com.google.gson.JsonParser
-import eu.kanade.tachiyomi.source.Source
-import eu.kanade.tachiyomi.source.model.SManga
-import mu.KotlinLogging
-import org.jetbrains.exposed.sql.and
-import org.jetbrains.exposed.sql.insert
-import org.jetbrains.exposed.sql.select
-import org.jetbrains.exposed.sql.transactions.transaction
-import org.jetbrains.exposed.sql.update
-import suwayomi.tachidesk.manga.impl.Category.createCategory
-import suwayomi.tachidesk.manga.impl.Category.getCategoryList
-import suwayomi.tachidesk.manga.impl.backup.legacy.LegacyBackupValidator.ValidationResult
-import suwayomi.tachidesk.manga.impl.backup.legacy.LegacyBackupValidator.validate
-import suwayomi.tachidesk.manga.impl.backup.legacy.models.Backup
-import suwayomi.tachidesk.manga.impl.backup.legacy.models.DHistory
-import suwayomi.tachidesk.manga.impl.backup.models.CategoryImpl
-import suwayomi.tachidesk.manga.impl.backup.models.Chapter
-import suwayomi.tachidesk.manga.impl.backup.models.ChapterImpl
-import suwayomi.tachidesk.manga.impl.backup.models.Manga
-import suwayomi.tachidesk.manga.impl.backup.models.MangaImpl
-import suwayomi.tachidesk.manga.impl.backup.models.Track
-import suwayomi.tachidesk.manga.impl.backup.models.TrackImpl
-import suwayomi.tachidesk.manga.impl.util.GetHttpSource.getHttpSource
-import suwayomi.tachidesk.manga.impl.util.lang.awaitSingle
-import suwayomi.tachidesk.manga.model.table.MangaTable
-import java.io.InputStream
-import java.util.Date
-
-/*
- * Copyright (C) Contributors to the Suwayomi project
- * 
- * This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
-
-private val logger = KotlinLogging.logger {}
-
-object LegacyBackupImport : LegacyBackupBase() {
-    suspend fun restoreLegacyBackup(sourceStream: InputStream): ValidationResult {
-        val reader = sourceStream.bufferedReader()
-        val json = JsonParser.parseReader(reader).asJsonObject
-
-        val validationResult = validate(json)
-
-        val mangasJson = json.get(Backup.MANGAS).asJsonArray
-
-        // Restore categories
-        json.get(Backup.CATEGORIES)?.let { restoreCategories(it) }
-
-        // Store source mapping for error messages
-        sourceMapping = LegacyBackupValidator.getSourceMapping(json)
-
-        // Restore individual manga
-        mangasJson.forEach {
-            restoreManga(it.asJsonObject)
-        }
-
-        logger.info {
-            """
-                Restore Errors:
-                ${ errors.joinToString("\n") { "${it.first} - ${it.second}" } }
-                Restore Summary:
-                - Missing Sources:
-                ${validationResult.missingSources.joinToString("\n")}
-                - Missing Trackers:
-                ${validationResult.missingTrackers.joinToString("\n")}
-            """.trimIndent()
-        }
-
-        return validationResult
-    }
-
-    private fun restoreCategories(jsonCategories: JsonElement) {
-        val backupCategories = parser.fromJson<List<CategoryImpl>>(jsonCategories)
-        val dbCategories = getCategoryList()
-
-        // Iterate over them and create missing categories
-        backupCategories.forEach { category ->
-            if (dbCategories.none { it.name == category.name }) {
-                createCategory(category.name)
-            }
-        }
-    }
-
-    private suspend fun restoreManga(mangaJson: JsonObject) {
-        val manga = parser.fromJson<MangaImpl>(
-            mangaJson.get(
-                Backup.MANGA
-            )
-        )
-        val chapters = parser.fromJson<List<ChapterImpl>>(
-            mangaJson.get(Backup.CHAPTERS)
-                ?: JsonArray()
-        )
-        val categories = parser.fromJson<List<String>>(
-            mangaJson.get(Backup.CATEGORIES)
-                ?: JsonArray()
-        )
-        val history = parser.fromJson<List<DHistory>>(
-            mangaJson.get(Backup.HISTORY)
-                ?: JsonArray()
-        )
-        val tracks = parser.fromJson<List<TrackImpl>>(
-            mangaJson.get(Backup.TRACK)
-                ?: JsonArray()
-        )
-
-        val source = try {
-            getHttpSource(manga.source)
-        } catch (e: NullPointerException) {
-            null
-        } catch (e: NoSuchElementException) {
-            null
-        }
-        val sourceName = sourceMapping[manga.source] ?: manga.source.toString()
-
-        logger.debug("Restoring Manga: ${manga.title} from $sourceName")
-
-        try {
-            if (source != null) {
-                restoreMangaData(manga, source, chapters, categories, history, tracks)
-            } else {
-                errors.add(Date() to "${manga.title} [$sourceName]: Source not found: $sourceName (${manga.source})")
-            }
-        } catch (e: Exception) {
-            errors.add(Date() to "${manga.title} [$sourceName]: ${e.message}")
-        }
-    }
-
-    /**
-     * @param manga manga data from json
-     * @param source source to get manga data from
-     * @param chapters chapters data from json
-     * @param categories categories data from json
-     * @param history history data from json
-     * @param tracks tracking data from json
-     */
-    @Suppress("UNUSED_PARAMETER")
-    private suspend fun restoreMangaData(
-        manga: Manga,
-        source: Source,
-        chapters: List<Chapter>,
-        categories: List<String>,
-        history: List<DHistory>,
-        tracks: List<Track>
-    ) {
-        val fetchedManga = fetchManga(source, manga)
-
-        updateChapters(source, fetchedManga, chapters)
-
-        // TODO
-//        backupManager.restoreCategoriesForManga(manga, categories)
-
-//        backupManager.restoreHistoryForManga(history)
-
-//        backupManager.restoreTrackForManga(manga, tracks)
-
-//        updateTracking(fetchedManga, tracks)
-    }
-
-    /**
-     * Fetches manga information
-     *
-     * @param source source of manga
-     * @param manga manga that needs updating
-     * @return Updated manga.
-     */
-    private suspend fun fetchManga(source: Source, manga: Manga): SManga {
-        // make sure we have the manga record in library
-        transaction {
-            if (MangaTable.select { (MangaTable.url eq manga.url) and (MangaTable.sourceReference eq manga.source) }.firstOrNull() == null) {
-                MangaTable.insert {
-                    it[url] = manga.url
-                    it[title] = manga.title
-
-                    it[sourceReference] = manga.source
-                }
-            }
-            MangaTable.update({ (MangaTable.url eq manga.url) and (MangaTable.sourceReference eq manga.source) }) {
-                it[MangaTable.inLibrary] = true
-            }
-        }
-
-        // update manga details
-        val fetchedManga = source.fetchMangaDetails(manga).awaitSingle()
-        transaction {
-            MangaTable.update({ (MangaTable.url eq manga.url) and (MangaTable.sourceReference eq manga.source) }) {
-
-                it[artist] = fetchedManga.artist
-                it[author] = fetchedManga.author
-                it[description] = fetchedManga.description
-                it[genre] = fetchedManga.genre
-                it[status] = fetchedManga.status
-                if (fetchedManga.thumbnail_url != null && fetchedManga.thumbnail_url.orEmpty().isNotEmpty())
-                    it[MangaTable.thumbnail_url] = fetchedManga.thumbnail_url
-            }
-        }
-
-        return fetchedManga
-    }
-
-    @Suppress("UNUSED_PARAMETER") // TODO: remove this suppress when update Chapters is written
-    private fun updateChapters(source: Source, fetchedManga: SManga, chapters: List<Chapter>) {
-        // TODO("Not yet implemented")
-    }
-}

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/legacy/LegacyBackupValidator.kt

@@ -1,71 +0,0 @@
-package suwayomi.tachidesk.manga.impl.backup.legacy
-
-/*
- * Copyright (C) Contributors to the Suwayomi project
- * 
- * This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
-
-import com.google.gson.JsonObject
-import org.jetbrains.exposed.sql.select
-import org.jetbrains.exposed.sql.transactions.transaction
-import suwayomi.tachidesk.manga.impl.backup.legacy.models.Backup
-import suwayomi.tachidesk.manga.model.table.SourceTable
-
-object LegacyBackupValidator {
-    data class ValidationResult(val missingSources: List<String>, val missingTrackers: List<String>)
-
-    /**
-     * Checks for critical backup file data.
-     *
-     * @throws Exception if version or manga cannot be found.
-     * @return List of missing sources or missing trackers.
-     */
-    fun validate(json: JsonObject): ValidationResult {
-        val version = json.get(Backup.VERSION)
-        val mangasJson = json.get(Backup.MANGAS)
-        if (version == null || mangasJson == null) {
-            throw Exception("File is missing data.")
-        }
-
-        val mangas = mangasJson.asJsonArray
-        if (mangas.size() == 0) {
-            throw Exception("Backup does not contain any manga.")
-        }
-
-        val sources = getSourceMapping(json)
-        val missingSources = transaction {
-            sources
-                .filter { SourceTable.select { SourceTable.id eq it.key }.firstOrNull() == null }
-                .map { "${it.value} (${it.key})" }
-                .sorted()
-        }
-
-//        val trackers = mangas
-//            .filter { it.asJsonObject.has("track") }
-//            .flatMap { it.asJsonObject["track"].asJsonArray }
-//            .map { it.asJsonObject["s"].asInt }
-//            .distinct()
-
-        val missingTrackers = listOf("")
-//        val missingTrackers = trackers
-//                .mapNotNull { trackManager.getService(it) }
-//                .filter { !it.isLogged }
-//                .map { context.getString(it.nameRes()) }
-//                .sorted()
-
-        return ValidationResult(missingSources, missingTrackers)
-    }
-
-    fun getSourceMapping(json: JsonObject): Map<Long, String> {
-        val extensionsMapping = json.get(Backup.EXTENSIONS) ?: return emptyMap()
-
-        return extensionsMapping.asJsonArray
-            .map {
-                val items = it.asString.split(":")
-                items[0].toLong() to items[1]
-            }
-            .toMap()
-    }
-}

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/legacy/models/Backup.kt

@@ -1,25 +0,0 @@
-package suwayomi.tachidesk.manga.impl.backup.legacy.models
-
-import java.text.SimpleDateFormat
-import java.util.Date
-import java.util.Locale
-
-/**
- * Json values
- */
-object Backup {
-    const val CURRENT_VERSION = 2
-    const val MANGA = "manga"
-    const val MANGAS = "mangas"
-    const val TRACK = "track"
-    const val CHAPTERS = "chapters"
-    const val CATEGORIES = "categories"
-    const val EXTENSIONS = "extensions"
-    const val HISTORY = "history"
-    const val VERSION = "version"
-
-    fun getDefaultFilename(): String {
-        val date = SimpleDateFormat("yyyy-MM-dd_HH-mm", Locale.getDefault()).format(Date())
-        return "tachiyomi_$date.json"
-    }
-}

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/legacy/models/DHistory.kt

@@ -1,3 +0,0 @@
-package suwayomi.tachidesk.manga.impl.backup.legacy.models
-
-data class DHistory(val url: String, val lastRead: Long)

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/legacy/serializer/CategoryTypeAdapter.kt

@@ -1,31 +0,0 @@
-package suwayomi.tachidesk.manga.impl.backup.legacy.serializer
-
-import com.github.salomonbrys.kotson.typeAdapter
-import com.google.gson.TypeAdapter
-import suwayomi.tachidesk.manga.impl.backup.models.CategoryImpl
-
-/**
- * JSON Serializer used to write / read [CategoryImpl] to / from json
- */
-object CategoryTypeAdapter {
-
-    fun build(): TypeAdapter<CategoryImpl> {
-        return typeAdapter {
-            write {
-                beginArray()
-                value(it.name)
-                value(it.order)
-                endArray()
-            }
-
-            read {
-                beginArray()
-                val category = CategoryImpl()
-                category.name = nextString()
-                category.order = nextInt()
-                endArray()
-                category
-            }
-        }
-    }
-}

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/legacy/serializer/ChapterTypeAdapter.kt

@@ -1,59 +0,0 @@
-package suwayomi.tachidesk.manga.impl.backup.legacy.serializer
-
-import com.github.salomonbrys.kotson.typeAdapter
-import com.google.gson.TypeAdapter
-import com.google.gson.stream.JsonToken
-import suwayomi.tachidesk.manga.impl.backup.models.ChapterImpl
-
-/**
- * JSON Serializer used to write / read [ChapterImpl] to / from json
- */
-object ChapterTypeAdapter {
-
-    private const val URL = "u"
-    private const val READ = "r"
-    private const val BOOKMARK = "b"
-    private const val LAST_READ = "l"
-
-    fun build(): TypeAdapter<ChapterImpl> {
-        return typeAdapter {
-            write {
-                if (it.read || it.bookmark || it.last_page_read != 0) {
-                    beginObject()
-                    name(URL)
-                    value(it.url)
-                    if (it.read) {
-                        name(READ)
-                        value(1)
-                    }
-                    if (it.bookmark) {
-                        name(BOOKMARK)
-                        value(1)
-                    }
-                    if (it.last_page_read != 0) {
-                        name(LAST_READ)
-                        value(it.last_page_read)
-                    }
-                    endObject()
-                }
-            }
-
-            read {
-                val chapter = ChapterImpl()
-                beginObject()
-                while (hasNext()) {
-                    if (peek() == JsonToken.NAME) {
-                        when (nextName()) {
-                            URL -> chapter.url = nextString()
-                            READ -> chapter.read = nextInt() == 1
-                            BOOKMARK -> chapter.bookmark = nextInt() == 1
-                            LAST_READ -> chapter.last_page_read = nextInt()
-                        }
-                    }
-                }
-                endObject()
-                chapter
-            }
-        }
-    }
-}

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/legacy/serializer/HistoryTypeAdapter.kt

@@ -1,32 +0,0 @@
-package suwayomi.tachidesk.manga.impl.backup.legacy.serializer
-
-import com.github.salomonbrys.kotson.typeAdapter
-import com.google.gson.TypeAdapter
-import suwayomi.tachidesk.manga.impl.backup.legacy.models.DHistory
-
-/**
- * JSON Serializer used to write / read [DHistory] to / from json
- */
-object HistoryTypeAdapter {
-
-    fun build(): TypeAdapter<DHistory> {
-        return typeAdapter {
-            write {
-                if (it.lastRead != 0L) {
-                    beginArray()
-                    value(it.url)
-                    value(it.lastRead)
-                    endArray()
-                }
-            }
-
-            read {
-                beginArray()
-                val url = nextString()
-                val lastRead = nextLong()
-                endArray()
-                DHistory(url, lastRead)
-            }
-        }
-    }
-}

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/legacy/serializer/MangaTypeAdapter.kt

@@ -1,37 +0,0 @@
-package suwayomi.tachidesk.manga.impl.backup.legacy.serializer
-
-import com.github.salomonbrys.kotson.typeAdapter
-import com.google.gson.TypeAdapter
-import suwayomi.tachidesk.manga.impl.backup.models.MangaImpl
-
-/**
- * JSON Serializer used to write / read [MangaImpl] to / from json
- */
-object MangaTypeAdapter {
-
-    fun build(): TypeAdapter<MangaImpl> {
-        return typeAdapter {
-            write {
-                beginArray()
-                value(it.url)
-                value(it.title)
-                value(it.source)
-                value(it.viewer)
-                value(it.chapter_flags)
-                endArray()
-            }
-
-            read {
-                beginArray()
-                val manga = MangaImpl()
-                manga.url = nextString()
-                manga.title = nextString()
-                manga.source = nextLong()
-                manga.viewer = nextInt()
-                manga.chapter_flags = nextInt()
-                endArray()
-                manga
-            }
-        }
-    }
-}

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/legacy/serializer/TrackTypeAdapter.kt

@@ -1,59 +0,0 @@
-package suwayomi.tachidesk.manga.impl.backup.legacy.serializer
-
-import com.github.salomonbrys.kotson.typeAdapter
-import com.google.gson.TypeAdapter
-import com.google.gson.stream.JsonToken
-import suwayomi.tachidesk.manga.impl.backup.models.TrackImpl
-
-/**
- * JSON Serializer used to write / read [TrackImpl] to / from json
- */
-object TrackTypeAdapter {
-
-    private const val SYNC = "s"
-    private const val MEDIA = "r"
-    private const val LIBRARY = "ml"
-    private const val TITLE = "t"
-    private const val LAST_READ = "l"
-    private const val TRACKING_URL = "u"
-
-    fun build(): TypeAdapter<TrackImpl> {
-        return typeAdapter {
-            write {
-                beginObject()
-                name(TITLE)
-                value(it.title)
-                name(SYNC)
-                value(it.sync_id)
-                name(MEDIA)
-                value(it.media_id)
-                name(LIBRARY)
-                value(it.library_id)
-                name(LAST_READ)
-                value(it.last_chapter_read)
-                name(TRACKING_URL)
-                value(it.tracking_url)
-                endObject()
-            }
-
-            read {
-                val track = TrackImpl()
-                beginObject()
-                while (hasNext()) {
-                    if (peek() == JsonToken.NAME) {
-                        when (nextName()) {
-                            TITLE -> track.title = nextString()
-                            SYNC -> track.sync_id = nextInt()
-                            MEDIA -> track.media_id = nextInt()
-                            LIBRARY -> track.library_id = nextLong()
-                            LAST_READ -> track.last_chapter_read = nextInt()
-                            TRACKING_URL -> track.tracking_url = nextString()
-                        }
-                    }
-                }
-                endObject()
-                track
-            }
-        }
-    }
-}

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/models/Manga.kt

@@ -1,8 +1,18 @@
 package suwayomi.tachidesk.manga.impl.backup.models
 
 import eu.kanade.tachiyomi.source.model.SManga
+// import eu.kanade.tachiyomi.ui.reader.setting.OrientationType
+// import eu.kanade.tachiyomi.ui.reader.setting.ReadingModeType
 
-// import tachiyomi.source.model.MangaInfo
+// substitute for eu.kanade.tachiyomi.ui.reader.setting.OrientationType
+object OrientationType {
+    const val MASK = 0x00000038
+}
+
+// substitute for eu.kanade.tachiyomi.ui.reader.setting.ReadingModeType
+object ReadingModeType {
+    const val MASK = 0x00000007
+}
 
 interface Manga : SManga {
 
@@ -10,85 +20,100 @@ interface Manga : SManga {
 
     var source: Long
 
-    /** is in library */
     var favorite: Boolean
 
+    // last time the chapter list changed in any way
     var last_update: Long
 
+    // predicted next update time based on latest (by date) 4 chapters' deltas
+    var next_update: Long
+
     var date_added: Long
 
-    var viewer: Int
+    var viewer_flags: Int
 
     var chapter_flags: Int
 
     var cover_last_modified: Long
 
     fun setChapterOrder(order: Int) {
-        setFlags(order, SORT_MASK)
+        setChapterFlags(order, CHAPTER_SORT_MASK)
     }
 
     fun sortDescending(): Boolean {
-        return chapter_flags and SORT_MASK == SORT_DESC
+        return chapter_flags and CHAPTER_SORT_MASK == CHAPTER_SORT_DESC
     }
 
     fun getGenres(): List<String>? {
         return genre?.split(", ")?.map { it.trim() }
     }
 
-    private fun setFlags(flag: Int, mask: Int) {
+    private fun setChapterFlags(flag: Int, mask: Int) {
         chapter_flags = chapter_flags and mask.inv() or (flag and mask)
     }
 
+    private fun setViewerFlags(flag: Int, mask: Int) {
+        viewer_flags = viewer_flags and mask.inv() or (flag and mask)
+    }
+
     // Used to display the chapter's title one way or another
     var displayMode: Int
-        get() = chapter_flags and DISPLAY_MASK
+        get() = chapter_flags and CHAPTER_DISPLAY_MASK
-        set(mode) = setFlags(mode, DISPLAY_MASK)
+        set(mode) = setChapterFlags(mode, CHAPTER_DISPLAY_MASK)
 
     var readFilter: Int
-        get() = chapter_flags and READ_MASK
+        get() = chapter_flags and CHAPTER_READ_MASK
-        set(filter) = setFlags(filter, READ_MASK)
+        set(filter) = setChapterFlags(filter, CHAPTER_READ_MASK)
 
     var downloadedFilter: Int
-        get() = chapter_flags and DOWNLOADED_MASK
+        get() = chapter_flags and CHAPTER_DOWNLOADED_MASK
-        set(filter) = setFlags(filter, DOWNLOADED_MASK)
+        set(filter) = setChapterFlags(filter, CHAPTER_DOWNLOADED_MASK)
 
     var bookmarkedFilter: Int
-        get() = chapter_flags and BOOKMARKED_MASK
+        get() = chapter_flags and CHAPTER_BOOKMARKED_MASK
-        set(filter) = setFlags(filter, BOOKMARKED_MASK)
+        set(filter) = setChapterFlags(filter, CHAPTER_BOOKMARKED_MASK)
 
     var sorting: Int
-        get() = chapter_flags and SORTING_MASK
+        get() = chapter_flags and CHAPTER_SORTING_MASK
-        set(sort) = setFlags(sort, SORTING_MASK)
+        set(sort) = setChapterFlags(sort, CHAPTER_SORTING_MASK)
+
+    var readingModeType: Int
+        get() = viewer_flags and ReadingModeType.MASK
+        set(readingMode) = setViewerFlags(readingMode, ReadingModeType.MASK)
+
+    var orientationType: Int
+        get() = viewer_flags and OrientationType.MASK
+        set(rotationType) = setViewerFlags(rotationType, OrientationType.MASK)
 
     companion object {
 
-        const val SORT_DESC = 0x00000000
-        const val SORT_ASC = 0x00000001
-        const val SORT_MASK = 0x00000001
-
         // Generic filter that does not filter anything
         const val SHOW_ALL = 0x00000000
 
-        const val SHOW_UNREAD = 0x00000002
+        const val CHAPTER_SORT_DESC = 0x00000000
-        const val SHOW_READ = 0x00000004
+        const val CHAPTER_SORT_ASC = 0x00000001
-        const val READ_MASK = 0x00000006
+        const val CHAPTER_SORT_MASK = 0x00000001
 
-        const val SHOW_DOWNLOADED = 0x00000008
+        const val CHAPTER_SHOW_UNREAD = 0x00000002
-        const val SHOW_NOT_DOWNLOADED = 0x00000010
+        const val CHAPTER_SHOW_READ = 0x00000004
-        const val DOWNLOADED_MASK = 0x00000018
+        const val CHAPTER_READ_MASK = 0x00000006
 
-        const val SHOW_BOOKMARKED = 0x00000020
+        const val CHAPTER_SHOW_DOWNLOADED = 0x00000008
-        const val SHOW_NOT_BOOKMARKED = 0x00000040
+        const val CHAPTER_SHOW_NOT_DOWNLOADED = 0x00000010
-        const val BOOKMARKED_MASK = 0x00000060
+        const val CHAPTER_DOWNLOADED_MASK = 0x00000018
 
-        const val SORTING_SOURCE = 0x00000000
+        const val CHAPTER_SHOW_BOOKMARKED = 0x00000020
-        const val SORTING_NUMBER = 0x00000100
+        const val CHAPTER_SHOW_NOT_BOOKMARKED = 0x00000040
-        const val SORTING_UPLOAD_DATE = 0x00000200
+        const val CHAPTER_BOOKMARKED_MASK = 0x00000060
-        const val SORTING_MASK = 0x00000300
 
-        const val DISPLAY_NAME = 0x00000000
+        const val CHAPTER_SORTING_SOURCE = 0x00000000
-        const val DISPLAY_NUMBER = 0x00100000
+        const val CHAPTER_SORTING_NUMBER = 0x00000100
-        const val DISPLAY_MASK = 0x00100000
+        const val CHAPTER_SORTING_UPLOAD_DATE = 0x00000200
+        const val CHAPTER_SORTING_MASK = 0x00000300
+
+        const val CHAPTER_DISPLAY_NAME = 0x00000000
+        const val CHAPTER_DISPLAY_NUMBER = 0x00100000
+        const val CHAPTER_DISPLAY_MASK = 0x00100000
 
         fun create(source: Long): Manga = MangaImpl().apply {
             this.source = source

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/models/MangaImpl.kt

@@ -5,7 +5,7 @@ import suwayomi.tachidesk.manga.model.table.MangaTable
 
 open class MangaImpl : Manga {
 
-    override var id: Long? = 0
+    override var id: Long? = null
 
     override var source: Long = -1
 
@@ -29,6 +29,8 @@ open class MangaImpl : Manga {
 
     override var last_update: Long = 0
 
+    override var next_update: Long = 0
+
     override var date_added: Long = 0
 
     override var initialized: Boolean = false
@@ -42,7 +44,7 @@ open class MangaImpl : Manga {
      * 4 -> Webtoon
      * 5 -> Continues Vertical
      */
-    override var viewer: Int = 0
+    override var viewer_flags: Int = 0
 
     /** Contains some useful info about
      */
@@ -70,7 +72,7 @@ open class MangaImpl : Manga {
                 url = mangaRecord[MangaTable.url]
                 title = mangaRecord[MangaTable.title]
                 source = mangaRecord[MangaTable.sourceReference]
-                viewer = 0 // TODO: implement
+                viewer_flags = 0 // TODO: implement
                 chapter_flags = 0 // TODO: implement
             }
         }

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/ProtoBackupBase.kt

@@ -0,0 +1,16 @@
+package suwayomi.tachidesk.manga.impl.backup.proto
+
+/*
+ * Copyright (C) Contributors to the Suwayomi project
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
+
+import kotlinx.serialization.protobuf.ProtoBuf
+
+open class ProtoBackupBase {
+    var sourceMapping: Map<Long, String> = emptyMap()
+
+    val parser = ProtoBuf
+}

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/ProtoBackupExport.kt

@@ -0,0 +1,137 @@
+package suwayomi.tachidesk.manga.impl.backup.proto
+
+/*
+ * Copyright (C) Contributors to the Suwayomi project
+ * 
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
+
+import kotlinx.coroutines.runBlocking
+import okio.buffer
+import okio.gzip
+import okio.sink
+import org.jetbrains.exposed.sql.Query
+import org.jetbrains.exposed.sql.SortOrder
+import org.jetbrains.exposed.sql.select
+import org.jetbrains.exposed.sql.selectAll
+import org.jetbrains.exposed.sql.transactions.transaction
+import suwayomi.tachidesk.manga.impl.CategoryManga
+import suwayomi.tachidesk.manga.impl.Chapter
+import suwayomi.tachidesk.manga.impl.backup.BackupFlags
+import suwayomi.tachidesk.manga.impl.backup.proto.models.Backup
+import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupCategory
+import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupChapter
+import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupManga
+import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupSerializer
+import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupSource
+import suwayomi.tachidesk.manga.model.table.CategoryTable
+import suwayomi.tachidesk.manga.model.table.MangaStatus
+import suwayomi.tachidesk.manga.model.table.MangaTable
+import suwayomi.tachidesk.manga.model.table.SourceTable
+import suwayomi.tachidesk.manga.model.table.toDataClass
+import java.io.ByteArrayOutputStream
+import java.io.InputStream
+
+object ProtoBackupExport : ProtoBackupBase() {
+    suspend fun createBackup(flags: BackupFlags): InputStream {
+        // Create root object
+
+        val databaseManga = transaction { MangaTable.select { MangaTable.inLibrary eq true } }
+
+        val backup: Backup = transaction {
+            Backup(
+                backupManga(databaseManga, flags),
+                backupCategories(),
+                emptyList(),
+                backupExtensionInfo(databaseManga)
+            )
+        }
+
+        val byteArray = parser.encodeToByteArray(BackupSerializer, backup)
+
+        val byteStream = ByteArrayOutputStream()
+        byteStream.sink().gzip().buffer().use { it.write(byteArray) }
+
+        return byteStream.toByteArray().inputStream()
+    }
+
+    private fun backupManga(databaseManga: Query, flags: BackupFlags): List<BackupManga> {
+        return databaseManga.map { mangaRow ->
+            val backupManga = BackupManga(
+                mangaRow[MangaTable.sourceReference],
+                mangaRow[MangaTable.url],
+                mangaRow[MangaTable.title],
+                mangaRow[MangaTable.artist],
+                mangaRow[MangaTable.author],
+                mangaRow[MangaTable.description],
+                mangaRow[MangaTable.genre]?.split(", ") ?: emptyList(),
+                MangaStatus.valueOf(mangaRow[MangaTable.status]).value,
+                mangaRow[MangaTable.thumbnail_url],
+                0, // not supported in Tachidesk
+                0, // not supported in Tachidesk
+            )
+
+            val mangaId = mangaRow[MangaTable.id].value
+
+            if (flags.includeChapters) {
+                val chapters = runBlocking { Chapter.getChapterList(mangaId) }
+                backupManga.chapters = chapters.map {
+                    BackupChapter(
+                        it.url,
+                        it.name,
+                        it.scanlator,
+                        it.read,
+                        it.bookmarked,
+                        it.lastPageRead,
+                        0, // not supported in Tachidesk
+                        it.uploadDate,
+                        it.chapterNumber,
+                        it.index,
+                    )
+                }
+            }
+
+            if (flags.includeCategories) {
+                backupManga.categories = CategoryManga.getMangaCategories(mangaId).map { it.order }
+            }
+
+//            if(flags.includeTracking) {
+//                backupManga.tracking = TODO()
+//            }
+
+//            if (flags.includeHistory) {
+//                backupManga.history = TODO()
+//            }
+
+            backupManga
+        }
+    }
+
+    private fun backupCategories(): List<BackupCategory> {
+        return CategoryTable.selectAll().orderBy(CategoryTable.order to SortOrder.ASC).map {
+            CategoryTable.toDataClass(it)
+        }.map {
+            BackupCategory(
+                it.name,
+                it.order,
+                0, // not supported in Tachidesk
+            )
+        }
+    }
+
+    private fun backupExtensionInfo(mangas: Query): List<BackupSource> {
+        return mangas
+            .asSequence()
+            .map { it[MangaTable.sourceReference] }
+            .distinct()
+            .map {
+                val sourceRow = SourceTable.select { SourceTable.id eq it }.firstOrNull()
+                BackupSource(
+                    sourceRow?.get(SourceTable.name) ?: "",
+                    it
+                )
+            }
+            .toList()
+    }
+}

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/ProtoBackupImport.kt

@@ -0,0 +1,237 @@
+package suwayomi.tachidesk.manga.impl.backup.proto
+
+/*
+ * Copyright (C) Contributors to the Suwayomi project
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
+
+import mu.KotlinLogging
+import okio.buffer
+import okio.gzip
+import okio.source
+import org.jetbrains.exposed.sql.and
+import org.jetbrains.exposed.sql.insert
+import org.jetbrains.exposed.sql.insertAndGetId
+import org.jetbrains.exposed.sql.select
+import org.jetbrains.exposed.sql.transactions.transaction
+import org.jetbrains.exposed.sql.update
+import suwayomi.tachidesk.manga.impl.Category
+import suwayomi.tachidesk.manga.impl.CategoryManga
+import suwayomi.tachidesk.manga.impl.backup.AbstractBackupValidator.ValidationResult
+import suwayomi.tachidesk.manga.impl.backup.models.Chapter
+import suwayomi.tachidesk.manga.impl.backup.models.Manga
+import suwayomi.tachidesk.manga.impl.backup.models.Track
+import suwayomi.tachidesk.manga.impl.backup.proto.ProtoBackupValidator.validate
+import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupCategory
+import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupHistory
+import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupManga
+import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupSerializer
+import suwayomi.tachidesk.manga.model.table.CategoryTable
+import suwayomi.tachidesk.manga.model.table.ChapterTable
+import suwayomi.tachidesk.manga.model.table.MangaTable
+import java.io.InputStream
+import java.lang.Integer.max
+import java.util.Date
+
+object ProtoBackupImport : ProtoBackupBase() {
+    private val logger = KotlinLogging.logger {}
+
+    private var restoreAmount = 0
+
+    private val errors = mutableListOf<Pair<Date, String>>()
+
+    suspend fun performRestore(sourceStream: InputStream): ValidationResult {
+        val backupString = sourceStream.source().gzip().buffer().use { it.readByteArray() }
+        val backup = parser.decodeFromByteArray(BackupSerializer, backupString)
+
+        val validationResult = validate(backup)
+
+        restoreAmount = backup.backupManga.size + 1 // +1 for categories
+
+        // Restore categories
+        if (backup.backupCategories.isNotEmpty()) {
+            restoreCategories(backup.backupCategories)
+        }
+
+        val categoryMapping = transaction {
+            backup.backupCategories.associate {
+                it.order to CategoryTable.select { CategoryTable.name eq it.name }.first()[CategoryTable.id].value
+            }
+        }
+
+        // Store source mapping for error messages
+        sourceMapping = backup.getSourceMap()
+
+        // Restore individual manga
+        backup.backupManga.forEach {
+            restoreManga(it, backup.backupCategories, categoryMapping)
+        }
+
+        logger.info {
+            """
+                Restore Errors:
+                ${errors.joinToString("\n") { "${it.first} - ${it.second}" }}
+                Restore Summary:
+                - Missing Sources:
+                    ${validationResult.missingSources.joinToString("\n                    ")}
+                - Missing Trackers:
+                    ${validationResult.missingTrackers.joinToString("\n                    ")}
+            """.trimIndent()
+        }
+
+        return validationResult
+    }
+
+    private fun restoreCategories(backupCategories: List<BackupCategory>) {
+        val dbCategories = Category.getCategoryList()
+
+        // Iterate over them and create missing categories
+        backupCategories.forEach { category ->
+            if (dbCategories.none { it.name == category.name }) {
+                Category.createCategory(category.name)
+            }
+        }
+    }
+
+    private fun restoreManga(
+        backupManga: BackupManga,
+        backupCategories: List<BackupCategory>,
+        categoryMapping: Map<Int, Int>
+    ) {
+        val manga = backupManga.getMangaImpl()
+        val chapters = backupManga.getChaptersImpl()
+        val categories = backupManga.categories
+        val history = backupManga.brokenHistory.map { BackupHistory(it.url, it.lastRead) } + backupManga.history
+        val tracks = backupManga.getTrackingImpl()
+
+        try {
+            restoreMangaData(manga, chapters, categories, history, tracks, backupCategories, categoryMapping)
+        } catch (e: Exception) {
+            val sourceName = sourceMapping[manga.source] ?: manga.source.toString()
+            errors.add(Date() to "${manga.title} [$sourceName]: ${e.message}")
+        }
+    }
+
+    @Suppress("UNUSED_PARAMETER") // TODO: remove
+    private fun restoreMangaData(
+        manga: Manga,
+        chapters: List<Chapter>,
+        categories: List<Int>,
+        history: List<BackupHistory>,
+        tracks: List<Track>,
+        backupCategories: List<BackupCategory>,
+        categoryMapping: Map<Int, Int>
+    ) {
+        val dbManga = transaction {
+            MangaTable.select { (MangaTable.url eq manga.url) and (MangaTable.sourceReference eq manga.source) }
+                .firstOrNull()
+        }
+
+        if (dbManga == null) { // Manga not in database
+            transaction {
+                // insert manga to database
+                val mangaId = MangaTable.insertAndGetId {
+                    it[url] = manga.url
+                    it[title] = manga.title
+
+                    it[artist] = manga.artist
+                    it[author] = manga.author
+                    it[description] = manga.description
+                    it[genre] = manga.genre
+                    it[status] = manga.status
+                    it[thumbnail_url] = manga.thumbnail_url
+
+                    it[sourceReference] = manga.source
+
+                    it[initialized] = manga.description != null
+
+                    it[inLibrary] = manga.favorite
+                }.value
+
+                // insert chapter data
+                val chaptersLength = chapters.size
+                chapters.forEach { chapter ->
+                    ChapterTable.insert {
+                        it[url] = chapter.url
+                        it[name] = chapter.name
+                        it[date_upload] = chapter.date_upload
+                        it[chapter_number] = chapter.chapter_number
+                        it[scanlator] = chapter.scanlator
+
+                        it[chapterIndex] = chaptersLength - chapter.source_order
+                        it[ChapterTable.manga] = mangaId
+
+                        it[isRead] = chapter.read
+                        it[lastPageRead] = chapter.last_page_read
+                        it[isBookmarked] = chapter.bookmark
+                    }
+                }
+
+                // insert categories
+                categories.forEach { backupCategoryOrder ->
+                    CategoryManga.addMangaToCategory(mangaId, categoryMapping[backupCategoryOrder]!!)
+                }
+            }
+        } else { // Manga in database
+            transaction {
+                val mangaId = dbManga[MangaTable.id].value
+
+                // Merge manga data
+                MangaTable.update({ MangaTable.id eq mangaId }) {
+                    it[artist] = manga.artist ?: dbManga[artist]
+                    it[author] = manga.author ?: dbManga[author]
+                    it[description] = manga.description ?: dbManga[description]
+                    it[genre] = manga.genre ?: dbManga[genre]
+                    it[status] = manga.status
+                    it[thumbnail_url] = manga.thumbnail_url ?: dbManga[thumbnail_url]
+
+                    it[initialized] = dbManga[initialized] || manga.description != null
+
+                    it[inLibrary] = manga.favorite || dbManga[inLibrary]
+                }
+
+                // merge chapter data
+                val chaptersLength = chapters.size
+                val dbChapters = ChapterTable.select { ChapterTable.manga eq mangaId }
+
+                chapters.forEach { chapter ->
+                    val dbChapter = dbChapters.find { it[ChapterTable.url] == chapter.url }
+
+                    if (dbChapter == null) {
+                        ChapterTable.insert {
+                            it[url] = chapter.url
+                            it[name] = chapter.name
+                            it[date_upload] = chapter.date_upload
+                            it[chapter_number] = chapter.chapter_number
+                            it[scanlator] = chapter.scanlator
+
+                            it[chapterIndex] = chaptersLength - chapter.source_order
+                            it[ChapterTable.manga] = mangaId
+
+                            it[isRead] = chapter.read
+                            it[lastPageRead] = chapter.last_page_read
+                            it[isBookmarked] = chapter.bookmark
+                        }
+                    } else {
+                        ChapterTable.update({ (ChapterTable.url eq dbChapter[ChapterTable.url]) and (ChapterTable.manga eq mangaId) }) {
+                            it[isRead] = chapter.read || dbChapter[isRead]
+                            it[lastPageRead] = max(chapter.last_page_read, dbChapter[lastPageRead])
+                            it[isBookmarked] = chapter.bookmark || dbChapter[isBookmarked]
+                        }
+                    }
+                }
+
+                // merge categories
+                categories.forEach { backupCategoryOrder ->
+                    CategoryManga.addMangaToCategory(mangaId, categoryMapping[backupCategoryOrder]!!)
+                }
+            }
+        }
+
+        // TODO: insert/merge history
+
+        // TODO: insert/merge tracking
+    }
+}

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/ProtoBackupValidator.kt

@@ -0,0 +1,57 @@
+package suwayomi.tachidesk.manga.impl.backup.proto
+
+/*
+ * Copyright (C) Contributors to the Suwayomi project
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
+
+import okio.buffer
+import okio.gzip
+import okio.source
+import org.jetbrains.exposed.sql.select
+import org.jetbrains.exposed.sql.transactions.transaction
+import suwayomi.tachidesk.manga.impl.backup.AbstractBackupValidator
+import suwayomi.tachidesk.manga.impl.backup.proto.models.Backup
+import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupSerializer
+import suwayomi.tachidesk.manga.model.table.SourceTable
+import java.io.InputStream
+
+object ProtoBackupValidator : AbstractBackupValidator() {
+    fun validate(backup: Backup): ValidationResult {
+        if (backup.backupManga.isEmpty()) {
+            throw Exception("Backup does not contain any manga.")
+        }
+
+        val sources = backup.getSourceMap()
+
+        val missingSources = transaction {
+            sources
+                .filter { SourceTable.select { SourceTable.id eq it.key }.firstOrNull() == null }
+                .map { "${it.value} (${it.key})" }
+                .sorted()
+        }
+
+//        val trackers = backup.backupManga
+//            .flatMap { it.tracking }
+//            .map { it.syncId }
+//            .distinct()
+
+        val missingTrackers = listOf("")
+//        val missingTrackers = trackers
+//            .mapNotNull { trackManager.getService(it) }
+//            .filter { !it.isLogged }
+//            .map { context.getString(it.nameRes()) }
+//            .sorted()
+
+        return ValidationResult(missingSources, missingTrackers)
+    }
+
+    suspend fun validate(sourceStream: InputStream): ValidationResult {
+        val backupString = sourceStream.source().gzip().buffer().use { it.readByteArray() }
+        val backup = ProtoBackupImport.parser.decodeFromByteArray(BackupSerializer, backupString)
+
+        return validate(backup)
+    }
+}

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/models/Backup.kt

@@ -0,0 +1,18 @@
+package suwayomi.tachidesk.manga.impl.backup.proto.models
+
+import kotlinx.serialization.Serializable
+import kotlinx.serialization.protobuf.ProtoNumber
+
+@Serializable
+data class Backup(
+    @ProtoNumber(1) val backupManga: List<BackupManga>,
+    @ProtoNumber(2) var backupCategories: List<BackupCategory> = emptyList(),
+    // Bump by 100 to specify this is a 0.x value
+    @ProtoNumber(100) var brokenBackupSources: List<BrokenBackupSource> = emptyList(),
+    @ProtoNumber(101) var backupSources: List<BackupSource> = emptyList(),
+) {
+    fun getSourceMap(): Map<Long, String> {
+        return (brokenBackupSources.map { BackupSource(it.name, it.sourceId) } + backupSources)
+            .associate { it.sourceId to it.name }
+    }
+}

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/models/BackupCategory.kt

@@ -0,0 +1,33 @@
+package suwayomi.tachidesk.manga.impl.backup.proto.models
+
+import kotlinx.serialization.Serializable
+import kotlinx.serialization.protobuf.ProtoNumber
+import suwayomi.tachidesk.manga.impl.backup.models.Category
+import suwayomi.tachidesk.manga.impl.backup.models.CategoryImpl
+
+@Serializable
+class BackupCategory(
+    @ProtoNumber(1) var name: String,
+    @ProtoNumber(2) var order: Int = 0,
+    // @ProtoNumber(3) val updateInterval: Int = 0, 1.x value not used in 0.x
+    // Bump by 100 to specify this is a 0.x value
+    @ProtoNumber(100) var flags: Int = 0,
+) {
+    fun getCategoryImpl(): CategoryImpl {
+        return CategoryImpl().apply {
+            name = this@BackupCategory.name
+            flags = this@BackupCategory.flags
+            order = this@BackupCategory.order
+        }
+    }
+
+    companion object {
+        fun copyFrom(category: Category): BackupCategory {
+            return BackupCategory(
+                name = category.name,
+                order = category.order,
+                flags = category.flags
+            )
+        }
+    }
+}

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/models/BackupChapter.kt

@@ -0,0 +1,56 @@
+package suwayomi.tachidesk.manga.impl.backup.proto.models
+
+import kotlinx.serialization.Serializable
+import kotlinx.serialization.protobuf.ProtoNumber
+import suwayomi.tachidesk.manga.impl.backup.models.Chapter
+import suwayomi.tachidesk.manga.impl.backup.models.ChapterImpl
+
+@Serializable
+data class BackupChapter(
+    // in 1.x some of these values have different names
+    // url is called key in 1.x
+    @ProtoNumber(1) var url: String,
+    @ProtoNumber(2) var name: String,
+    @ProtoNumber(3) var scanlator: String? = null,
+    @ProtoNumber(4) var read: Boolean = false,
+    @ProtoNumber(5) var bookmark: Boolean = false,
+    // lastPageRead is called progress in 1.x
+    @ProtoNumber(6) var lastPageRead: Int = 0,
+    @ProtoNumber(7) var dateFetch: Long = 0,
+    @ProtoNumber(8) var dateUpload: Long = 0,
+    // chapterNumber is called number is 1.x
+    @ProtoNumber(9) var chapterNumber: Float = 0F,
+    @ProtoNumber(10) var sourceOrder: Int = 0,
+) {
+    fun toChapterImpl(): ChapterImpl {
+        return ChapterImpl().apply {
+            url = this@BackupChapter.url
+            name = this@BackupChapter.name
+            chapter_number = this@BackupChapter.chapterNumber
+            scanlator = this@BackupChapter.scanlator
+            read = this@BackupChapter.read
+            bookmark = this@BackupChapter.bookmark
+            last_page_read = this@BackupChapter.lastPageRead
+            date_fetch = this@BackupChapter.dateFetch
+            date_upload = this@BackupChapter.dateUpload
+            source_order = this@BackupChapter.sourceOrder
+        }
+    }
+
+    companion object {
+        fun copyFrom(chapter: Chapter): BackupChapter {
+            return BackupChapter(
+                url = chapter.url,
+                name = chapter.name,
+                chapterNumber = chapter.chapter_number,
+                scanlator = chapter.scanlator,
+                read = chapter.read,
+                bookmark = chapter.bookmark,
+                lastPageRead = chapter.last_page_read,
+                dateFetch = chapter.date_fetch,
+                dateUpload = chapter.date_upload,
+                sourceOrder = chapter.source_order
+            )
+        }
+    }
+}

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/models/BackupFull.kt

@@ -0,0 +1,12 @@
+package suwayomi.tachidesk.manga.impl.backup.proto.models
+
+import java.text.SimpleDateFormat
+import java.util.Date
+import java.util.Locale
+
+object BackupFull {
+    fun getDefaultFilename(): String {
+        val date = SimpleDateFormat("yyyy-MM-dd_HH-mm", Locale.getDefault()).format(Date())
+        return "tachiyomi_$date.proto.gz"
+    }
+}

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/models/BackupHistory.kt

@@ -0,0 +1,16 @@
+package suwayomi.tachidesk.manga.impl.backup.proto.models
+
+import kotlinx.serialization.Serializable
+import kotlinx.serialization.protobuf.ProtoNumber
+
+@Serializable
+data class BrokenBackupHistory(
+    @ProtoNumber(0) var url: String,
+    @ProtoNumber(1) var lastRead: Long
+)
+
+@Serializable
+data class BackupHistory(
+    @ProtoNumber(1) var url: String,
+    @ProtoNumber(2) var lastRead: Long
+)

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/models/BackupManga.kt

@@ -0,0 +1,90 @@
+package suwayomi.tachidesk.manga.impl.backup.proto.models
+
+import kotlinx.serialization.Serializable
+import kotlinx.serialization.protobuf.ProtoNumber
+import suwayomi.tachidesk.manga.impl.backup.models.ChapterImpl
+import suwayomi.tachidesk.manga.impl.backup.models.Manga
+import suwayomi.tachidesk.manga.impl.backup.models.MangaImpl
+import suwayomi.tachidesk.manga.impl.backup.models.TrackImpl
+
+@Serializable
+data class BackupManga(
+    // in 1.x some of these values have different names
+    @ProtoNumber(1) var source: Long,
+    // url is called key in 1.x
+    @ProtoNumber(2) var url: String,
+    @ProtoNumber(3) var title: String = "",
+    @ProtoNumber(4) var artist: String? = null,
+    @ProtoNumber(5) var author: String? = null,
+    @ProtoNumber(6) var description: String? = null,
+    @ProtoNumber(7) var genre: List<String> = emptyList(),
+    @ProtoNumber(8) var status: Int = 0,
+    // thumbnailUrl is called cover in 1.x
+    @ProtoNumber(9) var thumbnailUrl: String? = null,
+    // @ProtoNumber(10) val customCover: String = "", 1.x value, not used in 0.x
+    // @ProtoNumber(11) val lastUpdate: Long = 0, 1.x value, not used in 0.x
+    // @ProtoNumber(12) val lastInit: Long = 0, 1.x value, not used in 0.x
+    @ProtoNumber(13) var dateAdded: Long = 0,
+    @ProtoNumber(14) var viewer: Int = 0, // Replaced by viewer_flags
+    // @ProtoNumber(15) val flags: Int = 0, 1.x value, not used in 0.x
+    @ProtoNumber(16) var chapters: List<BackupChapter> = emptyList(),
+    @ProtoNumber(17) var categories: List<Int> = emptyList(),
+    @ProtoNumber(18) var tracking: List<BackupTracking> = emptyList(),
+    // Bump by 100 for values that are not saved/implemented in 1.x but are used in 0.x
+    @ProtoNumber(100) var favorite: Boolean = true,
+    @ProtoNumber(101) var chapterFlags: Int = 0,
+    @ProtoNumber(102) var brokenHistory: List<BrokenBackupHistory> = emptyList(),
+    @ProtoNumber(103) var viewer_flags: Int? = null,
+    @ProtoNumber(104) var history: List<BackupHistory> = emptyList(),
+) {
+    fun getMangaImpl(): MangaImpl {
+        return MangaImpl().apply {
+            url = this@BackupManga.url
+            title = this@BackupManga.title
+            artist = this@BackupManga.artist
+            author = this@BackupManga.author
+            description = this@BackupManga.description
+            genre = this@BackupManga.genre.joinToString()
+            status = this@BackupManga.status
+            thumbnail_url = this@BackupManga.thumbnailUrl
+            favorite = this@BackupManga.favorite
+            source = this@BackupManga.source
+            date_added = this@BackupManga.dateAdded
+            viewer_flags = this@BackupManga.viewer_flags ?: this@BackupManga.viewer
+            chapter_flags = this@BackupManga.chapterFlags
+        }
+    }
+
+    fun getChaptersImpl(): List<ChapterImpl> {
+        return chapters.map {
+            it.toChapterImpl()
+        }
+    }
+
+    fun getTrackingImpl(): List<TrackImpl> {
+        return tracking.map {
+            it.getTrackingImpl()
+        }
+    }
+
+    companion object {
+        fun copyFrom(manga: Manga): BackupManga {
+            return BackupManga(
+                url = manga.url,
+                title = manga.title,
+                artist = manga.artist,
+                author = manga.author,
+                description = manga.description,
+                genre = manga.getGenres() ?: emptyList(),
+                status = manga.status,
+                thumbnailUrl = manga.thumbnail_url,
+                favorite = manga.favorite,
+                source = manga.source,
+                dateAdded = manga.date_added,
+                viewer = manga.readingModeType,
+                viewer_flags = manga.viewer_flags,
+                chapterFlags = manga.chapter_flags
+            )
+        }
+    }
+}

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/models/BackupSerializer.kt

@@ -0,0 +1,6 @@
+package suwayomi.tachidesk.manga.impl.backup.proto.models
+
+import kotlinx.serialization.Serializer
+
+@Serializer(forClass = Backup::class)
+object BackupSerializer

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/models/BackupSource.kt

@@ -0,0 +1,26 @@
+package suwayomi.tachidesk.manga.impl.backup.proto.models
+
+import eu.kanade.tachiyomi.source.Source
+import kotlinx.serialization.Serializable
+import kotlinx.serialization.protobuf.ProtoNumber
+
+@Serializable
+data class BrokenBackupSource(
+    @ProtoNumber(0) var name: String = "",
+    @ProtoNumber(1) var sourceId: Long
+)
+
+@Serializable
+data class BackupSource(
+    @ProtoNumber(1) var name: String = "",
+    @ProtoNumber(2) var sourceId: Long
+) {
+    companion object {
+        fun copyFrom(source: Source): BackupSource {
+            return BackupSource(
+                name = source.name,
+                sourceId = source.id
+            )
+        }
+    }
+}

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/models/BackupTracking.kt

@@ -0,0 +1,65 @@
+package suwayomi.tachidesk.manga.impl.backup.proto.models
+
+import kotlinx.serialization.Serializable
+import kotlinx.serialization.protobuf.ProtoNumber
+import suwayomi.tachidesk.manga.impl.backup.models.Track
+import suwayomi.tachidesk.manga.impl.backup.models.TrackImpl
+
+@Serializable
+data class BackupTracking(
+    // in 1.x some of these values have different types or names
+    // syncId is called siteId in 1,x
+    @ProtoNumber(1) var syncId: Int,
+    // LibraryId is not null in 1.x
+    @ProtoNumber(2) var libraryId: Long,
+    @ProtoNumber(3) var mediaId: Int = 0,
+    // trackingUrl is called mediaUrl in 1.x
+    @ProtoNumber(4) var trackingUrl: String = "",
+    @ProtoNumber(5) var title: String = "",
+    // lastChapterRead is called last read, and it has been changed to a float in 1.x
+    @ProtoNumber(6) var lastChapterRead: Float = 0F,
+    @ProtoNumber(7) var totalChapters: Int = 0,
+    @ProtoNumber(8) var score: Float = 0F,
+    @ProtoNumber(9) var status: Int = 0,
+    // startedReadingDate is called startReadTime in 1.x
+    @ProtoNumber(10) var startedReadingDate: Long = 0,
+    // finishedReadingDate is called endReadTime in 1.x
+    @ProtoNumber(11) var finishedReadingDate: Long = 0,
+) {
+    fun getTrackingImpl(): TrackImpl {
+        return TrackImpl().apply {
+            sync_id = this@BackupTracking.syncId
+            media_id = this@BackupTracking.mediaId
+            library_id = this@BackupTracking.libraryId
+            title = this@BackupTracking.title
+            // convert from float to int because of 1.x types
+            last_chapter_read = this@BackupTracking.lastChapterRead.toInt()
+            total_chapters = this@BackupTracking.totalChapters
+            score = this@BackupTracking.score
+            status = this@BackupTracking.status
+            started_reading_date = this@BackupTracking.startedReadingDate
+            finished_reading_date = this@BackupTracking.finishedReadingDate
+            tracking_url = this@BackupTracking.trackingUrl
+        }
+    }
+
+    companion object {
+        fun copyFrom(track: Track): BackupTracking {
+            return BackupTracking(
+                syncId = track.sync_id,
+                mediaId = track.media_id,
+                // forced not null so its compatible with 1.x backup system
+                libraryId = track.library_id!!,
+                title = track.title,
+                // convert to float for 1.x
+                lastChapterRead = track.last_chapter_read.toFloat(),
+                totalChapters = track.total_chapters,
+                score = track.score,
+                status = track.status,
+                startedReadingDate = track.started_reading_date,
+                finishedReadingDate = track.finished_reading_date,
+                trackingUrl = track.tracking_url
+            )
+        }
+    }
+}

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/Downloader.kt

@@ -8,6 +8,7 @@ package suwayomi.tachidesk.manga.impl.download
  * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
 
 import kotlinx.coroutines.runBlocking
+import mu.KotlinLogging
 import org.jetbrains.exposed.sql.and
 import org.jetbrains.exposed.sql.transactions.transaction
 import org.jetbrains.exposed.sql.update
@@ -21,6 +22,8 @@ import suwayomi.tachidesk.manga.impl.download.model.DownloadState.Queued
 import suwayomi.tachidesk.manga.model.table.ChapterTable
 import java.util.concurrent.CopyOnWriteArrayList
 
+private val logger = KotlinLogging.logger {}
+
 class Downloader(private val downloadQueue: CopyOnWriteArrayList<DownloadChapter>, val notifier: () -> Unit) : Thread() {
     var shouldStop: Boolean = false
 
@@ -67,10 +70,10 @@ class Downloader(private val downloadQueue: CopyOnWriteArrayList<DownloadChapter
                 downloadQueue.removeIf { it.mangaId == download.mangaId && it.chapterIndex == download.chapterIndex }
                 step()
             } catch (e: DownloadShouldStopException) {
-                println("Downloader was stopped")
+                logger.debug("Downloader was stopped")
                 downloadQueue.filter { it.state == Downloading }.forEach { it.state = Queued }
             } catch (e: Exception) {
-                println("Downloader faced an exception")
+                logger.debug("Downloader faced an exception")
                 downloadQueue.filter { it.state == Downloading }.forEach { it.state = Error; it.tries++ }
                 e.printStackTrace()
             } finally {

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/Extension.kt

@@ -17,6 +17,7 @@ import mu.KotlinLogging
 import okhttp3.Request
 import okio.buffer
 import okio.sink
+import okio.source
 import org.jetbrains.exposed.sql.deleteWhere
 import org.jetbrains.exposed.sql.insert
 import org.jetbrains.exposed.sql.select
@@ -27,6 +28,8 @@ import org.kodein.di.conf.global
 import org.kodein.di.instance
 import suwayomi.tachidesk.manga.impl.extension.ExtensionsList.extensionTableAsDataClass
 import suwayomi.tachidesk.manga.impl.extension.github.ExtensionGithubApi
+import suwayomi.tachidesk.manga.impl.util.GetHttpSource
+import suwayomi.tachidesk.manga.impl.util.PackageTools
 import suwayomi.tachidesk.manga.impl.util.PackageTools.EXTENSION_FEATURE
 import suwayomi.tachidesk.manga.impl.util.PackageTools.LIB_VERSION_MAX
 import suwayomi.tachidesk.manga.impl.util.PackageTools.LIB_VERSION_MIN
@@ -34,11 +37,9 @@ import suwayomi.tachidesk.manga.impl.util.PackageTools.METADATA_NSFW
 import suwayomi.tachidesk.manga.impl.util.PackageTools.METADATA_SOURCE_CLASS
 import suwayomi.tachidesk.manga.impl.util.PackageTools.dex2jar
 import suwayomi.tachidesk.manga.impl.util.PackageTools.getPackageInfo
-import suwayomi.tachidesk.manga.impl.util.PackageTools.getSignatureHash
 import suwayomi.tachidesk.manga.impl.util.PackageTools.loadExtensionSources
-import suwayomi.tachidesk.manga.impl.util.PackageTools.trustedSignatures
 import suwayomi.tachidesk.manga.impl.util.network.await
-import suwayomi.tachidesk.manga.impl.util.storage.CachedImageResponse.getCachedImageResponse
+import suwayomi.tachidesk.manga.impl.util.storage.ImageResponse.getImageResponse
 import suwayomi.tachidesk.manga.model.table.ExtensionTable
 import suwayomi.tachidesk.manga.model.table.SourceTable
 import suwayomi.tachidesk.server.ApplicationDirs
@@ -50,10 +51,8 @@ object Extension {
     private val logger = KotlinLogging.logger {}
     private val applicationDirs by DI.global.instance<ApplicationDirs>()
 
-    data class InstallableAPK(
+    private fun Any.isNsfw(): Boolean =
-        val apkFilePath: String,
+        this::class.annotations.any { it.toString() == "@eu.kanade.tachiyomi.annotations.Nsfw()" }
-        val pkgName: String
-    )
 
     suspend fun installExtension(pkgName: String): Int {
         logger.debug("Installing $pkgName")
@@ -70,7 +69,23 @@ object Extension {
         }
     }
 
-    suspend fun installAPK(fetcher: suspend () -> String): Int {
+    suspend fun installExternalExtension(inputStream: InputStream, apkName: String): Int {
+        return installAPK(true) {
+            val savePath = "${applicationDirs.extensionsRoot}/$apkName"
+            logger.debug { "Saving apk at $apkName" }
+            // download apk file
+            val downloadedFile = File(savePath)
+            downloadedFile.sink().buffer().use { sink ->
+                inputStream.source().use { source ->
+                    sink.writeAll(source)
+                    sink.flush()
+                }
+            }
+            savePath
+        }
+    }
+
+    suspend fun installAPK(forceReinstall: Boolean = false, fetcher: suspend () -> String): Int {
         val apkFilePath = fetcher()
         val apkName = File(apkFilePath).name
 
@@ -80,16 +95,19 @@ object Extension {
             ExtensionTable.select { ExtensionTable.apkName eq apkName }.firstOrNull()
         }?.get(ExtensionTable.isInstalled) ?: false
 
-        if (!isInstalled) {
+        val fileNameWithoutType = apkName.substringBefore(".apk")
-            val fileNameWithoutType = apkName.substringBefore(".apk")
 
-            val dirPathWithoutType = "${applicationDirs.extensionsRoot}/$fileNameWithoutType"
+        val dirPathWithoutType = "${applicationDirs.extensionsRoot}/$fileNameWithoutType"
-            val jarFilePath = "$dirPathWithoutType.jar"
+        val jarFilePath = "$dirPathWithoutType.jar"
-            val dexFilePath = "$dirPathWithoutType.dex"
+        val dexFilePath = "$dirPathWithoutType.dex"
 
-            val packageInfo = getPackageInfo(apkFilePath)
+        val packageInfo = getPackageInfo(apkFilePath)
-            val pkgName = packageInfo.packageName
+        val pkgName = packageInfo.packageName
+        if (isInstalled && forceReinstall) {
+            uninstallExtension(pkgName)
+        }
 
+        if (!isInstalled || forceReinstall) {
             if (!packageInfo.reqFeatures.orEmpty().any { it.name == EXTENSION_FEATURE }) {
                 throw Exception("This apk is not a Tachiyomi extension")
             }
@@ -103,32 +121,34 @@ object Extension {
                 )
             }
 
-            val signatureHash = getSignatureHash(packageInfo)
+            // TODO: allow trusting keys
+//            val signatureHash = getSignatureHash(packageInfo)
 
-            if (signatureHash == null) {
+//            if (signatureHash == null) {
-                throw Exception("Package $pkgName isn't signed")
+//                throw Exception("Package $pkgName isn't signed")
-            } else if (signatureHash !in trustedSignatures) {
+//            } else if (signatureHash !in trustedSignatures) {
-                // TODO: allow trusting keys
+//                throw Exception("This apk is not a signed with the official tachiyomi signature")
-                throw Exception("This apk is not a signed with the official tachiyomi signature")
+//            }
-            }
 
             val isNsfw = packageInfo.applicationInfo.metaData.getString(METADATA_NSFW) == "1"
 
-            val className = packageInfo.packageName + packageInfo.applicationInfo.metaData.getString(METADATA_SOURCE_CLASS)
+            val className =
+                packageInfo.packageName + packageInfo.applicationInfo.metaData.getString(METADATA_SOURCE_CLASS)
 
             logger.debug("Main class for extension is $className")
 
             dex2jar(apkFilePath, jarFilePath, fileNameWithoutType)
 
             // clean up
-//            File(apkFilePath).delete()
+            File(apkFilePath).delete()
             File(dexFilePath).delete()
 
             // collect sources from the extension
-            val sources: List<CatalogueSource> = when (val instance = loadExtensionSources(jarFilePath, className)) {
+            val extensionMainClassInstance = loadExtensionSources(jarFilePath, className)
-                is Source -> listOf(instance)
+            val sources: List<CatalogueSource> = when (extensionMainClassInstance) {
-                is SourceFactory -> instance.createSources()
+                is Source -> listOf(extensionMainClassInstance)
-                else -> throw RuntimeException("Unknown source class type! ${instance.javaClass}")
+                is SourceFactory -> extensionMainClassInstance.createSources()
+                else -> throw RuntimeException("Unknown source class type! ${extensionMainClassInstance.javaClass}")
             }.map { it as CatalogueSource }
 
             val langs = sources.map { it.lang }.toSet()
@@ -155,11 +175,13 @@ object Extension {
                 }
 
                 ExtensionTable.update({ ExtensionTable.pkgName eq pkgName }) {
+                    it[this.apkName] = apkName
                     it[this.isInstalled] = true
                     it[this.classFQName] = className
                 }
 
-                val extensionId = ExtensionTable.select { ExtensionTable.pkgName eq pkgName }.first()[ExtensionTable.id].value
+                val extensionId =
+                    ExtensionTable.select { ExtensionTable.pkgName eq pkgName }.first()[ExtensionTable.id].value
 
                 sources.forEach { httpSource ->
                     SourceTable.insert {
@@ -167,8 +189,9 @@ object Extension {
                         it[name] = httpSource.name
                         it[lang] = httpSource.lang
                         it[extension] = extensionId
+                        it[SourceTable.isNsfw] = isNsfw || extensionMainClassInstance.isNsfw()
                     }
-                    logger.debug("Installed source ${httpSource.name} (${httpSource.lang}) with id:${httpSource.id}")
+                    logger.debug { "Installed source ${httpSource.name} (${httpSource.lang}) with id:${httpSource.id}" }
                 }
             }
             return 201 // we installed successfully
@@ -198,19 +221,30 @@ object Extension {
         val extensionRecord = transaction { ExtensionTable.select { ExtensionTable.pkgName eq pkgName }.first() }
         val fileNameWithoutType = extensionRecord[ExtensionTable.apkName].substringBefore(".apk")
         val jarPath = "${applicationDirs.extensionsRoot}/$fileNameWithoutType.jar"
-        transaction {
+        val sources = transaction {
             val extensionId = extensionRecord[ExtensionTable.id].value
 
+            val sources = SourceTable.select { SourceTable.extension eq extensionId }.map { it[SourceTable.id].value }
+
             SourceTable.deleteWhere { SourceTable.extension eq extensionId }
+
             if (extensionRecord[ExtensionTable.isObsolete])
                 ExtensionTable.deleteWhere { ExtensionTable.pkgName eq pkgName }
             else
                 ExtensionTable.update({ ExtensionTable.pkgName eq pkgName }) {
                     it[isInstalled] = false
                 }
+
+            sources
         }
 
         if (File(jarPath).exists()) {
+            // free up the file descriptor if exists
+            PackageTools.jarLoaderMap.remove(jarPath)?.close()
+
+            // clear all loaded sources
+            sources.forEach { GetHttpSource.invalidateSourceCache(it) }
+
             File(jarPath).delete()
         }
     }
@@ -233,12 +267,13 @@ object Extension {
         return installExtension(pkgName)
     }
 
-    suspend fun getExtensionIcon(apkName: String): Pair<InputStream, String> {
+    suspend fun getExtensionIcon(apkName: String, useCache: Boolean): Pair<InputStream, String> {
-        val iconUrl = transaction { ExtensionTable.select { ExtensionTable.apkName eq apkName }.first() }[ExtensionTable.iconUrl]
+        val iconUrl = if (apkName == "localSource") ""
+        else transaction { ExtensionTable.select { ExtensionTable.apkName eq apkName }.first() }[ExtensionTable.iconUrl]
 
         val saveDir = "${applicationDirs.extensionsRoot}/icon"
 
-        return getCachedImageResponse(saveDir, apkName) {
+        return getImageResponse(saveDir, apkName, useCache) {
             network.client.newCall(
                 GET(iconUrl)
             ).await()

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/ExtensionsList.kt

@@ -7,6 +7,7 @@ package suwayomi.tachidesk.manga.impl.extension
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
 
+import eu.kanade.tachiyomi.source.local.LocalSource
 import mu.KotlinLogging
 import org.jetbrains.exposed.sql.deleteWhere
 import org.jetbrains.exposed.sql.insert
@@ -46,7 +47,7 @@ object ExtensionsList {
     }
 
     fun extensionTableAsDataClass() = transaction {
-        ExtensionTable.selectAll().map {
+        ExtensionTable.selectAll().filter { it[ExtensionTable.name] != LocalSource.EXTENSION_NAME }.map {
             ExtensionDataClass(
                 it[ExtensionTable.apkName],
                 getExtensionIconUrl(it[ExtensionTable.apkName]),

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/BytecodeEditor.kt

@@ -15,15 +15,10 @@ import org.objectweb.asm.FieldVisitor
 import org.objectweb.asm.Handle
 import org.objectweb.asm.MethodVisitor
 import org.objectweb.asm.Opcodes
-import org.objectweb.asm.tree.ClassNode
+import java.nio.file.FileSystems
-import suwayomi.tachidesk.manga.impl.util.storage.use
+import java.nio.file.Files
-import java.io.File
+import java.nio.file.Path
-import java.io.IOException
+import kotlin.streams.asSequence
-import java.util.jar.JarEntry
-import java.util.jar.JarFile
-import java.util.jar.JarOutputStream
-import java.util.zip.ZipEntry
-import java.util.zip.ZipInputStream
 
 object BytecodeEditor {
     private val logger = KotlinLogging.logger {}
@@ -33,77 +28,52 @@ object BytecodeEditor {
      *
      * @param jarFile The JarFile to replace class references in
      */
-    fun fixAndroidClasses(jarFile: File) {
+    fun fixAndroidClasses(jarFile: Path) {
-        val nodes = loadClasses(jarFile)
+        FileSystems.newFileSystem(jarFile, null as ClassLoader?)?.use {
-            .mapValues { (className, classFileBuffer) ->
+            Files.walk(it.getPath("/")).asSequence()
-                logger.trace { "Processing class $className" }
+                .filterNotNull()
-                transform(classFileBuffer)
+                .filterNot(Files::isDirectory)
-            } + loadNonClasses(jarFile)
+                .mapNotNull(::getClassBytes)
-
+                .map(::transform)
-        saveAsJar(nodes, jarFile)
+                .forEach(::write)
-    }
-
-    /**
-     * Load all classes inside the [jar] [File]
-     *
-     * @param jar The JarFile to load classes from
-     *
-     * @return [Map] with class names and [ByteArray]s of bytecode
-     */
-    private fun loadClasses(jar: File): Map<String, ByteArray> {
-        return JarFile(jar).use { jarFile ->
-            jarFile.entries()
-                .asSequence()
-                .mapNotNull {
-                    readJar(jarFile, it)
-                }
-                .toMap()
         }
     }
 
     /**
-     * Get class file in [jar] for [entry]
+     * Get class bytes from a [Path]
      *
-     * @param jar The jar to get the class from
+     * @param path The path entry to get the class bytes from
-     * @param entry The entry in the jar
      *
-     * @return [Pair] of the class name plus the class [ByteArray], or null if it's not a valid class
+     * @return [Pair] of the [Path] plus the class [ByteArray], or null if it's not a valid class
      */
-    private fun readJar(jar: JarFile, entry: JarEntry): Pair<String, ByteArray>? {
+    private fun getClassBytes(path: Path): Pair<Path, ByteArray>? {
         return try {
-            jar.getInputStream(entry).use { stream ->
+            if (path.toString().endsWith(".class")) {
-                if (entry.name.endsWith(".class")) {
+                val bytes = Files.readAllBytes(path)
-                    val bytes = stream.readBytes()
+                if (bytes.size < 4) {
-                    if (bytes.size < 4) {
+                    // Invalid class size
-                        // Invalid class size
+                    return null
-                        return@use null
+                }
-                    }
+                val cafebabe = String.format(
-                    val cafebabe = String.format(
+                    "%02X%02X%02X%02X",
-                        "%02X%02X%02X%02X",
+                    bytes[0],
-                        bytes[0],
+                    bytes[1],
-                        bytes[1],
+                    bytes[2],
-                        bytes[2],
+                    bytes[3]
-                        bytes[3]
+                )
-                    )
+                if (cafebabe.lowercase() != "cafebabe") {
-                    if (cafebabe.lowercase() != "cafebabe") {
+                    // Corrupted class
-                        // Corrupted class
+                    return null
-                        return@use null
+                }
-                    }
 
-                    getNode(bytes).name to bytes
+                path to bytes
-                } else null
+            } else null
-            }
+        } catch (e: Exception) {
-        } catch (e: IOException) {
+            logger.error(e) { "Error loading class from Path: $path" }
-            logger.error(e) { "Error loading jar file" }
             null
         }
     }
 
-    private fun getNode(bytes: ByteArray): ClassNode {
-        val cr = ClassReader(bytes)
-        return ClassNode().also { cr.accept(it, ClassReader.EXPAND_FRAMES) }
-    }
-
     /**
      * The path where replacement classes will reside
      */
@@ -153,9 +123,9 @@ object BytecodeEditor {
      *
      * @return [ByteArray] with modified bytecode
      */
-    private fun transform(classfileBuffer: ByteArray): ByteArray {
+    private fun transform(pair: Pair<Path, ByteArray>): Pair<Path, ByteArray> {
         // Read the class and prepare to modify it
-        val cr = ClassReader(classfileBuffer)
+        val cr = ClassReader(pair.second)
         val cw = ClassWriter(cr, 0)
         // Modify the class
         cr.accept(
@@ -277,51 +247,10 @@ object BytecodeEditor {
             },
             0
         )
-        return cw.toByteArray()
+        return pair.first to cw.toByteArray()
     }
 
-    /**
+    private fun write(pair: Pair<Path, ByteArray>) {
-     * Load non-class files from the jar, such as icons and the manifest
+        Files.write(pair.first, pair.second)
-     *
-     * @param [jarFile] The file to load resources from
-     *
-     * @return [Map] of resources
-     */
-    private fun loadNonClasses(jarFile: File): Map<String, ByteArray> {
-        val entries = mutableMapOf<String, ByteArray>()
-        ZipInputStream(jarFile.inputStream()).use { stream ->
-            var nextEntry: ZipEntry?
-            while (stream.nextEntry.also { nextEntry = it } != null) {
-                nextEntry?.use(stream) { entry ->
-                    // If it ends with class or is a directory ignore it
-                    if (!entry.name.endsWith(".class") && !entry.isDirectory) {
-                        val bytes = stream.readBytes()
-                        entries[entry.name] = bytes
-                    }
-                }
-            }
-        }
-        return entries
-    }
-
-    /**
-     * Save jar with modified content
-     *
-     * @param outBytes [Map] of names and [ByteArray]s of content to save inside the jar
-     * @param file JarFile to save to
-     */
-    private fun saveAsJar(outBytes: Map<String, ByteArray>, file: File) {
-        JarOutputStream(file.outputStream()).use { out ->
-            outBytes.forEach { (entry, value) ->
-                // Append extension to class entries
-                out.putNextEntry(
-                    ZipEntry(
-                        entry + if (entry.contains(".")) "" else ".class"
-                    )
-                )
-                out.write(value)
-                out.closeEntry()
-            }
-        }
     }
 }