bump version

.github/workflows/publish.yml

@@ -88,5 +88,6 @@ jobs:
 
       - name: Run Docker build workflow
         run: |
+          sleep 10 # sleep a bit to make sure the release is actually inside github db
           curl -X POST -H "Accept: application/vnd.github.v3+json" -H "Authorization: token ${{ secrets.DEPLOY_PREVIEW_TOKEN }}" -d '{"ref":"main", "inputs":{"tachidesk_release_type": "stable"}}' https://api.github.com/repos/suwayomi/docker-tachidesk/actions/workflows/build_container_images.yml/dispatches
           

AndroidCompat/Config/src/main/java/xyz/nulldev/ts/config/ApplicationRootDir.kt

@@ -9,11 +9,12 @@ package xyz.nulldev.ts.config
 
 import net.harawata.appdirs.AppDirsFactory
 
+const val CONFIG_PREFIX = "suwayomi.tachidesk.config"
 
 val ApplicationRootDir: String
     get(): String {
         return System.getProperty(
-                "suwayomi.tachidesk.server.rootDir",
+                "$CONFIG_PREFIX.server.rootDir",
                 AppDirsFactory.getInstance().getUserDataDir("Tachidesk", null, null)
         )
     }

AndroidCompat/Config/src/main/java/xyz/nulldev/ts/config/ConfigManager.kt

@@ -48,7 +48,7 @@ open class ConfigManager {
         val baseConfig =
                 ConfigFactory.parseMap(
                         mapOf(
-                                "ts.server.rootDir" to ApplicationRootDir
+                                "androidcompat.rootDir" to "$ApplicationRootDir/android-compat" // override AndroidCompat's rootDir
                         )
                 )
 

AndroidCompat/Config/src/main/java/xyz/nulldev/ts/config/ConfigModule.kt

@@ -14,26 +14,31 @@ import kotlin.reflect.KProperty
 /**
  * Abstract config module.
  */
-abstract class ConfigModule(config: Config, moduleName: String = "") {
-    val overridableWithSysProperty = SystemPropertyOverrideDelegate(config, moduleName)
+@Suppress("UNUSED_PARAMETER")
+abstract class ConfigModule(config: Config)
+
+/**
+ * Abstract jvm-commandline-argument-overridable config module.
+ */
+abstract class SystemPropertyOverridableConfigModule(config: Config, moduleName: String): ConfigModule(config) {
+    val overridableConfig = SystemPropertyOverrideDelegate(config, moduleName)
 }
 
+/** Defines a config property that is overridable with jvm `-D` commandline arguments prefixed with [CONFIG_PREFIX] */
 class SystemPropertyOverrideDelegate(val config: Config, val moduleName: String) {
     inline operator fun <R, reified T> getValue(thisRef: R, property: KProperty<*>): T {
         val configValue: T = config.getValue(thisRef, property)
 
         val combined = System.getProperty(
-            "suwayomi.tachidesk.config.$moduleName.${property.name}",
+            "$CONFIG_PREFIX.$moduleName.${property.name}",
             configValue.toString()
         )
 
-        val asT = when(T::class.simpleName) {
+        return when(T::class.simpleName) {
             "Int" -> combined.toInt()
             "Boolean" -> combined.toBoolean()
             // add more types as needed
-            else -> combined
-        }
-
-        return asT as T
+            else -> combined // covers String
+        } as T
     }
 }

AndroidCompat/src/main/java/xyz/nulldev/androidcompat/config/ApplicationInfoConfigModule.kt

@@ -1,6 +1,7 @@
 package xyz.nulldev.androidcompat.config
 
 import com.typesafe.config.Config
+import io.github.config4k.getValue
 import xyz.nulldev.ts.config.ConfigModule
 
 /**
@@ -8,8 +9,8 @@ import xyz.nulldev.ts.config.ConfigModule
  */
 
 class ApplicationInfoConfigModule(config: Config) : ConfigModule(config) {
-    val packageName = config.getString("packageName")!!
-    val debug = config.getBoolean("debug")
+    val packageName: String by config
+    val debug: Boolean by config
 
     companion object {
         fun register(config: Config)

AndroidCompat/src/main/java/xyz/nulldev/androidcompat/config/FilesConfigModule.kt

@@ -1,6 +1,7 @@
 package xyz.nulldev.androidcompat.config
 
 import com.typesafe.config.Config
+import io.github.config4k.getValue
 import xyz.nulldev.ts.config.ConfigModule
 
 /**
@@ -8,23 +9,23 @@ import xyz.nulldev.ts.config.ConfigModule
  */
 
 class FilesConfigModule(config: Config) : ConfigModule(config) {
-    val dataDir = config.getString("dataDir")!!
-    val filesDir = config.getString("filesDir")!!
-    val noBackupFilesDir = config.getString("noBackupFilesDir")!!
-    val externalFilesDirs: MutableList<String> = config.getStringList("externalFilesDirs")!!
-    val obbDirs: MutableList<String> = config.getStringList("obbDirs")!!
-    val cacheDir = config.getString("cacheDir")!!
-    val codeCacheDir = config.getString("codeCacheDir")!!
-    val externalCacheDirs: MutableList<String> = config.getStringList("externalCacheDirs")!!
-    val externalMediaDirs: MutableList<String> = config.getStringList("externalMediaDirs")!!
-    val rootDir = config.getString("rootDir")!!
-    val externalStorageDir = config.getString("externalStorageDir")!!
-    val downloadCacheDir = config.getString("downloadCacheDir")!!
-    val databasesDir = config.getString("databasesDir")!!
+    val dataDir:String by config
+    val filesDir:String by config
+    val noBackupFilesDir:String by config
+    val externalFilesDirs: MutableList<String> by config
+    val obbDirs: MutableList<String> by config
+    val cacheDir:String by config
+    val codeCacheDir:String by config
+    val externalCacheDirs: MutableList<String> by config
+    val externalMediaDirs: MutableList<String> by config
+    val rootDir:String by config
+    val externalStorageDir:String by config
+    val downloadCacheDir:String by config
+    val databasesDir:String by config
 
-    val prefsDir = config.getString("prefsDir")!!
+    val prefsDir:String by config
 
-    val packageDir = config.getString("packageDir")!!
+    val packageDir:String by config
 
     companion object {
         fun register(config: Config)

AndroidCompat/src/main/java/xyz/nulldev/androidcompat/config/SystemConfigModule.kt

@@ -2,9 +2,10 @@ package xyz.nulldev.androidcompat.config
 
 import com.typesafe.config.Config
 import xyz.nulldev.ts.config.ConfigModule
+import io.github.config4k.getValue
 
 class SystemConfigModule(val config: Config) : ConfigModule(config) {
-    val isDebuggable = config.getBoolean("isDebuggable")
+    val isDebuggable: Boolean by config
 
     val propertyPrefix = "properties."
 

AndroidCompat/src/main/resources/compat-reference.conf

@@ -1,36 +1,12 @@
+# AndroidComapt Root dir
+androidcompat.rootDir = androidcompat-root
 
-# Allow/disallow preference changes (useful for demos)
-ts.server.allowConfigChanges = true
-
-# Enable the WebUI? Note: The API and multi-user sync server ui will remain available even if the WebUI is disabled
-ts.server.enableWebUi = true
-
-# 'true' to use the old, buggy/memory-leaking WebUI
-ts.server.useOldWebUi = false
-
-# 'true' to pretty print all JSON API responses
-ts.server.prettyPrintApi = false
-
-# List of blacklisted/whitelisted API endpoints/operation IDs
-ts.server.disabledApiEndpoints = []
-ts.server.enabledApiEndpoints = []
-
-# Message to print in the console when the API has finished booting
-ts.server.httpInitializedPrintMessage = ""
-
-# Use external folder for static files
-ts.server.useExternalStaticFiles = false
-ts.server.externalStaticFilesFolder = ""
-
-# Root storage dir
-ts.server.rootDir = tachiserver-data
-# Dir to store JVM patches
-ts.server.patchesDir = ${ts.server.rootDir}/patches
+####################### `android.files` (FilesConfigModule) #######################
 
 # Storage dir for the emulated Android app
-android.files.rootDir = ${ts.server.rootDir}/android-compat/appdata
+android.files.rootDir = ${androidcompat.rootDir}/appdata
 # External storage dir for the emulated Android app's
-android.files.externalStorageDir = ${ts.server.rootDir}/android-compat/extappdata
+android.files.externalStorageDir = ${androidcompat.rootDir}/extappdata
 
 # Internal Android directories
 android.files.dataDir = ${android.files.rootDir}/data
@@ -48,37 +24,16 @@ android.files.externalCacheDirs = [${android.files.externalStorageDir}/cache]
 android.files.externalMediaDirs = [${android.files.externalStorageDir}/media]
 android.files.downloadCacheDir = ${android.files.externalStorageDir}/downloadCache
 
-android.files.packageDir = ${ts.server.rootDir}/android-compat/packages
+android.files.packageDir = ${androidcompat.rootDir}/android-compat/packages
+
+####################### `android.app` (ApplicationInfoConfigModule) #######################
 
 # Emulated Android app package name
 android.app.packageName = eu.kanade.tachiyomi
 # Debug mode for the emulated Android app
 android.app.debug = true
 
+####################### `android.system` (SystemConfigModule) #######################
+
 # Whether or not the emulated Android system is debuggable
 android.system.isDebuggable = true
-
-# Is the multi-user sync server enabled? Does not affect the single-user sync server included in the API.
-ts.syncd.enable = false
-
-# The URL of this server (displayed in the sync server web ui)
-ts.syncd.baseUrl = "http://example.com"
-
-# 'true' to disable the API and only enable the multi-user sync server
-ts.syncd.syncOnlyMode = false
-
-# The root directory to store synchronized data
-ts.syncd.rootDir = ${ts.server.rootDir}/sync/accounts
-
-# Location to store config files for the sandbox
-ts.syncd.sandboxedConfig = ${ts.server.rootDir}/sync/sandboxed_config.config
-
-# Recaptcha stuff for signup/login
-ts.syncd.recaptcha.siteKey = ""
-ts.syncd.recaptcha.secret = ""
-
-# Sync server display name
-ts.syncd.name = "Tachiyomi sync server"
-
-# Header used to forward the IP to the multi-user sync server if the server is behind a reverse proxy
-ts.syncd.ipHeader = ""

README.md

@@ -31,7 +31,7 @@ Here is a list of current features:
     - A library to save your mangas and categories to put them into
     - Searching and browsing installed sources
     - Ability to download Manga for offline read
-    - Backup and restore support powered by Tachiyomi Legacy Backups
+    - Backup and restore support powered by Tachiyomi Backups
 - From Aniyomi
     - Installing and executing Aniyomi's Extensions
     - Searching and browsing installed sources.
@@ -88,9 +88,9 @@ See [CONTRIBUTING.md](./CONTRIBUTING.md).
 ## Credit
 This project is a spiritual successor of [TachiWeb-Server](https://github.com/Tachiweb/TachiWeb-server), Many of the ideas and the groundwork adopted in this project comes from TachiWeb.
 
-The `AndroidCompat` module was originally developed by [@null-dev](https://github.com/null-dev) for [TachiWeb-Server](https://github.com/Tachiweb/TachiWeb-server) and is licensed under `Apache License Version 2.0`.
+The `AndroidCompat` module was originally developed by [@null-dev](https://github.com/null-dev) for [TachiWeb-Server](https://github.com/Tachiweb/TachiWeb-server) and is licensed under `Apache License Version 2.0` and `Copyright 2019 Andy Bao and contributors`.
 
-Parts of [tachiyomi](https://github.com/tachiyomiorg/tachiyomi) is adopted into this codebase, also licensed under `Apache License Version 2.0`.
+Parts of [tachiyomi](https://github.com/tachiyomiorg/tachiyomi) is adopted into this codebase, also licensed under `Apache License Version 2.0` and `Copyright 2015 Javier Tomás`.
 
 You can obtain a copy of `Apache License Version 2.0` from  http://www.apache.org/licenses/LICENSE-2.0
 

build.gradle.kts

@@ -79,9 +79,10 @@ configure(projects) {
         // to get application content root
         implementation("net.harawata:appdirs:1.2.1")
 
-        // dex2jar: https://github.com/DexPatcher/dex2jar/releases/tag/v2.1-20190905-lanchon
-        // note: watch https://github.com/ThexXTURBOXx/dex2jar for future developments
-        implementation("com.github.DexPatcher.dex2jar:dex-tools:v2.1-20190905-lanchon")
+        // dex2jar
+        val dex2jarVersion = "v21"
+        implementation("com.github.ThexXTURBOXx.dex2jar:dex-translator:$dex2jarVersion")
+        implementation("com.github.ThexXTURBOXx.dex2jar:dex-tools:$dex2jarVersion")
 
         // APK parser
         implementation("net.dongliu:apk-parser:2.6.10")

buildSrc/src/main/kotlin/Constants.kt

@@ -12,9 +12,9 @@ const val kotlinVersion = "1.5.21"
 const val MainClass = "suwayomi.tachidesk.MainKt"
 
 // should be bumped with each stable release
-val tachideskVersion = System.getenv("ProductVersion") ?: "v0.4.7"
+val tachideskVersion = System.getenv("ProductVersion") ?: "v0.4.8"
 
-val webUIRevisionTag = System.getenv("WebUIRevision") ?: "r36"
+val webUIRevisionTag = System.getenv("WebUIRevision") ?: "r41"
 
 // counts commits on the the master branch
 val tachideskRevision = runCatching {

server/build.gradle.kts

@@ -9,7 +9,7 @@ plugins {
     application
     kotlin("plugin.serialization")
     id("com.github.johnrengelman.shadow") version "7.0.0"
-    id("org.jmailen.kotlinter") version "3.4.3"
+    id("org.jmailen.kotlinter") version "3.5.0"
     id("com.github.gmazzo.buildconfig") version "3.0.2"
 }
 
@@ -46,7 +46,7 @@ dependencies {
     implementation("com.h2database:h2:1.4.200")
 
     // Exposed Migrations
-    val exposedMigrationsVersion = "3.1.0"
+    val exposedMigrationsVersion = "3.1.1"
     implementation("com.github.Suwayomi:exposed-migrations:$exposedMigrationsVersion")
 
     // tray icon
@@ -58,17 +58,20 @@ dependencies {
     implementation("com.github.inorichi.injekt:injekt-core:65b0440")
     implementation("com.squareup.okhttp3:okhttp:4.9.1")
     implementation("io.reactivex:rxjava:1.3.8")
-    implementation("org.jsoup:jsoup:1.13.1")
-    implementation("com.google.code.gson:gson:2.8.6")
+    implementation("org.jsoup:jsoup:1.14.1")
+    implementation("com.google.code.gson:gson:2.8.7")
     implementation("com.github.salomonbrys.kotson:kotson:2.5.0")
 
 
-    // asm for fixing SimpleDateFormat (must match Dex2Jar version)
-    implementation("org.ow2.asm:asm-debug-all:5.0.3")
+    // asm for ByteCodeEditor(fixing SimpleDateFormat) (must match Dex2Jar version)
+    implementation("org.ow2.asm:asm:9.2")
 
     // extracting zip files
     implementation("net.lingala.zip4j:zip4j:2.9.0")
 
+    // CloudflareInterceptor
+    implementation("net.sourceforge.htmlunit:htmlunit:2.52.0")
+
     // Source models and interfaces from Tachiyomi 1.x
     // using source class from tachiyomi commit 9493577de27c40ce8b2b6122cc447d025e34c477 to not depend on tachiyomi.sourceapi
 //    implementation("tachiyomi.sourceapi:source-api:1.1")

server/src/main/kotlin/eu/kanade/tachiyomi/network/CloudflareInterceptor.kt

@@ -1,177 +0,0 @@
-package eu.kanade.tachiyomi.network
-
-/*
- * Copyright (C) Contributors to the Suwayomi project
- *
- * This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
-
-// import android.annotation.SuppressLint
-// import android.content.Context
-// import android.os.Build
-// import android.os.Handler
-// import android.os.Looper
-// import android.webkit.WebSettings
-// import android.webkit.WebView
-// import android.widget.Toast
-// import eu.kanade.tachiyomi.R
-// import eu.kanade.tachiyomi.util.lang.launchUI
-// import eu.kanade.tachiyomi.util.system.WebViewClientCompat
-// import eu.kanade.tachiyomi.util.system.WebViewUtil
-// import eu.kanade.tachiyomi.util.system.isOutdated
-// import eu.kanade.tachiyomi.util.system.setDefaultSettings
-// import eu.kanade.tachiyomi.util.system.toast
-import okhttp3.Interceptor
-import okhttp3.Response
-// import uy.kohesive.injekt.injectLazy
-
-class CloudflareInterceptor() : Interceptor {
-
-//    private val handler = Handler(Looper.getMainLooper())
-
-//    private val networkHelper = NetworkHelper()
-
-    /**
-     * When this is called, it initializes the WebView if it wasn't already. We use this to avoid
-     * blocking the main thread too much. If used too often we could consider moving it to the
-     * Application class.
-     */
-//    private val initWebView by lazy {
-//        WebSettings.getDefaultUserAgent(context)
-//    }
-
-    @Synchronized
-    override fun intercept(chain: Interceptor.Chain): Response {
-        val originalRequest = chain.request()
-        return chain.proceed(originalRequest)
-
-//        if (!WebViewUtil.supportsWebView(context)) {
-//            launchUI {
-//                context.toast(R.string.information_webview_required, Toast.LENGTH_LONG)
-//            }
-//            return chain.proceed(originalRequest)
-//        }
-//
-//        initWebView
-//
-//        val response = chain.proceed(originalRequest)
-//
-//        // Check if Cloudflare anti-bot is on
-//        if (response.code != 503 || response.header("Server") !in SERVER_CHECK) {
-//            return response
-//        }
-//
-//        try {
-//            response.close()
-//            networkHelper.cookieManager.remove(originalRequest.url, COOKIE_NAMES, 0)
-//            val oldCookie = networkHelper.cookieManager.get(originalRequest.url)
-//                .firstOrNull { it.name == "cf_clearance" }
-//            resolveWithWebView(originalRequest, oldCookie)
-//
-//            return chain.proceed(originalRequest)
-//        } catch (e: Exception) {
-//            // Because OkHttp's enqueue only handles IOExceptions, wrap the exception so that
-//            // we don't crash the entire app
-//            throw IOException(e)
-//        }
-    }
-//
-// //    @SuppressLint("SetJavaScriptEnabled")
-//    private fun resolveWithWebView(request: Request, oldCookie: Cookie?) {
-//        // We need to lock this thread until the WebView finds the challenge solution url, because
-//        // OkHttp doesn't support asynchronous interceptors.
-//        val latch = CountDownLatch(1)
-//
-//        var webView: WebView? = null
-//
-//        var challengeFound = false
-//        var cloudflareBypassed = false
-//        var isWebViewOutdated = false
-//
-//        val origRequestUrl = request.url.toString()
-//        val headers = request.headers.toMultimap().mapValues { it.value.getOrNull(0) ?: "" }.toMutableMap()
-//        headers["X-Requested-With"] = WebViewUtil.REQUESTED_WITH
-//
-//        handler.post {
-//            val webview = WebView(context)
-//            webView = webview
-//            webview.setDefaultSettings()
-//
-//            // Avoid sending empty User-Agent, Chromium WebView will reset to default if empty
-//            webview.settings.userAgentString = request.header("User-Agent")
-//                ?: HttpSource.DEFAULT_USERAGENT
-//
-//            webview.webViewClient = object : WebViewClientCompat() {
-//                override fun onPageFinished(view: WebView, url: String) {
-//                    fun isCloudFlareBypassed(): Boolean {
-//                        return networkHelper.cookieManager.get(origRequestUrl.toHttpUrl())
-//                            .firstOrNull { it.name == "cf_clearance" }
-//                            .let { it != null && it != oldCookie }
-//                    }
-//
-//                    if (isCloudFlareBypassed()) {
-//                        cloudflareBypassed = true
-//                        latch.countDown()
-//                    }
-//
-//                    // HTTP error codes are only received since M
-//                    if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M &&
-//                        url == origRequestUrl && !challengeFound
-//                    ) {
-//                        // The first request didn't return the challenge, abort.
-//                        latch.countDown()
-//                    }
-//                }
-//
-//                override fun onReceivedErrorCompat(
-//                    view: WebView,
-//                    errorCode: Int,
-//                    description: String?,
-//                    failingUrl: String,
-//                    isMainFrame: Boolean
-//                ) {
-//                    if (isMainFrame) {
-//                        if (errorCode == 503) {
-//                            // Found the Cloudflare challenge page.
-//                            challengeFound = true
-//                        } else {
-//                            // Unlock thread, the challenge wasn't found.
-//                            latch.countDown()
-//                        }
-//                    }
-//                }
-//            }
-//
-//            webView?.loadUrl(origRequestUrl, headers)
-//        }
-//
-//        // Wait a reasonable amount of time to retrieve the solution. The minimum should be
-//        // around 4 seconds but it can take more due to slow networks or server issues.
-//        latch.await(12, TimeUnit.SECONDS)
-//
-//        handler.post {
-//            if (!cloudflareBypassed) {
-//                isWebViewOutdated = webView?.isOutdated() == true
-//            }
-//
-//            webView?.stopLoading()
-//            webView?.destroy()
-//        }
-//
-//        // Throw exception if we failed to bypass Cloudflare
-//        if (!cloudflareBypassed) {
-//            // Prompt user to update WebView if it seems too outdated
-//            if (isWebViewOutdated) {
-//                context.toast(R.string.information_webview_outdated, Toast.LENGTH_LONG)
-//            }
-//
-//            throw Exception(context.getString(R.string.information_cloudflare_bypass_failure))
-//        }
-//    }
-//
-//    companion object {
-//        private val SERVER_CHECK = arrayOf("cloudflare-nginx", "cloudflare")
-//        private val COOKIE_NAMES = listOf("__cfduid", "cf_clearance")
-//    }
-}

server/src/main/kotlin/eu/kanade/tachiyomi/network/NetworkHelper.kt

@@ -10,12 +10,16 @@ package eu.kanade.tachiyomi.network
 // import android.content.Context
 // import eu.kanade.tachiyomi.BuildConfig
 // import eu.kanade.tachiyomi.data.preference.PreferencesHelper
-import android.content.Context
 // import okhttp3.HttpUrl.Companion.toHttpUrl
-import okhttp3.OkHttpClient
 // import okhttp3.dnsoverhttps.DnsOverHttps
 // import okhttp3.logging.HttpLoggingInterceptor
 // import uy.kohesive.injekt.injectLazy
+import android.content.Context
+import eu.kanade.tachiyomi.network.interceptor.CloudflareInterceptor
+import eu.kanade.tachiyomi.network.interceptor.UserAgentInterceptor
+import okhttp3.OkHttpClient
+import okhttp3.logging.HttpLoggingInterceptor
+import suwayomi.tachidesk.server.serverConfig
 import java.util.concurrent.TimeUnit
 
 @Suppress("UNUSED_PARAMETER")
@@ -25,55 +29,44 @@ class NetworkHelper(context: Context) {
 
 //    private val cacheDir = File(context.cacheDir, "network_cache")
 
-    private val cacheSize = 5L * 1024 * 1024 // 5 MiB
+//    private val cacheSize = 5L * 1024 * 1024 // 5 MiB
 
-    val cookieManager = MemoryCookieJar()
+    val cookieManager = PersistentCookieJar(context)
 
-    val client by lazy {
-        val builder = OkHttpClient.Builder()
-            .cookieJar(cookieManager)
-//            .cache(Cache(cacheDir, cacheSize))
-            .connectTimeout(30, TimeUnit.SECONDS)
-            .readTimeout(5, TimeUnit.MINUTES)
-            .writeTimeout(5, TimeUnit.MINUTES)
-//            .dispatcher(Dispatcher(Executors.newFixedThreadPool(1)))
+    private val baseClientBuilder: OkHttpClient.Builder
+        get() {
+            val builder = OkHttpClient.Builder()
+                .cookieJar(cookieManager)
+                .connectTimeout(30, TimeUnit.SECONDS)
+                .readTimeout(30, TimeUnit.SECONDS)
+                .addInterceptor(UserAgentInterceptor())
 
-//            .addInterceptor(UserAgentInterceptor())
+            if (serverConfig.debugLogsEnabled) {
+                val httpLoggingInterceptor = HttpLoggingInterceptor().apply {
+                    level = HttpLoggingInterceptor.Level.BASIC
+                }
+                builder.addInterceptor(httpLoggingInterceptor)
+            }
 
-//        if (BuildConfig.DEBUG) {
-//            val httpLoggingInterceptor = HttpLoggingInterceptor().apply {
-//                level = HttpLoggingInterceptor.Level.HEADERS
+//            when (preferences.dohProvider()) {
+//                PREF_DOH_CLOUDFLARE -> builder.dohCloudflare()
+//                PREF_DOH_GOOGLE -> builder.dohGoogle()
 //            }
-//            builder.addInterceptor(httpLoggingInterceptor)
-//        }
 
-//        if (preferences.enableDoh()) {
-//            builder.dns(
-//                DnsOverHttps.Builder().client(builder.build())
-//                    .url("https://cloudflare-dns.com/dns-query".toHttpUrl())
-//                    .bootstrapDnsHosts(
-//                        listOf(
-//                            InetAddress.getByName("162.159.36.1"),
-//                            InetAddress.getByName("162.159.46.1"),
-//                            InetAddress.getByName("1.1.1.1"),
-//                            InetAddress.getByName("1.0.0.1"),
-//                            InetAddress.getByName("162.159.132.53"),
-//                            InetAddress.getByName("2606:4700:4700::1111"),
-//                            InetAddress.getByName("2606:4700:4700::1001"),
-//                            InetAddress.getByName("2606:4700:4700::0064"),
-//                            InetAddress.getByName("2606:4700:4700::6400")
-//                        )
-//                    )
-//                    .build()
-//            )
-//        }
+            return builder
+        }
 
-        builder.build()
-    }
+//    val client by lazy { baseClientBuilder.cache(Cache(cacheDir, cacheSize)).build() }
+    val client by lazy { baseClientBuilder.build() }
 
     val cloudflareClient by lazy {
         client.newBuilder()
             .addInterceptor(CloudflareInterceptor())
             .build()
     }
+
+    // Tachidesk -->
+    val cookies: PersistentCookieStore
+        get() = cookieManager.store
+    // Tachidesk <--
 }

server/src/main/kotlin/eu/kanade/tachiyomi/network/PersistentCookieJar.kt

@@ -0,0 +1,20 @@
+package eu.kanade.tachiyomi.network
+
+import android.content.Context
+import okhttp3.Cookie
+import okhttp3.CookieJar
+import okhttp3.HttpUrl
+
+// from TachiWeb-Server
+class PersistentCookieJar(context: Context) : CookieJar {
+
+    val store = PersistentCookieStore(context)
+
+    override fun saveFromResponse(url: HttpUrl, cookies: List<Cookie>) {
+        store.addAll(url, cookies)
+    }
+
+    override fun loadForRequest(url: HttpUrl): List<Cookie> {
+        return store.get(url)
+    }
+}

server/src/main/kotlin/eu/kanade/tachiyomi/network/PersistentCookieStore.kt

@@ -0,0 +1,79 @@
+package eu.kanade.tachiyomi.network
+
+import android.content.Context
+import okhttp3.Cookie
+import okhttp3.HttpUrl
+import okhttp3.HttpUrl.Companion.toHttpUrlOrNull
+import java.net.URI
+import java.util.concurrent.ConcurrentHashMap
+
+// from TachiWeb-Server
+class PersistentCookieStore(context: Context) {
+
+    private val cookieMap = ConcurrentHashMap<String, List<Cookie>>()
+    private val prefs = context.getSharedPreferences("cookie_store", Context.MODE_PRIVATE)
+
+    init {
+        for ((key, value) in prefs.all) {
+            @Suppress("UNCHECKED_CAST")
+            val cookies = value as? Set<String>
+            if (cookies != null) {
+                try {
+                    val url = "http://$key".toHttpUrlOrNull() ?: continue
+                    val nonExpiredCookies = cookies.mapNotNull { Cookie.parse(url, it) }
+                        .filter { !it.hasExpired() }
+                    cookieMap.put(key, nonExpiredCookies)
+                } catch (e: Exception) {
+                    // Ignore
+                }
+            }
+        }
+    }
+
+    @Synchronized
+    fun addAll(url: HttpUrl, cookies: List<Cookie>) {
+        val key = url.toUri().host
+
+        // Append or replace the cookies for this domain.
+        val cookiesForDomain = cookieMap[key].orEmpty().toMutableList()
+        for (cookie in cookies) {
+            // Find a cookie with the same name. Replace it if found, otherwise add a new one.
+            val pos = cookiesForDomain.indexOfFirst { it.name == cookie.name }
+            if (pos == -1) {
+                cookiesForDomain.add(cookie)
+            } else {
+                cookiesForDomain[pos] = cookie
+            }
+        }
+        cookieMap.put(key, cookiesForDomain)
+
+        // Get cookies to be stored in disk
+        val newValues = cookiesForDomain.asSequence()
+            .filter { it.persistent && !it.hasExpired() }
+            .map(Cookie::toString)
+            .toSet()
+
+        prefs.edit().putStringSet(key, newValues).apply()
+    }
+
+    @Synchronized
+    fun removeAll() {
+        prefs.edit().clear().apply()
+        cookieMap.clear()
+    }
+
+    fun remove(uri: URI) {
+        prefs.edit().remove(uri.host).apply()
+        cookieMap.remove(uri.host)
+    }
+
+    fun get(url: HttpUrl) = get(url.toUri().host)
+
+    fun get(uri: URI) = get(uri.host)
+
+    private fun get(url: String): List<Cookie> {
+        return cookieMap[url].orEmpty().filter { !it.hasExpired() }
+    }
+
+    private fun Cookie.hasExpired() = System.currentTimeMillis() >= expiresAt
+}

server/src/main/kotlin/eu/kanade/tachiyomi/network/interceptor/CloudflareInterceptor.kt

@@ -0,0 +1,110 @@
+package eu.kanade.tachiyomi.network.interceptor
+
+import com.gargoylesoftware.htmlunit.BrowserVersion
+import com.gargoylesoftware.htmlunit.WebClient
+import com.gargoylesoftware.htmlunit.html.HtmlPage
+import eu.kanade.tachiyomi.network.NetworkHelper
+import mu.KotlinLogging
+import okhttp3.Cookie
+import okhttp3.HttpUrl
+import okhttp3.Interceptor
+import okhttp3.Request
+import okhttp3.Response
+import uy.kohesive.injekt.injectLazy
+import java.io.IOException
+
+// from TachiWeb-Server
+class CloudflareInterceptor : Interceptor {
+    private val logger = KotlinLogging.logger {}
+
+    private val network: NetworkHelper by injectLazy()
+
+    @Synchronized
+    override fun intercept(chain: Interceptor.Chain): Response {
+        val originalRequest = chain.request()
+
+        logger.trace { "CloudflareInterceptor is being used." }
+
+        val response = chain.proceed(originalRequest)
+
+        // Check if Cloudflare anti-bot is on
+        if (response.code != 503 || response.header("Server") !in SERVER_CHECK) {
+            return response
+        }
+
+        logger.debug { "Cloudflare anti-bot is on, CloudflareInterceptor is kicking in..." }
+
+        return try {
+            response.close()
+            network.cookies.remove(originalRequest.url.toUri())
+
+            chain.proceed(resolveChallenge(response))
+        } catch (e: Exception) {
+            // Because OkHttp's enqueue only handles IOExceptions, wrap the exception so that
+            // we don't crash the entire app
+            throw IOException(e)
+        }
+    }
+
+    private fun resolveChallenge(response: Response): Request {
+        val browserVersion = BrowserVersion.BrowserVersionBuilder(BrowserVersion.BEST_SUPPORTED)
+            .setUserAgent(response.request.header("User-Agent") ?: BrowserVersion.BEST_SUPPORTED.userAgent)
+            .build()
+        val convertedCookies = WebClient(browserVersion).use { webClient ->
+            webClient.options.isThrowExceptionOnFailingStatusCode = false
+            webClient.options.isThrowExceptionOnScriptError = false
+            webClient.getPage<HtmlPage>(response.request.url.toString())
+            webClient.waitForBackgroundJavaScript(10000)
+            // Challenge solved, process cookies
+            webClient.cookieManager.cookies.filter {
+                // Only include Cloudflare cookies
+                it.name.startsWith("__cf") || it.name.startsWith("cf_")
+            }.map {
+                // Convert cookies -> OkHttp format
+                Cookie.Builder()
+                    .domain(it.domain.removePrefix("."))
+                    .expiresAt(it.expires?.time ?: Long.MAX_VALUE)
+                    .name(it.name)
+                    .path(it.path)
+                    .value(it.value).apply {
+                        if (it.isHttpOnly) httpOnly()
+                        if (it.isSecure) secure()
+                    }.build()
+            }
+        }
+
+        // Copy cookies to cookie store
+        convertedCookies.forEach {
+            network.cookies.addAll(
+                HttpUrl.Builder()
+                    .scheme("http")
+                    .host(it.domain)
+                    .build(),
+                listOf(it)
+            )
+        }
+        // Merge new and existing cookies for this request
+        // Find the cookies that we need to merge into this request
+        val convertedForThisRequest = convertedCookies.filter {
+            it.matches(response.request.url)
+        }
+        // Extract cookies from current request
+        val existingCookies = Cookie.parseAll(
+            response.request.url,
+            response.request.headers
+        )
+        // Filter out existing values of cookies that we are about to merge in
+        val filteredExisting = existingCookies.filter { existing ->
+            convertedForThisRequest.none { converted -> converted.name == existing.name }
+        }
+        val newCookies = filteredExisting + convertedForThisRequest
+        return response.request.newBuilder()
+            .header("Cookie", newCookies.map { it.toString() }.joinToString("; "))
+            .build()
+    }
+
+    companion object {
+        private val SERVER_CHECK = arrayOf("cloudflare-nginx", "cloudflare")
+        private val COOKIE_NAMES = listOf("cf_clearance")
+    }
+}

server/src/main/kotlin/eu/kanade/tachiyomi/network/interceptor/UserAgentInterceptor.kt

@@ -0,0 +1,22 @@
+package eu.kanade.tachiyomi.network.interceptor
+
+import eu.kanade.tachiyomi.source.online.HttpSource
+import okhttp3.Interceptor
+import okhttp3.Response
+
+class UserAgentInterceptor : Interceptor {
+    override fun intercept(chain: Interceptor.Chain): Response {
+        val originalRequest = chain.request()
+
+        return if (originalRequest.header("User-Agent").isNullOrEmpty()) {
+            val newRequest = originalRequest
+                .newBuilder()
+                .removeHeader("User-Agent")
+                .addHeader("User-Agent", HttpSource.DEFAULT_USER_AGENT)
+                .build()
+            chain.proceed(newRequest)
+        } else {
+            chain.proceed(originalRequest)
+        }
+    }
+}

server/src/main/kotlin/eu/kanade/tachiyomi/source/online/HttpSource.kt

@@ -75,7 +75,7 @@ abstract class HttpSource : CatalogueSource {
      * Headers builder for requests. Implementations can override this method for custom headers.
      */
     protected open fun headersBuilder() = Headers.Builder().apply {
-        add("User-Agent", DEFAULT_USERAGENT)
+        add("User-Agent", DEFAULT_USER_AGENT)
     }
 
     /**
@@ -372,6 +372,6 @@ abstract class HttpSource : CatalogueSource {
     override fun getFilterList() = FilterList()
 
     companion object {
-        const val DEFAULT_USERAGENT = "Mozilla/5.0 (Windows NT 6.3; WOW64)"
+        const val DEFAULT_USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.150 Safari/537.36 Edg/88.0.705.63"
     }
 }

server/src/main/kotlin/suwayomi/tachidesk/manga/MangaAPI.kt

@@ -42,10 +42,10 @@ object MangaAPI {
             get(":sourceId/preferences", SourceController::getPreferences)
             post(":sourceId/preferences", SourceController::setPreference)
 
-            post(":sourceId/filters", SourceController::filters) // TODO
+            get(":sourceId/filters", SourceController::filters)
 
             get(":sourceId/search/:searchTerm/:pageNum", SourceController::searchSingle)
-            get("search/:searchTerm/:pageNum", SourceController::searchSingle) // TODO
+//            get("search/:searchTerm/:pageNum", SourceController::searchGlobal)
         }
 
         path("manga") {
@@ -78,10 +78,7 @@ object MangaAPI {
             patch(":categoryId", CategoryController::categoryModify)
             delete(":categoryId", CategoryController::categoryDelete)
 
-            patch(
-                ":categoryId/reorder",
-                CategoryController::categoryReorder
-            ) // TODO: the underlying code doesn't need `:categoryId`, remove it
+            patch("reorder", CategoryController::categoryReorder)
         }
 
         path("backup") {

server/src/main/kotlin/suwayomi/tachidesk/manga/controller/SourceController.kt

@@ -12,7 +12,6 @@ import suwayomi.tachidesk.manga.impl.MangaList
 import suwayomi.tachidesk.manga.impl.Search
 import suwayomi.tachidesk.manga.impl.Source
 import suwayomi.tachidesk.manga.impl.Source.SourcePreferenceChange
-import suwayomi.tachidesk.server.JavalinSetup
 import suwayomi.tachidesk.server.JavalinSetup.future
 
 object SourceController {
@@ -63,9 +62,11 @@ object SourceController {
     }
 
     /** fetch filters of source with id `sourceId` */
-    fun filters(ctx: Context) { // TODO
+    fun filters(ctx: Context) {
         val sourceId = ctx.pathParam("sourceId").toLong()
-        ctx.json(Search.sourceFilters(sourceId))
+        val reset = ctx.queryParam("reset", "false").toBoolean()
+
+        ctx.json(Search.getInitialFilterList(sourceId, reset))
     }
 
     /** single source search */
@@ -73,7 +74,7 @@ object SourceController {
         val sourceId = ctx.pathParam("sourceId").toLong()
         val searchTerm = ctx.pathParam("searchTerm")
         val pageNum = ctx.pathParam("pageNum").toInt()
-        ctx.json(JavalinSetup.future { Search.sourceSearch(sourceId, searchTerm, pageNum) })
+        ctx.json(future { Search.sourceSearch(sourceId, searchTerm, pageNum) })
     }
 
     /** all source search */

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Category.kt

@@ -28,15 +28,17 @@ object Category {
      * The new category will be placed at the end of the list
      */
     fun createCategory(name: String) {
+        // creating a category named Default is illegal
         if (name.equals(DEFAULT_CATEGORY_NAME, ignoreCase = true)) return
 
         transaction {
-            val count = CategoryTable.selectAll().count()
-            if (CategoryTable.select { CategoryTable.name eq name }.firstOrNull() == null)
+            if (CategoryTable.select { CategoryTable.name eq name }.firstOrNull() == null) {
                 CategoryTable.insert {
                     it[CategoryTable.name] = name
-                    it[CategoryTable.order] = count.toInt() + 1
+                    it[CategoryTable.order] = Int.MAX_VALUE
                 }
+                normalizeCategories()
+            }
         }
     }
 
@@ -50,7 +52,7 @@ object Category {
     }
 
     /**
-     * Move the category from position `from` to `to`
+     * Move the category from order number `from` to `to`
      */
     fun reorderCategory(from: Int, to: Int) {
         transaction {
@@ -70,6 +72,19 @@ object Category {
                 removeMangaFromCategory(it[CategoryMangaTable.manga].value, categoryId)
             }
             CategoryTable.deleteWhere { CategoryTable.id eq categoryId }
+            normalizeCategories()
+        }
+    }
+
+    /** make sure category order numbers starts from 1 and is consecutive */
+    private fun normalizeCategories() {
+        transaction {
+            val categories = CategoryTable.selectAll().orderBy(CategoryTable.order to SortOrder.ASC)
+            categories.forEachIndexed { index, cat ->
+                CategoryTable.update({ CategoryTable.id eq cat[CategoryTable.id].value }) {
+                    it[CategoryTable.order] = index + 1
+                }
+            }
         }
     }
 

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Manga.kt

@@ -9,7 +9,6 @@ package suwayomi.tachidesk.manga.impl
 
 import eu.kanade.tachiyomi.network.GET
 import eu.kanade.tachiyomi.source.model.SManga
-import org.jetbrains.exposed.dao.id.EntityID
 import org.jetbrains.exposed.sql.and
 import org.jetbrains.exposed.sql.insert
 import org.jetbrains.exposed.sql.select
@@ -61,17 +60,17 @@ object Manga {
                 MangaStatus.valueOf(mangaEntry[MangaTable.status]).name,
                 mangaEntry[MangaTable.inLibrary],
                 getSource(mangaEntry[MangaTable.sourceReference]),
-                getMangaMetaMap(mangaEntry[MangaTable.id]),
+                getMangaMetaMap(mangaId),
+                mangaEntry[MangaTable.realUrl],
                 false
             )
         } else { // initialize manga
             val source = getHttpSource(mangaEntry[MangaTable.sourceReference])
-            val fetchedManga = source.fetchMangaDetails(
-                SManga.create().apply {
-                    url = mangaEntry[MangaTable.url]
-                    title = mangaEntry[MangaTable.title]
-                }
-            ).awaitSingle()
+            val sManga = SManga.create().apply {
+                url = mangaEntry[MangaTable.url]
+                title = mangaEntry[MangaTable.title]
+            }
+            val fetchedManga = source.fetchMangaDetails(sManga).awaitSingle()
 
             transaction {
                 MangaTable.update({ MangaTable.id eq mangaId }) {
@@ -85,6 +84,8 @@ object Manga {
                     it[MangaTable.status] = fetchedManga.status
                     if (fetchedManga.thumbnail_url != null && fetchedManga.thumbnail_url.orEmpty().isNotEmpty())
                         it[MangaTable.thumbnail_url] = fetchedManga.thumbnail_url
+
+                    it[MangaTable.realUrl] = try { source.mangaDetailsRequest(sManga).url.toString() } catch (e: Exception) { null }
                 }
             }
 
@@ -109,13 +110,14 @@ object Manga {
                 MangaStatus.valueOf(fetchedManga.status).name,
                 mangaEntry[MangaTable.inLibrary],
                 getSource(mangaEntry[MangaTable.sourceReference]),
-                getMangaMetaMap(mangaEntry[MangaTable.id]),
+                getMangaMetaMap(mangaId),
+                mangaEntry[MangaTable.realUrl],
                 true
             )
         }
     }
 
-    fun getMangaMetaMap(manga: EntityID<Int>): Map<String, String> {
+    fun getMangaMetaMap(manga: Int): Map<String, String> {
         return transaction {
             MangaMetaTable.select { MangaMetaTable.ref eq manga }
                 .associate { it[MangaMetaTable.key] to it[MangaMetaTable.value] }
@@ -126,7 +128,8 @@ object Manga {
         transaction {
             val manga = MangaMetaTable.select { (MangaTable.id eq mangaId) }
                 .first()[MangaTable.id]
-            val meta = transaction { MangaMetaTable.select { (MangaMetaTable.ref eq manga) and (MangaMetaTable.key eq key) } }.firstOrNull()
+            val meta =
+                transaction { MangaMetaTable.select { (MangaMetaTable.ref eq manga) and (MangaMetaTable.key eq key) } }.firstOrNull()
             if (meta == null) {
                 MangaMetaTable.insert {
                     it[MangaMetaTable.key] = key

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/MangaList.kt

@@ -41,7 +41,7 @@ object MangaList {
         val mangasPage = this
         val mangaList = transaction {
             return@transaction mangasPage.mangas.map { manga ->
-                val mangaEntry = MangaTable.select { MangaTable.url eq manga.url }.firstOrNull()
+                var mangaEntry = MangaTable.select { MangaTable.url eq manga.url }.firstOrNull()
                 if (mangaEntry == null) { // create manga entry
                     val mangaId = MangaTable.insertAndGetId {
                         it[url] = manga.url
@@ -57,6 +57,8 @@ object MangaList {
                         it[sourceReference] = sourceId
                     }.value
 
+                    mangaEntry = MangaTable.select { MangaTable.url eq manga.url }.first()
+
                     MangaDataClass(
                         mangaId,
                         sourceId.toString(),
@@ -71,7 +73,11 @@ object MangaList {
                         manga.author,
                         manga.description,
                         manga.genre,
-                        MangaStatus.valueOf(manga.status).name
+                        MangaStatus.valueOf(manga.status).name,
+                        false, // It's a new manga entry
+                        meta = getMangaMetaMap(mangaId),
+                        realUrl = mangaEntry[MangaTable.realUrl],
+                        freshData = true
                     )
                 } else {
                     val mangaId = mangaEntry[MangaTable.id].value
@@ -91,7 +97,9 @@ object MangaList {
                         mangaEntry[MangaTable.genre],
                         MangaStatus.valueOf(mangaEntry[MangaTable.status]).name,
                         mangaEntry[MangaTable.inLibrary],
-                        meta = getMangaMetaMap(mangaEntry[MangaTable.id])
+                        meta = getMangaMetaMap(mangaId),
+                        realUrl = mangaEntry[MangaTable.realUrl],
+                        freshData = false
                     )
                 }
             }

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Search.kt

@@ -7,6 +7,8 @@ package suwayomi.tachidesk.manga.impl
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
 
+import eu.kanade.tachiyomi.source.model.Filter
+import eu.kanade.tachiyomi.source.model.FilterList
 import suwayomi.tachidesk.manga.impl.MangaList.processEntries
 import suwayomi.tachidesk.manga.impl.util.GetHttpSource.getHttpSource
 import suwayomi.tachidesk.manga.impl.util.lang.awaitSingle
@@ -15,28 +17,53 @@ import suwayomi.tachidesk.manga.model.dataclass.PagedMangaListDataClass
 object Search {
     suspend fun sourceSearch(sourceId: Long, searchTerm: String, pageNum: Int): PagedMangaListDataClass {
         val source = getHttpSource(sourceId)
-        val searchManga = source.fetchSearchManga(pageNum, searchTerm, source.getFilterList()).awaitSingle()
+        val searchManga = source.fetchSearchManga(pageNum, searchTerm, getFilterListOf(sourceId)).awaitSingle()
         return searchManga.processEntries(sourceId)
     }
 
-    // TODO
-    @Suppress("UNUSED_PARAMETER", "UNUSED_VARIABLE")
-    fun sourceFilters(sourceId: Long) {
-        val source = getHttpSource(sourceId)
-        // source.getFilterList().toItems()
+    private val filterListCache = mutableMapOf<Long, FilterList>()
+
+    private fun getFilterListOf(sourceId: Long, reset: Boolean = false): FilterList {
+        if (reset || !filterListCache.containsKey(sourceId)) {
+            filterListCache[sourceId] = getHttpSource(sourceId).getFilterList()
+        }
+        return filterListCache[sourceId]!!
     }
 
+    fun getInitialFilterList(sourceId: Long, reset: Boolean): List<FilterObject> {
+        return getFilterListOf(sourceId, reset).list.map {
+            FilterObject(
+                when (it) {
+                    is Filter.Header -> "Header"
+                    is Filter.Separator -> "Separator"
+                    is Filter.CheckBox -> "CheckBox"
+                    is Filter.TriState -> "TriState"
+                    is Filter.Text -> "Text"
+                    is Filter.Select<*> -> "Select"
+                    is Filter.Group<*> -> "Group"
+                    is Filter.Sort -> "Sort"
+                },
+//                when (it) {
+//                    is Filter.Select<*> -> it.getValuesType()
+//                    else -> null
+//                },
+                it
+            )
+        }
+    }
+
+//    private fun Filter.Select<*>.getValuesType(): String = values::class.java.componentType!!.simpleName
+
+    data class FilterObject(
+        val type: String,
+        val filter: Filter<*>
+    )
+
     @Suppress("UNUSED_PARAMETER")
     fun sourceGlobalSearch(searchTerm: String) {
         // TODO
     }
 
-    @Suppress("unused")
-    data class FilterWrapper(
-        val type: String,
-        val filter: Any
-    )
-
     /**
      * Note: Exhentai had a filter serializer (now in SY) that we might be able to steal
      */

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Source.kt

@@ -35,29 +35,42 @@ object Source {
     fun getSourceList(): List<SourceDataClass> {
         return transaction {
             SourceTable.selectAll().map {
+                val httpSource = getHttpSource(it[SourceTable.id].value)
+                val sourceExtension = ExtensionTable.select { ExtensionTable.id eq it[SourceTable.extension] }.first()
+
                 SourceDataClass(
                     it[SourceTable.id].value.toString(),
                     it[SourceTable.name],
                     it[SourceTable.lang],
-                    getExtensionIconUrl(ExtensionTable.select { ExtensionTable.id eq it[SourceTable.extension] }.first()[ExtensionTable.apkName]),
-                    getHttpSource(it[SourceTable.id].value).supportsLatest,
-                    getHttpSource(it[SourceTable.id].value) is ConfigurableSource
+                    getExtensionIconUrl(sourceExtension[ExtensionTable.apkName]),
+                    httpSource.supportsLatest,
+                    httpSource is ConfigurableSource,
+                    it[SourceTable.isNsfw]
                 )
             }
         }
     }
 
-    fun getSource(sourceId: Long): SourceDataClass {
+    fun getSource(sourceId: Long): SourceDataClass { // all the data extracted fresh form the source instance
         return transaction {
             val source = SourceTable.select { SourceTable.id eq sourceId }.firstOrNull()
+            val httpSource = source?.let { getHttpSource(sourceId) }
+            val extension = source?.let {
+                ExtensionTable.select { ExtensionTable.id eq source[SourceTable.extension] }.first()
+            }
 
             SourceDataClass(
                 sourceId.toString(),
                 source?.get(SourceTable.name),
                 source?.get(SourceTable.lang),
-                source?.let { getExtensionIconUrl(ExtensionTable.select { ExtensionTable.id eq source[SourceTable.extension] }.first()[ExtensionTable.apkName]) },
-                source?.let { getHttpSource(sourceId).supportsLatest },
-                source?.let { getHttpSource(sourceId) is ConfigurableSource },
+                source?.let {
+                    getExtensionIconUrl(
+                        extension!![ExtensionTable.apkName]
+                    )
+                },
+                httpSource?.supportsLatest,
+                httpSource?.let { it is ConfigurableSource },
+                source?.get(SourceTable.isNsfw)
             )
         }
     }
@@ -65,7 +78,7 @@ object Source {
     private val context by DI.global.instance<CustomContext>()
 
     /**
-     * Clients should support these types for extensions to work properly (in order of importance)
+     * (2021-08) Clients should support these types for extensions to work properly
      * - EditTextPreference
      * - SwitchPreferenceCompat
      * - ListPreference
@@ -85,7 +98,8 @@ object Source {
         val source = getHttpSource(sourceId)
 
         if (source is ConfigurableSource) {
-            val sourceShardPreferences = Injekt.get<Application>().getSharedPreferences(source.getPreferenceKey(), Context.MODE_PRIVATE)
+            val sourceShardPreferences =
+                Injekt.get<Application>().getSharedPreferences(source.getPreferenceKey(), Context.MODE_PRIVATE)
 
             val screen = PreferenceScreen(context)
             screen.sharedPreferences = sourceShardPreferences

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/ProtoBackupImport.kt

@@ -16,6 +16,7 @@ import org.jetbrains.exposed.sql.insert
 import org.jetbrains.exposed.sql.insertAndGetId
 import org.jetbrains.exposed.sql.select
 import org.jetbrains.exposed.sql.transactions.transaction
+import org.jetbrains.exposed.sql.update
 import suwayomi.tachidesk.manga.impl.Category
 import suwayomi.tachidesk.manga.impl.CategoryManga
 import suwayomi.tachidesk.manga.impl.backup.AbstractBackupValidator.ValidationResult
@@ -31,6 +32,7 @@ import suwayomi.tachidesk.manga.model.table.CategoryTable
 import suwayomi.tachidesk.manga.model.table.ChapterTable
 import suwayomi.tachidesk.manga.model.table.MangaTable
 import java.io.InputStream
+import java.lang.Integer.max
 import java.util.Date
 
 object ProtoBackupImport : ProtoBackupBase() {
@@ -70,7 +72,7 @@ object ProtoBackupImport : ProtoBackupBase() {
         logger.info {
             """
                 Restore Errors:
-                ${ errors.joinToString("\n") { "${it.first} - ${it.second}" } }
+                ${errors.joinToString("\n") { "${it.first} - ${it.second}" }}
                 Restore Summary:
                 - Missing Sources:
                     ${validationResult.missingSources.joinToString("\n                    ")}
@@ -97,7 +99,7 @@ object ProtoBackupImport : ProtoBackupBase() {
         backupManga: BackupManga,
         backupCategories: List<BackupCategory>,
         categoryMapping: Map<Int, Int>
-    ) { // TODO
+    ) {
         val manga = backupManga.getMangaImpl()
         val chapters = backupManga.getChaptersImpl()
         val categories = backupManga.categories
@@ -112,6 +114,7 @@ object ProtoBackupImport : ProtoBackupBase() {
         }
     }
 
+    @Suppress("UNUSED_PARAMETER") // TODO: remove
     private fun restoreMangaData(
         manga: Manga,
         chapters: List<Chapter>,
@@ -125,6 +128,7 @@ object ProtoBackupImport : ProtoBackupBase() {
             MangaTable.select { (MangaTable.url eq manga.url) and (MangaTable.sourceReference eq manga.source) }
                 .firstOrNull()
         }
+
         if (dbManga == null) { // Manga not in database
             transaction {
                 // insert manga to database
@@ -143,10 +147,11 @@ object ProtoBackupImport : ProtoBackupBase() {
 
                     it[initialized] = manga.description != null
 
-                    it[inLibrary] = true
+                    it[inLibrary] = manga.favorite
                 }.value
 
                 // insert chapter data
+                val chaptersLength = chapters.size
                 chapters.forEach { chapter ->
                     ChapterTable.insert {
                         it[url] = chapter.url
@@ -155,7 +160,7 @@ object ProtoBackupImport : ProtoBackupBase() {
                         it[chapter_number] = chapter.chapter_number
                         it[scanlator] = chapter.scanlator
 
-                        it[chapterIndex] = chapter.source_order
+                        it[chapterIndex] = chaptersLength - chapter.source_order
                         it[ChapterTable.manga] = mangaId
 
                         it[isRead] = chapter.read
@@ -170,9 +175,59 @@ object ProtoBackupImport : ProtoBackupBase() {
                 }
             }
         } else { // Manga in database
-            // merge chapter data
+            transaction {
+                val mangaId = dbManga[MangaTable.id].value
 
-            // merge categories
+                // Merge manga data
+                MangaTable.update({ MangaTable.id eq mangaId }) {
+                    it[artist] = manga.artist ?: dbManga[artist]
+                    it[author] = manga.author ?: dbManga[author]
+                    it[description] = manga.description ?: dbManga[description]
+                    it[genre] = manga.genre ?: dbManga[genre]
+                    it[status] = manga.status
+                    it[thumbnail_url] = manga.thumbnail_url ?: dbManga[thumbnail_url]
+
+                    it[initialized] = dbManga[initialized] || manga.description != null
+
+                    it[inLibrary] = manga.favorite || dbManga[inLibrary]
+                }
+
+                // merge chapter data
+                val chaptersLength = chapters.size
+                val dbChapters = ChapterTable.select { ChapterTable.manga eq mangaId }
+
+                chapters.forEach { chapter ->
+                    val dbChapter = dbChapters.find { it[ChapterTable.url] == chapter.url }
+
+                    if (dbChapter == null) {
+                        ChapterTable.insert {
+                            it[url] = chapter.url
+                            it[name] = chapter.name
+                            it[date_upload] = chapter.date_upload
+                            it[chapter_number] = chapter.chapter_number
+                            it[scanlator] = chapter.scanlator
+
+                            it[chapterIndex] = chaptersLength - chapter.source_order
+                            it[ChapterTable.manga] = mangaId
+
+                            it[isRead] = chapter.read
+                            it[lastPageRead] = chapter.last_page_read
+                            it[isBookmarked] = chapter.bookmark
+                        }
+                    } else {
+                        ChapterTable.update({ (ChapterTable.url eq dbChapter[ChapterTable.url]) and (ChapterTable.manga eq mangaId) }) {
+                            it[isRead] = chapter.read || dbChapter[isRead]
+                            it[lastPageRead] = max(chapter.last_page_read, dbChapter[lastPageRead])
+                            it[isBookmarked] = chapter.bookmark || dbChapter[isBookmarked]
+                        }
+                    }
+                }
+
+                // merge categories
+                categories.forEach { backupCategoryOrder ->
+                    CategoryManga.addMangaToCategory(mangaId, categoryMapping[backupCategoryOrder]!!)
+                }
+            }
         }
 
         // TODO: insert/merge history

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/Extension.kt

@@ -50,10 +50,8 @@ object Extension {
     private val logger = KotlinLogging.logger {}
     private val applicationDirs by DI.global.instance<ApplicationDirs>()
 
-    data class InstallableAPK(
-        val apkFilePath: String,
-        val pkgName: String
-    )
+    private fun Any.isNsfw(): Boolean =
+        this::class.annotations.any { it.toString() == "@eu.kanade.tachiyomi.annotations.Nsfw()" }
 
     suspend fun installExtension(pkgName: String): Int {
         logger.debug("Installing $pkgName")
@@ -114,7 +112,8 @@ object Extension {
 
             val isNsfw = packageInfo.applicationInfo.metaData.getString(METADATA_NSFW) == "1"
 
-            val className = packageInfo.packageName + packageInfo.applicationInfo.metaData.getString(METADATA_SOURCE_CLASS)
+            val className =
+                packageInfo.packageName + packageInfo.applicationInfo.metaData.getString(METADATA_SOURCE_CLASS)
 
             logger.debug("Main class for extension is $className")
 
@@ -125,10 +124,11 @@ object Extension {
             File(dexFilePath).delete()
 
             // collect sources from the extension
-            val sources: List<CatalogueSource> = when (val instance = loadExtensionSources(jarFilePath, className)) {
-                is Source -> listOf(instance)
-                is SourceFactory -> instance.createSources()
-                else -> throw RuntimeException("Unknown source class type! ${instance.javaClass}")
+            val extensionMainClassInstance = loadExtensionSources(jarFilePath, className)
+            val sources: List<CatalogueSource> = when (extensionMainClassInstance) {
+                is Source -> listOf(extensionMainClassInstance)
+                is SourceFactory -> extensionMainClassInstance.createSources()
+                else -> throw RuntimeException("Unknown source class type! ${extensionMainClassInstance.javaClass}")
             }.map { it as CatalogueSource }
 
             val langs = sources.map { it.lang }.toSet()
@@ -159,7 +159,8 @@ object Extension {
                     it[this.classFQName] = className
                 }
 
-                val extensionId = ExtensionTable.select { ExtensionTable.pkgName eq pkgName }.first()[ExtensionTable.id].value
+                val extensionId =
+                    ExtensionTable.select { ExtensionTable.pkgName eq pkgName }.first()[ExtensionTable.id].value
 
                 sources.forEach { httpSource ->
                     SourceTable.insert {
@@ -167,8 +168,9 @@ object Extension {
                         it[name] = httpSource.name
                         it[lang] = httpSource.lang
                         it[extension] = extensionId
+                        it[SourceTable.isNsfw] = isNsfw || extensionMainClassInstance.isNsfw()
                     }
-                    logger.debug("Installed source ${httpSource.name} (${httpSource.lang}) with id:${httpSource.id}")
+                    logger.debug { "Installed source ${httpSource.name} (${httpSource.lang}) with id:${httpSource.id}" }
                 }
             }
             return 201 // we installed successfully
@@ -234,7 +236,8 @@ object Extension {
     }
 
     suspend fun getExtensionIcon(apkName: String): Pair<InputStream, String> {
-        val iconUrl = transaction { ExtensionTable.select { ExtensionTable.apkName eq apkName }.first() }[ExtensionTable.iconUrl]
+        val iconUrl =
+            transaction { ExtensionTable.select { ExtensionTable.apkName eq apkName }.first() }[ExtensionTable.iconUrl]
 
         val saveDir = "${applicationDirs.extensionsRoot}/icon"
 

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/PackageTools.kt

@@ -137,7 +137,7 @@ object PackageTools {
     }
 
     /**
-     * loads the extension main class called $className from the jar located at $jarPath
+     * loads the extension main class called [className] from the jar located at [jarPath]
      * It may return an instance of HttpSource or SourceFactory depending on the extension.
      */
     fun loadExtensionSources(jarPath: String, className: String): Any {

server/src/main/kotlin/suwayomi/tachidesk/manga/model/dataclass/MangaDataClass.kt

@@ -26,9 +26,13 @@ data class MangaDataClass(
     val status: String = MangaStatus.UNKNOWN.name,
     val inLibrary: Boolean = false,
     val source: SourceDataClass? = null,
+
+    /** meta data for clients */
     val meta: Map<String, String> = emptyMap(),
 
-    val freshData: Boolean = false
+    val realUrl: String? = null,
+
+    val freshData: Boolean = false,
 )
 
 data class PagedMangaListDataClass(

server/src/main/kotlin/suwayomi/tachidesk/manga/model/dataclass/SourceDataClass.kt

@@ -1,5 +1,7 @@
 package suwayomi.tachidesk.manga.model.dataclass
 
+import eu.kanade.tachiyomi.source.ConfigurableSource
+
 /*
  * Copyright (C) Contributors to the Suwayomi project
  * 
@@ -12,6 +14,13 @@ data class SourceDataClass(
     val name: String?,
     val lang: String?,
     val iconUrl: String?,
+
+    /** The Source provides a latest listing */
     val supportsLatest: Boolean?,
-    val isConfigurable: Boolean?
+
+    /** The Source implements [ConfigurableSource] */
+    val isConfigurable: Boolean?,
+
+    /** The Source class has a @Nsfw annotation */
+    val isNSFW: Boolean?,
 )

server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/MangaTable.kt

@@ -31,8 +31,11 @@ object MangaTable : IntIdTable() {
     val inLibrary = bool("in_library").default(false)
     val defaultCategory = bool("default_category").default(true)
 
-    // source is used by some ancestor of IntIdTable
+    // the [source] field name is used by some ancestor of IntIdTable
     val sourceReference = long("source")
+
+    /** the real url of a manga used for the "open in WebView" feature */
+    val realUrl = varchar("real_url", 2048).nullable()
 }
 
 fun MangaTable.toDataClass(mangaEntry: ResultRow) =
@@ -52,7 +55,8 @@ fun MangaTable.toDataClass(mangaEntry: ResultRow) =
         mangaEntry[genre],
         Companion.valueOf(mangaEntry[status]).name,
         mangaEntry[inLibrary],
-        meta = getMangaMetaMap(mangaEntry[id])
+        meta = getMangaMetaMap(mangaEntry[id].value),
+        realUrl = mangaEntry[realUrl],
     )
 
 enum class MangaStatus(val value: Int) {

server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/SourceTable.kt

@@ -14,5 +14,5 @@ object SourceTable : IdTable<Long>() {
     val name = varchar("name", 128)
     val lang = varchar("lang", 10)
     val extension = reference("extension", ExtensionTable)
-    val partOfFactorySource = bool("part_of_factory_source").default(false)
+    val isNsfw = bool("is_nsfw").default(false)
 }

server/src/main/kotlin/suwayomi/tachidesk/server/ServerConfig.kt

@@ -8,31 +8,32 @@ package suwayomi.tachidesk.server
  * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
 
 import com.typesafe.config.Config
-import xyz.nulldev.ts.config.ConfigModule
 import xyz.nulldev.ts.config.GlobalConfigManager
+import xyz.nulldev.ts.config.SystemPropertyOverridableConfigModule
 import xyz.nulldev.ts.config.debugLogsEnabled
 
-class ServerConfig(config: Config, moduleName: String = "") : ConfigModule(config, moduleName) {
-    val ip: String by overridableWithSysProperty
-    val port: Int by overridableWithSysProperty
+private const val MODULE_NAME = "server"
+class ServerConfig(config: Config, moduleName: String = MODULE_NAME) : SystemPropertyOverridableConfigModule(config, moduleName) {
+    val ip: String by overridableConfig
+    val port: Int by overridableConfig
 
     // proxy
-    val socksProxyEnabled: Boolean by overridableWithSysProperty
+    val socksProxyEnabled: Boolean by overridableConfig
 
-    val socksProxyHost: String by overridableWithSysProperty
-    val socksProxyPort: String by overridableWithSysProperty
+    val socksProxyHost: String by overridableConfig
+    val socksProxyPort: String by overridableConfig
 
     // misc
     val debugLogsEnabled: Boolean = debugLogsEnabled(GlobalConfigManager.config)
-    val systemTrayEnabled: Boolean by overridableWithSysProperty
+    val systemTrayEnabled: Boolean by overridableConfig
 
     // webUI
-    val webUIEnabled: Boolean by overridableWithSysProperty
-    val initialOpenInBrowserEnabled: Boolean by overridableWithSysProperty
-    val webUIInterface: String by overridableWithSysProperty
-    val electronPath: String by overridableWithSysProperty
+    val webUIEnabled: Boolean by overridableConfig
+    val initialOpenInBrowserEnabled: Boolean by overridableConfig
+    val webUIInterface: String by overridableConfig
+    val electronPath: String by overridableConfig
 
     companion object {
-        fun register(config: Config) = ServerConfig(config.getConfig("server"), "server")
+        fun register(config: Config) = ServerConfig(config.getConfig(MODULE_NAME))
     }
 }

server/src/main/kotlin/suwayomi/tachidesk/server/database/migration/M0011_SourceDropPartOfFactorySource.kt

@@ -0,0 +1,16 @@
+package suwayomi.tachidesk.server.database.migration
+
+/*
+ * Copyright (C) Contributors to the Suwayomi project
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
+
+import de.neonew.exposed.migrations.helpers.DropColumnMigration
+
+@Suppress("ClassName", "unused")
+class M0011_SourceDropPartOfFactorySource : DropColumnMigration(
+    "Source",
+    "part_of_factory_source",
+)

server/src/main/kotlin/suwayomi/tachidesk/server/database/migration/M0012_SourceIsNsfw.kt

@@ -0,0 +1,18 @@
+package suwayomi.tachidesk.server.database.migration
+
+/*
+ * Copyright (C) Contributors to the Suwayomi project
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
+
+import de.neonew.exposed.migrations.helpers.AddColumnMigration
+
+@Suppress("ClassName", "unused")
+class M0012_SourceIsNsfw : AddColumnMigration(
+    "Source",
+    "is_nsfw",
+    "BOOLEAN",
+    "FALSE"
+)

server/src/main/kotlin/suwayomi/tachidesk/server/database/migration/M0013_MangaRealUrl.kt

@@ -0,0 +1,18 @@
+package suwayomi.tachidesk.server.database.migration
+
+/*
+ * Copyright (C) Contributors to the Suwayomi project
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
+
+import de.neonew.exposed.migrations.helpers.AddColumnMigration
+
+@Suppress("ClassName", "unused")
+class M0013_MangaRealUrl : AddColumnMigration(
+    "Manga",
+    "real_url",
+    "VARCHAR(2048)",
+    "NULL"
+)

server/src/test/resources/server-reference.conf

@@ -3,11 +3,16 @@ server.ip = "0.0.0.0"
 server.port = 4567
 
 # Socks5 proxy
-server.socksProxy = false
+server.socksProxyEnabled = false
 server.socksProxyHost = ""
 server.socksProxyPort = ""
 
 # misc
 server.debugLogsEnabled = true
 server.systemTrayEnabled = false
+
+# webUI
+server.webUIEnabled = true
 server.initialOpenInBrowserEnabled = true
+server.webUIInterface = "browser" # "browser" or "electron"
+server.electronPath = ""