admin/Suwayomi-Server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Support Custom Repos (#803)

Browse Source 
* Support custom repos

* Fix migration

* Make extension after update optional
This commit is contained in:
Mitchell Syer
2024-01-05 19:14:09 -05:00
committed by GitHub
parent abf1af41a3
commit 230427e758
19 changed files with 149 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
AndroidCompat/Config/src/main/java/xyz/nulldev/ts/config/ConfigModule.kt
+10 -2
View File
@@ -8,6 +8,8 @@ package xyz.nulldev.ts.config
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
import com.typesafe.config.Config
import com.typesafe.config.ConfigFactory
import com.typesafe.config.ConfigValueFactory
import io.github.config4k.getValue
import kotlin.reflect.KProperty
@@ -30,18 +32,24 @@ class SystemPropertyOverrideDelegate(val getConfig: () -> Config, val moduleName
thisRef: R,
property: KProperty<*>,
): T {
val configValue: T = getConfig().getValue(thisRef, property)
val config = getConfig()
val configValue: T = config.getValue(thisRef, property)
val combined =
System.getProperty(
"$CONFIG_PREFIX.$moduleName.${property.name}",
configValue.toString(),
if (T::class.simpleName == "List") {
ConfigValueFactory.fromAnyRef(configValue).render()
} else {
configValue.toString()
},
)
return when (T::class.simpleName) {
"Int" -> combined.toInt()
"Boolean" -> combined.toBoolean()
"Double" -> combined.toDouble()
"List" -> ConfigFactory.parseString("internal=" + combined).getStringList("internal").orEmpty()
// add more types as needed
else -> combined // covers String
} as T
server/src/main/kotlin/suwayomi/tachidesk/graphql/mutations/ExtensionMutation.kt
+3 -2
View File
@@ -20,7 +20,7 @@ class ExtensionMutation {
data class UpdateExtensionPayload(
val clientMutationId: String?,
val extension: ExtensionType,
val extension: ExtensionType?,
)
data class UpdateExtensionInput(
@@ -77,7 +77,8 @@ class ExtensionMutation {
}.thenApply {
val extension =
transaction {
ExtensionType(ExtensionTable.select { ExtensionTable.pkgName eq id }.first())
ExtensionTable.select { ExtensionTable.pkgName eq id }.firstOrNull()
?.let { ExtensionType(it) }
}
UpdateExtensionPayload(
server/src/main/kotlin/suwayomi/tachidesk/graphql/mutations/SettingsMutation.kt
+3
View File
@@ -55,6 +55,9 @@ class SettingsMutation {
updateSetting(settings.excludeEntryWithUnreadChapters, serverConfig.excludeEntryWithUnreadChapters)
updateSetting(settings.autoDownloadAheadLimit, serverConfig.autoDownloadAheadLimit)
// extension
updateSetting(settings.extensionRepos, serverConfig.extensionRepos)
// requests
updateSetting(settings.maxSourcesInParallel, serverConfig.maxSourcesInParallel)
server/src/main/kotlin/suwayomi/tachidesk/graphql/queries/ExtensionQuery.kt
+4
View File
@@ -82,6 +82,7 @@ class ExtensionQuery {
}
data class ExtensionCondition(
val repo: String? = null,
val apkName: String? = null,
val iconUrl: String? = null,
val name: String? = null,
@@ -96,6 +97,7 @@ class ExtensionQuery {
) : HasGetOp {
override fun getOp(): Op<Boolean>? {
val opAnd = OpAnd()
opAnd.eq(repo, ExtensionTable.repo)
opAnd.eq(apkName, ExtensionTable.apkName)
opAnd.eq(iconUrl, ExtensionTable.iconUrl)
opAnd.eq(name, ExtensionTable.name)
@@ -112,6 +114,7 @@ class ExtensionQuery {
}
data class ExtensionFilter(
val repo: StringFilter? = null,
val apkName: StringFilter? = null,
val iconUrl: StringFilter? = null,
val name: StringFilter? = null,
@@ -129,6 +132,7 @@ class ExtensionQuery {
) : Filter<ExtensionFilter> {
override fun getOpList(): List<Op<Boolean>> {
return listOfNotNull(
andFilterWithCompareString(ExtensionTable.repo, repo),
andFilterWithCompareString(ExtensionTable.apkName, apkName),
andFilterWithCompareString(ExtensionTable.iconUrl, iconUrl),
andFilterWithCompareString(ExtensionTable.name, name),
server/src/main/kotlin/suwayomi/tachidesk/graphql/types/ExtensionType.kt
+2
View File
@@ -20,6 +20,7 @@ import suwayomi.tachidesk.manga.model.table.ExtensionTable
import java.util.concurrent.CompletableFuture
class ExtensionType(
val repo: String?,
val apkName: String,
val iconUrl: String,
val name: String,
@@ -33,6 +34,7 @@ class ExtensionType(
val isObsolete: Boolean,
) : Node {
constructor(row: ResultRow) : this(
repo = row[ExtensionTable.repo],
apkName = row[ExtensionTable.apkName],
iconUrl = Extension.getExtensionIconUrl(row[ExtensionTable.apkName]),
name = row[ExtensionTable.name],
server/src/main/kotlin/suwayomi/tachidesk/graphql/types/MangaType.kt
+4
View File
@@ -8,6 +8,7 @@
package suwayomi.tachidesk.graphql.types
import com.expediagroup.graphql.server.extensions.getValueFromDataLoader
import eu.kanade.tachiyomi.source.model.UpdateStrategy
import graphql.schema.DataFetchingEnvironment
import org.jetbrains.exposed.sql.ResultRow
import suwayomi.tachidesk.graphql.server.primitives.Cursor
@@ -37,6 +38,7 @@ class MangaType(
val status: MangaStatus,
val inLibrary: Boolean,
val inLibraryAt: Long,
val updateStrategy: UpdateStrategy,
val realUrl: String?,
var lastFetchedAt: Long?, // todo
var chaptersLastFetchedAt: Long?, // todo
@@ -73,6 +75,7 @@ class MangaType(
MangaStatus.valueOf(row[MangaTable.status]),
row[MangaTable.inLibrary],
row[MangaTable.inLibraryAt],
UpdateStrategy.valueOf(row[MangaTable.updateStrategy]),
row[MangaTable.realUrl],
row[MangaTable.lastFetchedAt],
row[MangaTable.chaptersLastFetchedAt],
@@ -92,6 +95,7 @@ class MangaType(
MangaStatus.valueOf(dataClass.status),
dataClass.inLibrary,
dataClass.inLibraryAt,
dataClass.updateStrategy,
dataClass.realUrl,
dataClass.lastFetchedAt,
dataClass.chaptersLastFetchedAt,
server/src/main/kotlin/suwayomi/tachidesk/graphql/types/SettingsType.kt
+9
View File
@@ -40,6 +40,9 @@ interface Settings : Node {
val excludeEntryWithUnreadChapters: Boolean?
val autoDownloadAheadLimit: Int?
// extension
val extensionRepos: List<String>?
// requests
val maxSourcesInParallel: Int?
@@ -90,6 +93,8 @@ data class PartialSettingsType(
override val autoDownloadNewChapters: Boolean?,
override val excludeEntryWithUnreadChapters: Boolean?,
override val autoDownloadAheadLimit: Int?,
// extension
override val extensionRepos: List<String>?,
// requests
override val maxSourcesInParallel: Int?,
// updater
@@ -135,6 +140,8 @@ class SettingsType(
override val autoDownloadNewChapters: Boolean,
override val excludeEntryWithUnreadChapters: Boolean,
override val autoDownloadAheadLimit: Int,
// extension
override val extensionRepos: List<String>,
// requests
override val maxSourcesInParallel: Int,
// updater
@@ -179,6 +186,8 @@ class SettingsType(
config.autoDownloadNewChapters.value,
config.excludeEntryWithUnreadChapters.value,
config.autoDownloadAheadLimit.value,
// extension
config.extensionRepos.value,
// requests
config.maxSourcesInParallel.value,
// updater
server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Chapter.kt
+1 -1
View File
@@ -143,7 +143,7 @@ object Chapter {
val chapterNumber = ChapterRecognition.parseChapterNumber(manga.title, chapter.name, chapter.chapter_number.toDouble())
chapter.chapter_number = chapterNumber.toFloat()
chapter.name = chapter.name.sanitize(manga.title)
chapter.scanlator = chapter.scanlator?.ifBlank { null }
chapter.scanlator = chapter.scanlator?.ifBlank { null }?.trim()
}
val now = Instant.now().epochSecond
server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/ExtensionsList.kt
+16 -2
View File
@@ -23,6 +23,7 @@ import suwayomi.tachidesk.manga.impl.extension.github.ExtensionGithubApi
import suwayomi.tachidesk.manga.impl.extension.github.OnlineExtension
import suwayomi.tachidesk.manga.model.dataclass.ExtensionDataClass
import suwayomi.tachidesk.manga.model.table.ExtensionTable
import suwayomi.tachidesk.server.serverConfig
import java.util.concurrent.ConcurrentHashMap
import kotlin.time.Duration.Companion.seconds
@@ -38,7 +39,17 @@ object ExtensionsList {
logger.debug("Getting extensions list from the internet")
lastUpdateCheck = System.currentTimeMillis()
val foundExtensions = ExtensionGithubApi.findExtensions()
val extensions =
(listOf(ExtensionGithubApi.REPO_URL_PREFIX) + serverConfig.extensionRepos.value).map { repo ->
kotlin.runCatching {
ExtensionGithubApi.findExtensions(repo)
}.onFailure {
logger.warn(it) {
"Failed to fetch extensions for repo: $repo"
}
}
}
val foundExtensions = extensions.mapNotNull { it.getOrNull() }.flatten()
updateExtensionDatabase(foundExtensions)
} else {
logger.debug("used cached extension list")
@@ -54,6 +65,7 @@ object ExtensionsList {
transaction {
ExtensionTable.selectAll().filter { it[ExtensionTable.name] != LocalSource.EXTENSION_NAME }.map {
ExtensionDataClass(
it[ExtensionTable.repo],
it[ExtensionTable.apkName],
getExtensionIconUrl(it[ExtensionTable.apkName]),
it[ExtensionTable.name],
@@ -77,7 +89,7 @@ object ExtensionsList {
val extensionsToUpdate = mutableListOf<Pair<OnlineExtension, ResultRow>>()
val extensionsToInsert = mutableListOf<OnlineExtension>()
val extensionsToDelete =
installedExtensions.mapNotNull { (pkgName, extension) ->
installedExtensions.filter { it.value[ExtensionTable.repo] != null }.mapNotNull { (pkgName, extension) ->
extension.takeUnless { foundExtensions.any { it.pkgName == pkgName } }
}
foundExtensions.forEach {
@@ -124,6 +136,7 @@ object ExtensionsList {
extensionsToFullyUpdate.forEach { (foundExtension, extensionRecord) ->
addBatch(EntityID(extensionRecord[ExtensionTable.id].value, ExtensionTable))
// extension is not installed, so we can overwrite the data without a care
this[ExtensionTable.repo] = foundExtension.repo
this[ExtensionTable.name] = foundExtension.name
this[ExtensionTable.versionName] = foundExtension.versionName
this[ExtensionTable.versionCode] = foundExtension.versionCode
@@ -138,6 +151,7 @@ object ExtensionsList {
}
if (extensionsToInsert.isNotEmpty()) {
ExtensionTable.batchInsert(extensionsToInsert) { foundExtension ->
this[ExtensionTable.repo] = foundExtension.repo
this[ExtensionTable.name] = foundExtension.name
this[ExtensionTable.pkgName] = foundExtension.pkgName
this[ExtensionTable.versionName] = foundExtension.versionName
server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/github/ExtensionGithubApi.kt
+27 -8
View File
@@ -20,7 +20,7 @@ import suwayomi.tachidesk.manga.model.dataclass.ExtensionDataClass
import uy.kohesive.injekt.injectLazy
object ExtensionGithubApi {
private const val REPO_URL_PREFIX = "https://raw.githubusercontent.com/tachiyomiorg/tachiyomi-extensions/repo/"
const val REPO_URL_PREFIX = "https://raw.githubusercontent.com/tachiyomiorg/tachiyomi-extensions/repo/"
private const val FALLBACK_REPO_URL_PREFIX = "https://gcore.jsdelivr.net/gh/tachiyomiorg/tachiyomi-extensions@repo/"
private val logger = KotlinLogging.logger {}
private val json: Json by injectLazy()
@@ -49,13 +49,13 @@ object ExtensionGithubApi {
private var requiresFallbackSource = false
suspend fun findExtensions(): List<OnlineExtension> {
suspend fun findExtensions(repo: String): List<OnlineExtension> {
val githubResponse =
if (requiresFallbackSource) {
null
} else {
try {
client.newCall(GET("${REPO_URL_PREFIX}index.min.json")).awaitSuccess()
client.newCall(GET("${repo.repoUrlReplace()}index.min.json")).awaitSuccess()
} catch (e: Throwable) {
logger.error(e) { "Failed to get extensions from GitHub" }
requiresFallbackSource = true
@@ -65,18 +65,18 @@ object ExtensionGithubApi {
val response =
githubResponse ?: run {
client.newCall(GET("${FALLBACK_REPO_URL_PREFIX}index.min.json")).awaitSuccess()
client.newCall(GET("${repo.fallbackRepoUrlReplace()}index.min.json")).awaitSuccess()
}
return with(json) {
response
.parseAs<List<ExtensionJsonObject>>()
.toExtensions()
.toExtensions(repo.repoUrlReplace())
}
}
fun getApkUrl(extension: ExtensionDataClass): String {
return "$REPO_URL_PREFIX/apk/${extension.apkName}"
return "${extension.repo!!.repoUrlReplace()}/apk/${extension.apkName}"
}
private val client by lazy {
@@ -91,7 +91,7 @@ object ExtensionGithubApi {
.build()
}
private fun List<ExtensionJsonObject>.toExtensions(): List<OnlineExtension> {
private fun List<ExtensionJsonObject>.toExtensions(repo: String): List<OnlineExtension> {
return this
.filter {
val libVersion = it.version.substringBeforeLast('.').toDouble()
@@ -99,6 +99,7 @@ object ExtensionGithubApi {
}
.map {
OnlineExtension(
repo = repo,
name = it.name.substringAfter("Tachiyomi: "),
pkgName = it.pkg,
versionName = it.version,
@@ -109,7 +110,7 @@ object ExtensionGithubApi {
hasChangelog = it.hasChangelog == 1,
sources = it.sources?.toExtensionSources() ?: emptyList(),
apkName = it.apk,
iconUrl = "${REPO_URL_PREFIX}icon/${it.pkg}.png",
iconUrl = "${repo}icon/${it.pkg}.png",
)
}
}
@@ -124,4 +125,22 @@ object ExtensionGithubApi {
)
}
}
private fun String.repoUrlReplace() =
replace(repoMatchRegex) {
"https://raw.githubusercontent.com/${it.groupValues[1]}/${it.groupValues[2]}/" +
"${it.groupValues.getOrNull(3)?.ifBlank { null } ?: "repo"}/"
}
private fun String.fallbackRepoUrlReplace() =
replace(repoMatchRegex) {
"https://gcore.jsdelivr.net/gh/${it.groupValues[1]}/${it.groupValues[2]}@" +
"${it.groupValues.getOrNull(3)?.ifBlank { null } ?: "repo"}/"
}
private val repoMatchRegex =
(
"https:\\/\\/(?:www|raw)?(?:github|githubusercontent)\\.com" +
"\\/([^\\/]+)\\/([^\\/]+)(?:\\/(?:tree|blob)\\/(.*))?\\/?"
).toRegex()
}
server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/github/OnlineExtension.kt
+1
View File
@@ -15,6 +15,7 @@ data class OnlineExtensionSource(
)
data class OnlineExtension(
val repo: String,
val name: String,
val pkgName: String,
val apkName: String,
server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/PackageTools.kt
+1 -2
View File
@@ -44,8 +44,7 @@ object PackageTools {
const val LIB_VERSION_MAX = 1.5
private const val OFFICIAL_SIGNATURE = "7ce04da7773d41b489f4693a366c36bcd0a11fc39b547168553c285bd7348e23" // inorichi's key
private const val UNOFFICIAL_SIGNATURE = "64feb21075ba97ebc9cc981243645b331595c111cef1b0d084236a0403b00581" // ArMor's key
val trustedSignatures = setOf(OFFICIAL_SIGNATURE, UNOFFICIAL_SIGNATURE)
val trustedSignatures = setOf(OFFICIAL_SIGNATURE)
/**
* Convert dex to jar, a wrapper for the dex2jar library
server/src/main/kotlin/suwayomi/tachidesk/manga/model/dataclass/ExtensionDataClass.kt
+1
View File
@@ -8,6 +8,7 @@ package suwayomi.tachidesk.manga.model.dataclass
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
data class ExtensionDataClass(
val repo: String?,
val apkName: String,
val iconUrl: String,
val name: String,
server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/ExtensionTable.kt
+1
View File
@@ -11,6 +11,7 @@ import org.jetbrains.exposed.dao.id.IntIdTable
object ExtensionTable : IntIdTable() {
val apkName = varchar("apk_name", 1024)
val repo = varchar("repo", 1024).nullable()
// default is the local source icon from tachiyomi
@Suppress("ktlint:standard:max-line-length")
server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/MangaTable.kt
+1 -1
View File
@@ -80,6 +80,6 @@ enum class MangaStatus(val value: Int) {
;
companion object {
fun valueOf(value: Int): MangaStatus = values().find { it.value == value } ?: UNKNOWN
fun valueOf(value: Int): MangaStatus = entries.find { it.value == value } ?: UNKNOWN
}
}
server/src/main/kotlin/suwayomi/tachidesk/server/ServerConfig.kt
+22 -4
View File
@@ -35,9 +35,14 @@ class ServerConfig(getConfig: () -> Config, val moduleName: String = SERVER_CONF
getConfig,
moduleName,
) {
inner class OverrideConfigValue<T>(private val configAdapter: ConfigAdapter<T>) {
open inner class OverrideConfigValue<T>(private val configAdapter: ConfigAdapter<out Any>) {
private var flow: MutableStateFlow<T>? = null
open fun getValueFromConfig(
thisRef: ServerConfig,
property: KProperty<*>,
): Any = configAdapter.toType(overridableConfig.getValue<ServerConfig, String>(thisRef, property))
operator fun getValue(
thisRef: ServerConfig,
property: KProperty<*>,
@@ -46,13 +51,13 @@ class ServerConfig(getConfig: () -> Config, val moduleName: String = SERVER_CONF
return flow!!
}
val getValueFromConfig = { configAdapter.toType(overridableConfig.getValue<ServerConfig, String>(thisRef, property)) }
val value = getValueFromConfig()
@Suppress("UNCHECKED_CAST")
val value = getValueFromConfig(thisRef, property) as T
val stateFlow = MutableStateFlow(value)
flow = stateFlow
stateFlow.drop(1).distinctUntilChanged().filter { it != getValueFromConfig() }
stateFlow.drop(1).distinctUntilChanged().filter { it != getValueFromConfig(thisRef, property) }
.onEach { GlobalConfigManager.updateValue("$moduleName.${property.name}", it as Any) }
.launchIn(mutableConfigValueScope)
@@ -60,6 +65,16 @@ class ServerConfig(getConfig: () -> Config, val moduleName: String = SERVER_CONF
}
}
inner class OverrideConfigValues<T>(private val configAdapter: ConfigAdapter<out Any>) : OverrideConfigValue<T>(configAdapter) {
override fun getValueFromConfig(
thisRef: ServerConfig,
property: KProperty<*>,
): Any {
return overridableConfig.getValue<ServerConfig, List<String>>(thisRef, property)
.map { configAdapter.toType(it) }
}
}
val ip: MutableStateFlow<String> by OverrideConfigValue(StringConfigAdapter)
val port: MutableStateFlow<Int> by OverrideConfigValue(IntConfigAdapter)
@@ -84,6 +99,9 @@ class ServerConfig(getConfig: () -> Config, val moduleName: String = SERVER_CONF
val excludeEntryWithUnreadChapters: MutableStateFlow<Boolean> by OverrideConfigValue(BooleanConfigAdapter)
val autoDownloadAheadLimit: MutableStateFlow<Int> by OverrideConfigValue(IntConfigAdapter)
// extensions
val extensionRepos: MutableStateFlow<List<String>> by OverrideConfigValues(StringConfigAdapter)
// requests
val maxSourcesInParallel: MutableStateFlow<Int> by OverrideConfigValue(IntConfigAdapter)
server/src/main/kotlin/suwayomi/tachidesk/server/database/migration/M0031_AddExtensionRepo.kt
+18
View File
@@ -0,0 +1,18 @@
package suwayomi.tachidesk.server.database.migration
/*
* Copyright (C) Contributors to the Suwayomi project
*
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
import de.neonew.exposed.migrations.helpers.AddColumnMigration
@Suppress("ClassName", "unused")
class M0031_AddExtensionRepo : AddColumnMigration(
"Extension",
"repo",
"VARCHAR(1024)",
"NULL",
)
server/src/main/kotlin/suwayomi/tachidesk/server/database/migration/M0032_FixExtensionRepos.kt
+20
View File
@@ -0,0 +1,20 @@
package suwayomi.tachidesk.server.database.migration
/*
* Copyright (C) Contributors to the Suwayomi project
*
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
import de.neonew.exposed.migrations.helpers.SQLMigration
@Suppress("ClassName", "unused")
class M0032_FixExtensionRepos : SQLMigration() {
// language=h2
override val sql =
"""
UPDATE EXTENSION
SET REPO = 'https://raw.githubusercontent.com/tachiyomiorg/tachiyomi-extensions/repo/';
""".trimIndent()
}
server/src/main/resources/server-reference.conf
+5
View File
@@ -23,6 +23,11 @@ server.autoDownloadNewChapters = false # if new chapters that have been retrieve
server.excludeEntryWithUnreadChapters = true # ignore automatic chapter downloads of entries with unread chapters
server.autoDownloadAheadLimit = 0 # 0 to disable it - how many unread downloaded chapters should be available - if the limit is reached, new chapters won't be downloaded automatically. this limit will also be applied to the auto download of new chapters on an update
# extension repos
server.extensionRepos = [
# an example: https://github.com/MY_ACCOUNT/MY_REPO/tree/repo
]
# requests
server.maxSourcesInParallel = 6 # range: 1 <= n <= 20 - default: 6 - sets how many sources can do requests (updates, downloads) in parallel. updates/downloads are grouped by source and all mangas of a source are updated/downloaded synchronously