From e5476f8a01dc831a7cefba208b9a596b55c41301 Mon Sep 17 00:00:00 2001 From: Mitchell Syer Date: Tue, 9 Jan 2024 19:42:53 -0500 Subject: [PATCH] Extension repo fixes and improvements (#811) --- .github/ISSUE_TEMPLATE/bug_report.md | 2 +- .github/ISSUE_TEMPLATE/feature_request.md | 2 +- .../manga/impl/extension/Extension.kt | 6 ++- .../manga/impl/extension/ExtensionsList.kt | 26 ++++++++-- .../extension/github/ExtensionGithubApi.kt | 49 +++---------------- .../tachidesk/manga/impl/util/PackageTools.kt | 3 -- 6 files changed, 37 insertions(+), 51 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 3b2133b0..898eb3f9 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -11,7 +11,7 @@ I acknowledge that: - I have updated to the latest version of the app. - I have tried the troubleshooting guide described in `README.md` -- If this is a request for adding/changing an extension it should be brought up to Tachiyomi: https://github.com/tachiyomiorg/tachiyomi-extensions/issues/new/choose +- If this is a request for adding/changing an extension it should be brought up to your extension repo. - If this is an issue with some extension not working properly, It does work inside Tachiyomi as intended. - I have searched the existing issues and this is a new ticket **NOT** a duplicate or related to another open issue - I will fill out the title and the information in this template diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index e6cd027e..60a28fa0 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -11,7 +11,7 @@ I acknowledge that: - I have updated to the latest version of the app. - I have tried the troubleshooting guide described in `README.md` -- If this is a request for adding/changing an extension it should be brought up to Tachiyomi: https://github.com/tachiyomiorg/tachiyomi-extensions/issues/new/choose +- If this is a request for adding/changing an extension it should be brought up to your extension repo. - If this is an issue with some extension not working properly, It does work in Tachiyomi application as intended. - I have searched the existing issues and this is a new ticket **NOT** a duplicate or related to another open issue - I will fill out the title and the information in this template diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/Extension.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/Extension.kt index abc99802..0b804e5f 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/Extension.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/Extension.kt @@ -61,7 +61,11 @@ object Extension { val extensionRecord = extensionTableAsDataClass().first { it.pkgName == pkgName } return installAPK { - val apkURL = ExtensionGithubApi.getApkUrl(extensionRecord) + val apkURL = + ExtensionGithubApi.getApkUrl( + extensionRecord.repo ?: throw NullPointerException("Could not find extension repo"), + extensionRecord.apkName, + ) val apkName = Uri.parse(apkURL).lastPathSegment!! val apkSavePath = "${applicationDirs.extensionsRoot}/$apkName" // download apk file diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/ExtensionsList.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/ExtensionsList.kt index 00e21d18..40662fa1 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/ExtensionsList.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/ExtensionsList.kt @@ -36,9 +36,9 @@ object ExtensionsList { suspend fun fetchExtensions() { // update if 60 seconds has passed or requested offline and database is empty val extensions = - (listOf(ExtensionGithubApi.REPO_URL_PREFIX) + serverConfig.extensionRepos.value).map { repo -> + serverConfig.extensionRepos.value.map { repo -> kotlin.runCatching { - ExtensionGithubApi.findExtensions(repo) + ExtensionGithubApi.findExtensions(repo.repoUrlReplace()) }.onFailure { logger.warn(it) { "Failed to fetch extensions for repo: $repo" @@ -119,8 +119,9 @@ object ExtensionsList { BatchUpdateStatement(ExtensionTable).apply { installedExtensionsToUpdate.forEach { (foundExtension, extensionRecord) -> addBatch(EntityID(extensionRecord[ExtensionTable.id].value, ExtensionTable)) - // Always update icon url + // Always update icon url and repo this[ExtensionTable.iconUrl] = foundExtension.iconUrl + this[ExtensionTable.repo] = foundExtension.repo // add these because batch updates need matching columns this[ExtensionTable.hasUpdate] = extensionRecord[ExtensionTable.hasUpdate] @@ -199,4 +200,23 @@ object ExtensionsList { } } } + + private fun String.repoUrlReplace(): String { + return if (contains("github")) { + replace(repoMatchRegex) { + "https://raw.githubusercontent.com/${it.groupValues[2]}/${it.groupValues[3]}/" + + (it.groupValues.getOrNull(4)?.ifBlank { null } ?: "repo") + + "/" + + (it.groupValues.getOrNull(5)?.ifBlank { null } ?: "index.min.json") + } + } else { + this + } + } + + private val repoMatchRegex = + ( + "https:\\/\\/(?>www\\.|raw\\.)?(github|githubusercontent)\\.com" + + "\\/([^\\/]+)\\/([^\\/]+)(?>(?>\\/tree|\\/blob)?\\/([^\\/\\n]*))?(?>\\/([^\\/\\n]*\\.json)?)?" + ).toRegex() } diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/github/ExtensionGithubApi.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/github/ExtensionGithubApi.kt index bf23b11b..bace24bd 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/github/ExtensionGithubApi.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/github/ExtensionGithubApi.kt @@ -16,12 +16,9 @@ import kotlinx.serialization.json.Json import mu.KotlinLogging import suwayomi.tachidesk.manga.impl.util.PackageTools.LIB_VERSION_MAX import suwayomi.tachidesk.manga.impl.util.PackageTools.LIB_VERSION_MIN -import suwayomi.tachidesk.manga.model.dataclass.ExtensionDataClass import uy.kohesive.injekt.injectLazy object ExtensionGithubApi { - const val REPO_URL_PREFIX = "https://raw.githubusercontent.com/tachiyomiorg/tachiyomi-extensions/repo/" - private const val FALLBACK_REPO_URL_PREFIX = "https://gcore.jsdelivr.net/gh/tachiyomiorg/tachiyomi-extensions@repo/" private val logger = KotlinLogging.logger {} private val json: Json by injectLazy() @@ -47,36 +44,22 @@ object ExtensionGithubApi { val baseUrl: String, ) - private var requiresFallbackSource = false - suspend fun findExtensions(repo: String): List { - val githubResponse = - if (requiresFallbackSource) { - null - } else { - try { - client.newCall(GET("${repo.repoUrlReplace()}index.min.json")).awaitSuccess() - } catch (e: Throwable) { - logger.error(e) { "Failed to get extensions from GitHub" } - requiresFallbackSource = true - null - } - } - val response = - githubResponse ?: run { - client.newCall(GET("${repo.fallbackRepoUrlReplace()}index.min.json")).awaitSuccess() - } + client.newCall(GET(repo)).awaitSuccess() return with(json) { response .parseAs>() - .toExtensions(repo.repoUrlReplace()) + .toExtensions(repo.substringBeforeLast('/') + '/') } } - fun getApkUrl(extension: ExtensionDataClass): String { - return "${extension.repo!!.repoUrlReplace()}/apk/${extension.apkName}" + fun getApkUrl( + repo: String, + apkName: String, + ): String { + return "${repo}apk/$apkName" } private val client by lazy { @@ -125,22 +108,4 @@ object ExtensionGithubApi { ) } } - - private fun String.repoUrlReplace() = - replace(repoMatchRegex) { - "https://raw.githubusercontent.com/${it.groupValues[1]}/${it.groupValues[2]}/" + - "${it.groupValues.getOrNull(3)?.ifBlank { null } ?: "repo"}/" - } - - private fun String.fallbackRepoUrlReplace() = - replace(repoMatchRegex) { - "https://gcore.jsdelivr.net/gh/${it.groupValues[1]}/${it.groupValues[2]}@" + - "${it.groupValues.getOrNull(3)?.ifBlank { null } ?: "repo"}/" - } - - private val repoMatchRegex = - ( - "https:\\/\\/(?:www|raw)?(?:github|githubusercontent)\\.com" + - "\\/([^\\/]+)\\/([^\\/]+)(?:\\/(?:tree|blob)\\/(.*))?\\/?" - ).toRegex() } diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/PackageTools.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/PackageTools.kt index 4b446650..7d54053c 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/PackageTools.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/PackageTools.kt @@ -43,9 +43,6 @@ object PackageTools { const val LIB_VERSION_MIN = 1.3 const val LIB_VERSION_MAX = 1.5 - private const val OFFICIAL_SIGNATURE = "7ce04da7773d41b489f4693a366c36bcd0a11fc39b547168553c285bd7348e23" // inorichi's key - val trustedSignatures = setOf(OFFICIAL_SIGNATURE) - /** * Convert dex to jar, a wrapper for the dex2jar library */