From c0df7d314b4c15624446615115dbf9cb2f461449 Mon Sep 17 00:00:00 2001 From: Syer10 Date: Sat, 3 Apr 2021 16:42:13 -0400 Subject: [PATCH 1/5] Add initial testing suit --- .gitignore | 2 + build.gradle.kts | 1 + server/build.gradle.kts | 6 + .../ir/armor/tachidesk/impl/Extension.kt | 13 +- .../kotlin/ir/armor/tachidesk/impl/Manga.kt | 6 +- .../kotlin/ir/armor/tachidesk/impl/Page.kt | 6 +- .../kotlin/ir/armor/tachidesk/impl/Source.kt | 4 +- .../tachidesk/impl/util/GetHttpSource.kt | 6 +- .../armor/tachidesk/impl/util/PackageTools.kt | 6 +- .../tachidesk/model/dataclass/DBMangaer.kt | 6 +- .../model/dataclass/SourceDataClass.kt | 2 +- .../ir/armor/tachidesk/server/ServerSetup.kt | 63 +++--- .../ir/armor/tachidesk/TestExtensions.kt | 186 ++++++++++++++++++ .../kotlin/ir/armor/tachidesk/TestUtils.kt | 44 +++++ .../src/test/resources/server-reference.conf | 13 ++ 15 files changed, 327 insertions(+), 37 deletions(-) create mode 100644 server/src/test/kotlin/ir/armor/tachidesk/TestExtensions.kt create mode 100644 server/src/test/kotlin/ir/armor/tachidesk/TestUtils.kt create mode 100644 server/src/test/resources/server-reference.conf diff --git a/.gitignore b/.gitignore index 7e3ffdf7..54736099 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,5 @@ gradle.properties build server/src/main/resources/react +server/tmp/ +server/tachiserver-data/ \ No newline at end of file diff --git a/build.gradle.kts b/build.gradle.kts index a8dcfe6f..3a1bc601 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -53,6 +53,7 @@ configure(projects) { val coroutinesVersion = "1.4.2" implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:$coroutinesVersion") implementation("org.jetbrains.kotlinx:kotlinx-coroutines-jdk8:$coroutinesVersion") + implementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:$coroutinesVersion") // Dependency Injection diff --git a/server/build.gradle.kts b/server/build.gradle.kts index 3048893f..6b2dfb75 100644 --- a/server/build.gradle.kts +++ b/server/build.gradle.kts @@ -80,6 +80,9 @@ dependencies { // uncomment to test extensions directly // implementation(fileTree("lib/")) + + // Testing + testImplementation(kotlin("test-junit5")) } val name = "ir.armor.tachidesk.Main" @@ -137,6 +140,9 @@ tasks { ) } } + test { + useJUnit() + } } launch4j { //used for windows diff --git a/server/src/main/kotlin/ir/armor/tachidesk/impl/Extension.kt b/server/src/main/kotlin/ir/armor/tachidesk/impl/Extension.kt index f7988576..b90bb6df 100644 --- a/server/src/main/kotlin/ir/armor/tachidesk/impl/Extension.kt +++ b/server/src/main/kotlin/ir/armor/tachidesk/impl/Extension.kt @@ -39,12 +39,17 @@ import org.jetbrains.exposed.sql.insert import org.jetbrains.exposed.sql.select import org.jetbrains.exposed.sql.transactions.transaction import org.jetbrains.exposed.sql.update +import org.kodein.di.DI +import org.kodein.di.conf.global +import org.kodein.di.instance import uy.kohesive.injekt.injectLazy import java.io.File import java.io.InputStream object Extension { private val logger = KotlinLogging.logger {} + private val dirs by DI.global.instance() + data class InstallableAPK( val apkFilePath: String, @@ -58,7 +63,7 @@ object Extension { return installAPK { val apkURL = ExtensionGithubApi.getApkUrl(extensionRecord) val apkName = Uri.parse(apkURL).lastPathSegment!! - val apkSavePath = "${ApplicationDirs.extensionsRoot}/$apkName" + val apkSavePath = "${dirs.extensionsRoot}/$apkName" // download apk file downloadAPKFile(apkURL, apkSavePath) @@ -79,7 +84,7 @@ object Extension { if (!isInstalled) { val fileNameWithoutType = apkName.substringBefore(".apk") - val dirPathWithoutType = "${ApplicationDirs.extensionsRoot}/$fileNameWithoutType" + val dirPathWithoutType = "${dirs.extensionsRoot}/$fileNameWithoutType" val jarFilePath = "$dirPathWithoutType.jar" val dexFilePath = "$dirPathWithoutType.dex" @@ -193,7 +198,7 @@ object Extension { val extensionRecord = transaction { ExtensionTable.select { ExtensionTable.pkgName eq pkgName }.firstOrNull()!! } val fileNameWithoutType = extensionRecord[ExtensionTable.apkName].substringBefore(".apk") - val jarPath = "${ApplicationDirs.extensionsRoot}/$fileNameWithoutType.jar" + val jarPath = "${dirs.extensionsRoot}/$fileNameWithoutType.jar" transaction { val extensionId = extensionRecord[ExtensionTable.id].value @@ -232,7 +237,7 @@ object Extension { suspend fun getExtensionIcon(apkName: String): Pair { val iconUrl = transaction { ExtensionTable.select { ExtensionTable.apkName eq apkName }.firstOrNull()!! }[ExtensionTable.iconUrl] - val saveDir = "${ApplicationDirs.extensionsRoot}/icon" + val saveDir = "${dirs.extensionsRoot}/icon" return getCachedImageResponse(saveDir, apkName) { network.client.newCall( diff --git a/server/src/main/kotlin/ir/armor/tachidesk/impl/Manga.kt b/server/src/main/kotlin/ir/armor/tachidesk/impl/Manga.kt index 25b696b2..09a41cac 100644 --- a/server/src/main/kotlin/ir/armor/tachidesk/impl/Manga.kt +++ b/server/src/main/kotlin/ir/armor/tachidesk/impl/Manga.kt @@ -22,6 +22,9 @@ import ir.armor.tachidesk.server.ApplicationDirs import org.jetbrains.exposed.sql.select import org.jetbrains.exposed.sql.transactions.transaction import org.jetbrains.exposed.sql.update +import org.kodein.di.DI +import org.kodein.di.conf.global +import org.kodein.di.instance import java.io.InputStream object Manga { @@ -95,9 +98,10 @@ object Manga { } } + private val dirs by DI.global.instance() suspend fun getMangaThumbnail(mangaId: Int): Pair { val mangaEntry = transaction { MangaTable.select { MangaTable.id eq mangaId }.firstOrNull()!! } - val saveDir = ApplicationDirs.thumbnailsRoot + val saveDir = dirs.thumbnailsRoot val fileName = mangaId.toString() return getCachedImageResponse(saveDir, fileName) { diff --git a/server/src/main/kotlin/ir/armor/tachidesk/impl/Page.kt b/server/src/main/kotlin/ir/armor/tachidesk/impl/Page.kt index aff86daa..d8c0db43 100644 --- a/server/src/main/kotlin/ir/armor/tachidesk/impl/Page.kt +++ b/server/src/main/kotlin/ir/armor/tachidesk/impl/Page.kt @@ -21,6 +21,9 @@ import org.jetbrains.exposed.sql.and import org.jetbrains.exposed.sql.select import org.jetbrains.exposed.sql.transactions.transaction import org.jetbrains.exposed.sql.update +import org.kodein.di.DI +import org.kodein.di.conf.global +import org.kodein.di.instance import java.io.File import java.io.InputStream @@ -73,6 +76,7 @@ object Page { } // TODO: rewrite this to match tachiyomi + private val dirs by DI.global.instance() fun getChapterDir(mangaId: Int, chapterId: Int): String { val mangaEntry = transaction { MangaTable.select { MangaTable.id eq mangaId }.firstOrNull()!! } val sourceId = mangaEntry[MangaTable.sourceReference] @@ -88,7 +92,7 @@ object Page { val mangaTitle = mangaEntry[MangaTable.title] val sourceName = source.toString() - val mangaDir = "${ApplicationDirs.mangaRoot}/$sourceName/$mangaTitle/$chapterDir" + val mangaDir = "${dirs.mangaRoot}/$sourceName/$mangaTitle/$chapterDir" // make sure dirs exist File(mangaDir).mkdirs() return mangaDir diff --git a/server/src/main/kotlin/ir/armor/tachidesk/impl/Source.kt b/server/src/main/kotlin/ir/armor/tachidesk/impl/Source.kt index 9fec2bf1..d7e9f48b 100644 --- a/server/src/main/kotlin/ir/armor/tachidesk/impl/Source.kt +++ b/server/src/main/kotlin/ir/armor/tachidesk/impl/Source.kt @@ -24,7 +24,7 @@ object Source { return transaction { SourceTable.selectAll().map { SourceDataClass( - it[SourceTable.id].value.toString(), + it[SourceTable.id].value, it[SourceTable.name], it[SourceTable.lang], getExtensionIconUrl(ExtensionTable.select { ExtensionTable.id eq it[SourceTable.extension] }.first()[ExtensionTable.apkName]), @@ -39,7 +39,7 @@ object Source { val source = SourceTable.select { SourceTable.id eq sourceId }.firstOrNull() SourceDataClass( - sourceId.toString(), + sourceId, source?.get(SourceTable.name), source?.get(SourceTable.lang), source?.let { ExtensionTable.select { ExtensionTable.id eq source[SourceTable.extension] }.first()[ExtensionTable.iconUrl] }, diff --git a/server/src/main/kotlin/ir/armor/tachidesk/impl/util/GetHttpSource.kt b/server/src/main/kotlin/ir/armor/tachidesk/impl/util/GetHttpSource.kt index 41eb4ec9..62c6b0f9 100644 --- a/server/src/main/kotlin/ir/armor/tachidesk/impl/util/GetHttpSource.kt +++ b/server/src/main/kotlin/ir/armor/tachidesk/impl/util/GetHttpSource.kt @@ -16,10 +16,14 @@ import ir.armor.tachidesk.model.database.SourceTable import ir.armor.tachidesk.server.ApplicationDirs import org.jetbrains.exposed.sql.select import org.jetbrains.exposed.sql.transactions.transaction +import org.kodein.di.DI +import org.kodein.di.conf.global +import org.kodein.di.instance import java.util.concurrent.ConcurrentHashMap object GetHttpSource { private val sourceCache = ConcurrentHashMap() + private val dirs by DI.global.instance() fun getHttpSource(sourceId: Long): HttpSource { val cachedResult: HttpSource? = sourceCache[sourceId] @@ -39,7 +43,7 @@ object GetHttpSource { val apkName = extensionRecord[ExtensionTable.apkName] val className = extensionRecord[ExtensionTable.classFQName] val jarName = apkName.substringBefore(".apk") + ".jar" - val jarPath = "${ApplicationDirs.extensionsRoot}/$jarName" + val jarPath = "${dirs.extensionsRoot}/$jarName" when (val instance = loadExtensionSources(jarPath, className)) { is Source -> listOf(instance) diff --git a/server/src/main/kotlin/ir/armor/tachidesk/impl/util/PackageTools.kt b/server/src/main/kotlin/ir/armor/tachidesk/impl/util/PackageTools.kt index a2177dca..3b989b66 100644 --- a/server/src/main/kotlin/ir/armor/tachidesk/impl/util/PackageTools.kt +++ b/server/src/main/kotlin/ir/armor/tachidesk/impl/util/PackageTools.kt @@ -11,6 +11,9 @@ import ir.armor.tachidesk.server.ApplicationDirs import mu.KotlinLogging import net.dongliu.apk.parser.ApkFile import net.dongliu.apk.parser.ApkParsers +import org.kodein.di.DI +import org.kodein.di.conf.global +import org.kodein.di.instance import org.w3c.dom.Element import org.w3c.dom.Node import xyz.nulldev.androidcompat.pm.InstalledPackage.Companion.toList @@ -31,6 +34,7 @@ import javax.xml.parsers.DocumentBuilderFactory object PackageTools { private val logger = KotlinLogging.logger {} + private val dirs by DI.global.instance() const val EXTENSION_FEATURE = "tachiyomi.extension" const val METADATA_SOURCE_CLASS = "tachiyomi.extension.class" @@ -65,7 +69,7 @@ object PackageTools { .skipExceptions(false) .to(jarFilePath) if (handler.hasException()) { - val errorFile: Path = File(ApplicationDirs.extensionsRoot).toPath().resolve("$fileNameWithoutType-error.txt") + val errorFile: Path = File(dirs.extensionsRoot).toPath().resolve("$fileNameWithoutType-error.txt") logger.error( "Detail Error Information in File $errorFile\n" + "Please report this file to one of following link if possible (any one).\n" + diff --git a/server/src/main/kotlin/ir/armor/tachidesk/model/dataclass/DBMangaer.kt b/server/src/main/kotlin/ir/armor/tachidesk/model/dataclass/DBMangaer.kt index ec971dc9..1255d2f3 100644 --- a/server/src/main/kotlin/ir/armor/tachidesk/model/dataclass/DBMangaer.kt +++ b/server/src/main/kotlin/ir/armor/tachidesk/model/dataclass/DBMangaer.kt @@ -18,10 +18,14 @@ import ir.armor.tachidesk.server.ApplicationDirs import org.jetbrains.exposed.sql.Database import org.jetbrains.exposed.sql.SchemaUtils import org.jetbrains.exposed.sql.transactions.transaction +import org.kodein.di.DI +import org.kodein.di.conf.global +import org.kodein.di.instance object DBMangaer { val db by lazy { - Database.connect("jdbc:h2:${ApplicationDirs.dataRoot}/database", "org.h2.Driver") + val dirs by DI.global.instance() + Database.connect("jdbc:h2:${dirs.dataRoot}/database", "org.h2.Driver") } } diff --git a/server/src/main/kotlin/ir/armor/tachidesk/model/dataclass/SourceDataClass.kt b/server/src/main/kotlin/ir/armor/tachidesk/model/dataclass/SourceDataClass.kt index 6e225842..17df592b 100644 --- a/server/src/main/kotlin/ir/armor/tachidesk/model/dataclass/SourceDataClass.kt +++ b/server/src/main/kotlin/ir/armor/tachidesk/model/dataclass/SourceDataClass.kt @@ -8,7 +8,7 @@ package ir.armor.tachidesk.model.dataclass * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ data class SourceDataClass( - val id: String, + val id: Long, val name: String?, val lang: String?, val iconUrl: String?, diff --git a/server/src/main/kotlin/ir/armor/tachidesk/server/ServerSetup.kt b/server/src/main/kotlin/ir/armor/tachidesk/server/ServerSetup.kt index f8b03503..49567428 100644 --- a/server/src/main/kotlin/ir/armor/tachidesk/server/ServerSetup.kt +++ b/server/src/main/kotlin/ir/armor/tachidesk/server/ServerSetup.kt @@ -8,6 +8,7 @@ package ir.armor.tachidesk.server * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ import ch.qos.logback.classic.Level +import com.typesafe.config.Config import eu.kanade.tachiyomi.App import ir.armor.tachidesk.Main import ir.armor.tachidesk.model.dataclass.makeDataBaseTables @@ -15,7 +16,11 @@ import ir.armor.tachidesk.server.util.systemTray import mu.KotlinLogging import net.harawata.appdirs.AppDirsFactory import org.kodein.di.DI +import org.kodein.di.bind import org.kodein.di.conf.global +import org.kodein.di.instance +import org.kodein.di.singleton +import org.slf4j.Logger import xyz.nulldev.androidcompat.AndroidCompat import xyz.nulldev.androidcompat.AndroidCompatInitializer import xyz.nulldev.ts.config.ConfigKodeinModule @@ -24,43 +29,57 @@ import java.io.File private val logger = KotlinLogging.logger {} -object ApplicationDirs { - val dataRoot = AppDirsFactory.getInstance().getUserDataDir("Tachidesk", null, null)!! +class ApplicationDirs( + val dataRoot: String = AppDirsFactory.getInstance().getUserDataDir("Tachidesk", null, null) +) { val extensionsRoot = "$dataRoot/extensions" val thumbnailsRoot = "$dataRoot/thumbnails" val mangaRoot = "$dataRoot/manga" } -val serverConfig: ServerConfig by lazy { GlobalConfigManager.module() } +val serverConfig: ServerConfig by DI.global.instance() val systemTray by lazy { systemTray() } val androidCompat by lazy { AndroidCompat() } -fun applicationSetup() { - // register server config - GlobalConfigManager.registerModule( - ServerConfig.register(GlobalConfigManager.config) - ) - - // set application wide logging level - if (serverConfig.debugLogsEnabled) { - (mu.KotlinLogging.logger(org.slf4j.Logger.ROOT_LOGGER_NAME).underlyingLogger as ch.qos.logback.classic.Logger).level = Level.DEBUG +fun applicationSetup(rootDir: String? = null, config: Config = GlobalConfigManager.config) { + val dirs = if (rootDir != null) { + ApplicationDirs(rootDir) + } else { + ApplicationDirs() } // make dirs we need listOf( - ApplicationDirs.dataRoot, - ApplicationDirs.extensionsRoot, - "${ApplicationDirs.extensionsRoot}/icon", - ApplicationDirs.thumbnailsRoot + dirs.dataRoot, + dirs.extensionsRoot, + dirs.extensionsRoot + "/icon", + dirs.thumbnailsRoot ).forEach { File(it).mkdirs() } + // Application dirs + DI.global.addImport(DI.Module("Server") { + bind() with singleton { dirs } + bind() with singleton { ServerConfig.register(config) } + }) + // Load config API + DI.global.addImport(ConfigKodeinModule().create()) + // Load Android compatibility dependencies + AndroidCompatInitializer().init() + // start app + androidCompat.startApp(App()) + + // set application wide logging level + if (serverConfig.debugLogsEnabled) { + (KotlinLogging.logger(Logger.ROOT_LOGGER_NAME).underlyingLogger as ch.qos.logback.classic.Logger).level = Level.DEBUG + } + // create conf file if doesn't exist try { - val dataConfFile = File("${ApplicationDirs.dataRoot}/server.conf") + val dataConfFile = File("${dirs.dataRoot}/server.conf") if (!dataConfFile.exists()) { Main::class.java.getResourceAsStream("/server-reference.conf").use { input -> dataConfFile.outputStream().use { output -> @@ -75,19 +94,13 @@ fun applicationSetup() { makeDataBaseTables() // create system tray - if (serverConfig.systemTrayEnabled) + if (serverConfig.systemTrayEnabled) { try { systemTray } catch (e: Exception) { e.printStackTrace() } - - // Load config API - DI.global.addImport(ConfigKodeinModule().create()) - // Load Android compatibility dependencies - AndroidCompatInitializer().init() - // start app - androidCompat.startApp(App()) + } // Disable jetty's logging System.setProperty("org.eclipse.jetty.util.log.announce", "false") diff --git a/server/src/test/kotlin/ir/armor/tachidesk/TestExtensions.kt b/server/src/test/kotlin/ir/armor/tachidesk/TestExtensions.kt new file mode 100644 index 00000000..cc3aad6e --- /dev/null +++ b/server/src/test/kotlin/ir/armor/tachidesk/TestExtensions.kt @@ -0,0 +1,186 @@ +package ir.armor.tachidesk + +import eu.kanade.tachiyomi.source.model.SChapter +import eu.kanade.tachiyomi.source.model.SManga +import eu.kanade.tachiyomi.source.online.HttpSource +import ir.armor.tachidesk.impl.Extension.installExtension +import ir.armor.tachidesk.impl.Extension.uninstallExtension +import ir.armor.tachidesk.impl.Extension.updateExtension +import ir.armor.tachidesk.impl.ExtensionsList.getExtensionList +import ir.armor.tachidesk.impl.Source.getSourceList +import ir.armor.tachidesk.impl.util.GetHttpSource.getHttpSource +import ir.armor.tachidesk.impl.util.awaitSingle +import ir.armor.tachidesk.model.dataclass.ExtensionDataClass +import ir.armor.tachidesk.server.applicationSetup +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.sync.Semaphore +import kotlinx.coroutines.sync.withPermit +import mu.KotlinLogging +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.TestInstance +import rx.Observable +import java.io.File + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class TestExtensions { + private val logger = KotlinLogging.logger {} + private lateinit var extensions: List + private lateinit var sources: List + + private val mangaToFetch = mutableListOf>() + private val failedToFetch = mutableListOf>() + private val mangaFailedToFetch = mutableListOf>() + private val chaptersToFetch = mutableListOf>() + private val chaptersFailedToFetch = mutableListOf>() + private val chaptersPageListFailedToFetch = mutableListOf, Exception>>() + + @BeforeAll + fun setup() { + val dataRoot = File("tmp/TestDesk").absolutePath + applicationSetup(dataRoot, loadConfigs(dataRoot)) + setLoggingEnabled(false) + runBlocking { + extensions = getExtensionList() + extensions.forEach { + when { + it.obsolete -> { + uninstallExtension(it.pkgName) + } + it.hasUpdate -> { + updateExtension(it.pkgName) + } + else -> { + installExtension(it.pkgName) + } + } + } + sources = getSourceList().map { getHttpSource(it.id) } + } + setLoggingEnabled(true) + File("tmp/TestDesk/sources.txt").writeText(sources.joinToString("\n") { "${it.name} - ${it.lang.toUpperCase()} - ${it.id}" }) + } + + @Test + fun runTest() { + runBlocking(Dispatchers.Default) { + val semaphore = Semaphore(10) + sources.mapIndexed { index, source -> + async { + semaphore.withPermit { + logger.info { "$index - Now fetching popular manga from $source" } + try { + mangaToFetch += source to (source.fetchPopularManga(1) + .awaitSingleRepeat().mangas.firstOrNull() + ?: throw Exception("Source returned no manga")) + } catch (e: Exception) { + logger.warn { "Failed to fetch popular manga from $source: ${e.message}" } + failedToFetch += source to e + } + } + } + }.awaitAll() + File("tmp/TestDesk/failedToFetch.txt").writeText( + failedToFetch.joinToString("\n") { (source, exception) -> + "${source.name} (${source.lang.toUpperCase()}, ${source.id}):" + + " ${exception.message}" + } + ) + logger.info { "Now fetching manga info from ${mangaToFetch.size} sources" } + + mangaToFetch.mapIndexed { index, (source, manga) -> + async { + semaphore.withPermit { + logger.info { "$index - Now fetching manga from $source" } + try { + manga.copyFrom(source.fetchMangaDetails(manga).awaitSingleRepeat()) + manga.initialized = true + } catch (e: Exception) { + logger.warn { + "Failed to fetch manga info from $source for ${manga.title} (${source.mangaDetailsRequest(manga).url}): ${e.message}" + } + mangaFailedToFetch += Triple(source, manga, e) + } + } + } + }.awaitAll() + File("tmp/TestDesk/MangaFailedToFetch.txt").writeText( + mangaFailedToFetch.joinToString("\n") { (source, manga, exception) -> + "${source.name} (${source.lang}, ${source.id}):" + + " ${manga.title} (${source.mangaDetailsRequest(manga).url}):" + + " ${exception.message}" + } + ) + logger.info { "Now fetching manga chapters from ${mangaToFetch.size} sources" } + + mangaToFetch.filter { it.second.initialized }.mapIndexed { index, (source, manga) -> + async { + semaphore.withPermit { + logger.info { "$index - Now fetching manga chapters from $source" } + try { + chaptersToFetch += Triple( + source, + manga, + source.fetchChapterList(manga).awaitSingleRepeat().firstOrNull() ?: throw Exception("Source returned no chapters") + ) + } catch (e: Exception) { + logger.warn { + "Failed to fetch manga chapters from $source for ${manga.title} (${source.mangaDetailsRequest(manga).url}): ${e.message}" + } + chaptersFailedToFetch += Triple(source, manga, e) + } catch (e: NoClassDefFoundError) { + logger.warn { + "Failed to fetch manga chapters from $source for ${manga.title} (${source.mangaDetailsRequest(manga).url}): ${e.message}" + } + chaptersFailedToFetch += Triple(source, manga, e) + } + } + } + }.awaitAll() + + File("tmp/TestDesk/ChaptersFailedToFetch.txt").writeText( + chaptersFailedToFetch.joinToString("\n") { (source, manga, exception) -> + "${source.name} (${source.lang}, ${source.id}):" + + " ${manga.title} (${source.mangaDetailsRequest(manga).url}):" + + " ${exception.message}" + } + ) + + chaptersToFetch.mapIndexed { index, (source, manga, chapter) -> + async { + semaphore.withPermit { + logger.info { "$index - Now fetching page list from $source" } + try { + source.fetchPageList(chapter).awaitSingleRepeat() + } catch (e: Exception) { + logger.warn { + "Failed to fetch manga info from $source for ${manga.title} (${source.mangaDetailsRequest(manga).url}): ${e.message}" + } + chaptersPageListFailedToFetch += Triple(source, manga to chapter, e) + } + } + } + }.awaitAll() + + File("tmp/TestDesk/ChapterPageListFailedToFetch.txt").writeText( + chaptersPageListFailedToFetch.joinToString("\n") { (source, manga, exception) -> + "${source.name} (${source.lang}, ${source.id}):" + + " ${manga.first.title} (${source.mangaDetailsRequest(manga.first).url}):" + + " ${manga.second.name} (${manga.second.url}): ${exception.message}" + } + ) + } + } + + private suspend fun Observable.awaitSingleRepeat(): T { + for (i in 1..2) { + try { + return awaitSingle() + } catch (e: Exception) {} + } + return awaitSingle() + } +} \ No newline at end of file diff --git a/server/src/test/kotlin/ir/armor/tachidesk/TestUtils.kt b/server/src/test/kotlin/ir/armor/tachidesk/TestUtils.kt new file mode 100644 index 00000000..dab9e378 --- /dev/null +++ b/server/src/test/kotlin/ir/armor/tachidesk/TestUtils.kt @@ -0,0 +1,44 @@ +package ir.armor.tachidesk + +import ch.qos.logback.classic.Level +import com.typesafe.config.Config +import com.typesafe.config.ConfigFactory +import com.typesafe.config.ConfigRenderOptions +import mu.KotlinLogging +import org.slf4j.Logger +import java.io.File + +/** + * Load configs + */ +fun loadConfigs(dataRoot: String): Config { + val logger = KotlinLogging.logger {} + //Load reference configs + val compatConfig = ConfigFactory.parseResources("compat-reference.conf") + val serverConfig = ConfigFactory.parseResources("server-reference.conf") + + //Load user config + val userConfig = + File(dataRoot, "server.conf").let { + ConfigFactory.parseFile(it) + } + + val config = ConfigFactory.empty() + .withFallback(userConfig) + .withFallback(compatConfig) + .withFallback(serverConfig) + .resolve() + + logger.debug { + "Loaded config:\n" + config.root().render(ConfigRenderOptions.concise().setFormatted(true)) + } + + return config +} + +fun setLoggingEnabled(enabled: Boolean = true) { + val logger = (KotlinLogging.logger(Logger.ROOT_LOGGER_NAME).underlyingLogger as ch.qos.logback.classic.Logger) + logger.level = if (enabled) { + Level.DEBUG + } else Level.ERROR +} \ No newline at end of file diff --git a/server/src/test/resources/server-reference.conf b/server/src/test/resources/server-reference.conf new file mode 100644 index 00000000..9e49ccd8 --- /dev/null +++ b/server/src/test/resources/server-reference.conf @@ -0,0 +1,13 @@ +# Server ip and port bindings +server.ip = "0.0.0.0" +server.port = 4567 + +# Socks5 proxy +server.socksProxy = false +server.socksProxyHost = "" +server.socksProxyPort = "" + +# misc +server.debugLogsEnabled = true +server.systemTrayEnabled = false +server.initialOpenInBrowserEnabled = true From a211a4143be90cedb572e746e975ab975bb009a3 Mon Sep 17 00:00:00 2001 From: Syer10 Date: Sat, 3 Apr 2021 16:57:03 -0400 Subject: [PATCH 2/5] Revert source id to long --- server/src/main/kotlin/ir/armor/tachidesk/impl/Source.kt | 4 ++-- .../ir/armor/tachidesk/model/dataclass/SourceDataClass.kt | 2 +- server/src/test/kotlin/ir/armor/tachidesk/TestExtensions.kt | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/server/src/main/kotlin/ir/armor/tachidesk/impl/Source.kt b/server/src/main/kotlin/ir/armor/tachidesk/impl/Source.kt index d7e9f48b..9fec2bf1 100644 --- a/server/src/main/kotlin/ir/armor/tachidesk/impl/Source.kt +++ b/server/src/main/kotlin/ir/armor/tachidesk/impl/Source.kt @@ -24,7 +24,7 @@ object Source { return transaction { SourceTable.selectAll().map { SourceDataClass( - it[SourceTable.id].value, + it[SourceTable.id].value.toString(), it[SourceTable.name], it[SourceTable.lang], getExtensionIconUrl(ExtensionTable.select { ExtensionTable.id eq it[SourceTable.extension] }.first()[ExtensionTable.apkName]), @@ -39,7 +39,7 @@ object Source { val source = SourceTable.select { SourceTable.id eq sourceId }.firstOrNull() SourceDataClass( - sourceId, + sourceId.toString(), source?.get(SourceTable.name), source?.get(SourceTable.lang), source?.let { ExtensionTable.select { ExtensionTable.id eq source[SourceTable.extension] }.first()[ExtensionTable.iconUrl] }, diff --git a/server/src/main/kotlin/ir/armor/tachidesk/model/dataclass/SourceDataClass.kt b/server/src/main/kotlin/ir/armor/tachidesk/model/dataclass/SourceDataClass.kt index 17df592b..6e225842 100644 --- a/server/src/main/kotlin/ir/armor/tachidesk/model/dataclass/SourceDataClass.kt +++ b/server/src/main/kotlin/ir/armor/tachidesk/model/dataclass/SourceDataClass.kt @@ -8,7 +8,7 @@ package ir.armor.tachidesk.model.dataclass * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ data class SourceDataClass( - val id: Long, + val id: String, val name: String?, val lang: String?, val iconUrl: String?, diff --git a/server/src/test/kotlin/ir/armor/tachidesk/TestExtensions.kt b/server/src/test/kotlin/ir/armor/tachidesk/TestExtensions.kt index cc3aad6e..000ad356 100644 --- a/server/src/test/kotlin/ir/armor/tachidesk/TestExtensions.kt +++ b/server/src/test/kotlin/ir/armor/tachidesk/TestExtensions.kt @@ -58,7 +58,7 @@ class TestExtensions { } } } - sources = getSourceList().map { getHttpSource(it.id) } + sources = getSourceList().map { getHttpSource(it.id.toLong()) } } setLoggingEnabled(true) File("tmp/TestDesk/sources.txt").writeText(sources.joinToString("\n") { "${it.name} - ${it.lang.toUpperCase()} - ${it.id}" }) From b2d5354798355008aa922e3c77632480ab52261c Mon Sep 17 00:00:00 2001 From: Syer10 Date: Sat, 3 Apr 2021 17:25:53 -0400 Subject: [PATCH 3/5] dirs -> applicationDirs --- .../main/kotlin/ir/armor/tachidesk/impl/Extension.kt | 10 +++++----- .../src/main/kotlin/ir/armor/tachidesk/impl/Manga.kt | 4 ++-- server/src/main/kotlin/ir/armor/tachidesk/impl/Page.kt | 4 ++-- .../ir/armor/tachidesk/impl/util/GetHttpSource.kt | 4 ++-- .../ir/armor/tachidesk/impl/util/PackageTools.kt | 4 ++-- .../ir/armor/tachidesk/model/dataclass/DBMangaer.kt | 4 ++-- 6 files changed, 15 insertions(+), 15 deletions(-) diff --git a/server/src/main/kotlin/ir/armor/tachidesk/impl/Extension.kt b/server/src/main/kotlin/ir/armor/tachidesk/impl/Extension.kt index b90bb6df..3adef0b4 100644 --- a/server/src/main/kotlin/ir/armor/tachidesk/impl/Extension.kt +++ b/server/src/main/kotlin/ir/armor/tachidesk/impl/Extension.kt @@ -48,7 +48,7 @@ import java.io.InputStream object Extension { private val logger = KotlinLogging.logger {} - private val dirs by DI.global.instance() + private val applicationDirs by DI.global.instance() data class InstallableAPK( @@ -63,7 +63,7 @@ object Extension { return installAPK { val apkURL = ExtensionGithubApi.getApkUrl(extensionRecord) val apkName = Uri.parse(apkURL).lastPathSegment!! - val apkSavePath = "${dirs.extensionsRoot}/$apkName" + val apkSavePath = "${applicationDirs.extensionsRoot}/$apkName" // download apk file downloadAPKFile(apkURL, apkSavePath) @@ -84,7 +84,7 @@ object Extension { if (!isInstalled) { val fileNameWithoutType = apkName.substringBefore(".apk") - val dirPathWithoutType = "${dirs.extensionsRoot}/$fileNameWithoutType" + val dirPathWithoutType = "${applicationDirs.extensionsRoot}/$fileNameWithoutType" val jarFilePath = "$dirPathWithoutType.jar" val dexFilePath = "$dirPathWithoutType.dex" @@ -198,7 +198,7 @@ object Extension { val extensionRecord = transaction { ExtensionTable.select { ExtensionTable.pkgName eq pkgName }.firstOrNull()!! } val fileNameWithoutType = extensionRecord[ExtensionTable.apkName].substringBefore(".apk") - val jarPath = "${dirs.extensionsRoot}/$fileNameWithoutType.jar" + val jarPath = "${applicationDirs.extensionsRoot}/$fileNameWithoutType.jar" transaction { val extensionId = extensionRecord[ExtensionTable.id].value @@ -237,7 +237,7 @@ object Extension { suspend fun getExtensionIcon(apkName: String): Pair { val iconUrl = transaction { ExtensionTable.select { ExtensionTable.apkName eq apkName }.firstOrNull()!! }[ExtensionTable.iconUrl] - val saveDir = "${dirs.extensionsRoot}/icon" + val saveDir = "${applicationDirs.extensionsRoot}/icon" return getCachedImageResponse(saveDir, apkName) { network.client.newCall( diff --git a/server/src/main/kotlin/ir/armor/tachidesk/impl/Manga.kt b/server/src/main/kotlin/ir/armor/tachidesk/impl/Manga.kt index 09a41cac..4e3071b9 100644 --- a/server/src/main/kotlin/ir/armor/tachidesk/impl/Manga.kt +++ b/server/src/main/kotlin/ir/armor/tachidesk/impl/Manga.kt @@ -98,10 +98,10 @@ object Manga { } } - private val dirs by DI.global.instance() + private val applicationDirs by DI.global.instance() suspend fun getMangaThumbnail(mangaId: Int): Pair { val mangaEntry = transaction { MangaTable.select { MangaTable.id eq mangaId }.firstOrNull()!! } - val saveDir = dirs.thumbnailsRoot + val saveDir = applicationDirs.thumbnailsRoot val fileName = mangaId.toString() return getCachedImageResponse(saveDir, fileName) { diff --git a/server/src/main/kotlin/ir/armor/tachidesk/impl/Page.kt b/server/src/main/kotlin/ir/armor/tachidesk/impl/Page.kt index d8c0db43..299ad8f1 100644 --- a/server/src/main/kotlin/ir/armor/tachidesk/impl/Page.kt +++ b/server/src/main/kotlin/ir/armor/tachidesk/impl/Page.kt @@ -76,7 +76,7 @@ object Page { } // TODO: rewrite this to match tachiyomi - private val dirs by DI.global.instance() + private val applicationDirs by DI.global.instance() fun getChapterDir(mangaId: Int, chapterId: Int): String { val mangaEntry = transaction { MangaTable.select { MangaTable.id eq mangaId }.firstOrNull()!! } val sourceId = mangaEntry[MangaTable.sourceReference] @@ -92,7 +92,7 @@ object Page { val mangaTitle = mangaEntry[MangaTable.title] val sourceName = source.toString() - val mangaDir = "${dirs.mangaRoot}/$sourceName/$mangaTitle/$chapterDir" + val mangaDir = "${applicationDirs.mangaRoot}/$sourceName/$mangaTitle/$chapterDir" // make sure dirs exist File(mangaDir).mkdirs() return mangaDir diff --git a/server/src/main/kotlin/ir/armor/tachidesk/impl/util/GetHttpSource.kt b/server/src/main/kotlin/ir/armor/tachidesk/impl/util/GetHttpSource.kt index 62c6b0f9..0c01ea9b 100644 --- a/server/src/main/kotlin/ir/armor/tachidesk/impl/util/GetHttpSource.kt +++ b/server/src/main/kotlin/ir/armor/tachidesk/impl/util/GetHttpSource.kt @@ -23,7 +23,7 @@ import java.util.concurrent.ConcurrentHashMap object GetHttpSource { private val sourceCache = ConcurrentHashMap() - private val dirs by DI.global.instance() + private val applicationDirs by DI.global.instance() fun getHttpSource(sourceId: Long): HttpSource { val cachedResult: HttpSource? = sourceCache[sourceId] @@ -43,7 +43,7 @@ object GetHttpSource { val apkName = extensionRecord[ExtensionTable.apkName] val className = extensionRecord[ExtensionTable.classFQName] val jarName = apkName.substringBefore(".apk") + ".jar" - val jarPath = "${dirs.extensionsRoot}/$jarName" + val jarPath = "${applicationDirs.extensionsRoot}/$jarName" when (val instance = loadExtensionSources(jarPath, className)) { is Source -> listOf(instance) diff --git a/server/src/main/kotlin/ir/armor/tachidesk/impl/util/PackageTools.kt b/server/src/main/kotlin/ir/armor/tachidesk/impl/util/PackageTools.kt index 3b989b66..44ea5650 100644 --- a/server/src/main/kotlin/ir/armor/tachidesk/impl/util/PackageTools.kt +++ b/server/src/main/kotlin/ir/armor/tachidesk/impl/util/PackageTools.kt @@ -34,7 +34,7 @@ import javax.xml.parsers.DocumentBuilderFactory object PackageTools { private val logger = KotlinLogging.logger {} - private val dirs by DI.global.instance() + private val applicationDirs by DI.global.instance() const val EXTENSION_FEATURE = "tachiyomi.extension" const val METADATA_SOURCE_CLASS = "tachiyomi.extension.class" @@ -69,7 +69,7 @@ object PackageTools { .skipExceptions(false) .to(jarFilePath) if (handler.hasException()) { - val errorFile: Path = File(dirs.extensionsRoot).toPath().resolve("$fileNameWithoutType-error.txt") + val errorFile: Path = File(applicationDirs.extensionsRoot).toPath().resolve("$fileNameWithoutType-error.txt") logger.error( "Detail Error Information in File $errorFile\n" + "Please report this file to one of following link if possible (any one).\n" + diff --git a/server/src/main/kotlin/ir/armor/tachidesk/model/dataclass/DBMangaer.kt b/server/src/main/kotlin/ir/armor/tachidesk/model/dataclass/DBMangaer.kt index 1255d2f3..2d5c1f76 100644 --- a/server/src/main/kotlin/ir/armor/tachidesk/model/dataclass/DBMangaer.kt +++ b/server/src/main/kotlin/ir/armor/tachidesk/model/dataclass/DBMangaer.kt @@ -24,8 +24,8 @@ import org.kodein.di.instance object DBMangaer { val db by lazy { - val dirs by DI.global.instance() - Database.connect("jdbc:h2:${dirs.dataRoot}/database", "org.h2.Driver") + val applicationDirs by DI.global.instance() + Database.connect("jdbc:h2:${applicationDirs.dataRoot}/database", "org.h2.Driver") } } From e043cb569099c9f2065b23e3af6c9461e96be9c3 Mon Sep 17 00:00:00 2001 From: Syer10 Date: Sat, 3 Apr 2021 18:12:01 -0400 Subject: [PATCH 4/5] Use properties to set rootDir so that ConfigManager can use it --- .../xyz/nulldev/ts/config/ConfigManager.kt | 10 ++---- .../ir/armor/tachidesk/server/ServerSetup.kt | 11 +++++-- .../ir/armor/tachidesk/TestExtensions.kt | 2 +- .../kotlin/ir/armor/tachidesk/TestUtils.kt | 32 ------------------- 4 files changed, 12 insertions(+), 43 deletions(-) diff --git a/AndroidCompat/Config/src/main/java/xyz/nulldev/ts/config/ConfigManager.kt b/AndroidCompat/Config/src/main/java/xyz/nulldev/ts/config/ConfigManager.kt index 59a43c1a..c22d7d49 100644 --- a/AndroidCompat/Config/src/main/java/xyz/nulldev/ts/config/ConfigManager.kt +++ b/AndroidCompat/Config/src/main/java/xyz/nulldev/ts/config/ConfigManager.kt @@ -18,8 +18,6 @@ import java.io.File * Manages app config. */ open class ConfigManager { - private val dataRoot by lazy { AppDirsFactory.getInstance().getUserDataDir("Tachidesk", null, null)!! } - private val generatedModules = mutableMapOf, ConfigModule>() val config by lazy { loadConfigs() } @@ -27,8 +25,6 @@ open class ConfigManager { val loadedModules: Map, ConfigModule> get() = generatedModules - open val appConfigFile: String = "$dataRoot/server.conf" - val logger = KotlinLogging.logger {} /** @@ -51,8 +47,8 @@ open class ConfigManager { //Load user config val userConfig = - File(appConfigFile).let{ - ConfigFactory.parseFile(it) + File(System.getProperty("ir.armor.tachidesk.rootDir"), "server.conf").let { + ConfigFactory.parseFile(it) } val config = ConfigFactory.empty() @@ -69,7 +65,7 @@ open class ConfigManager { } fun registerModule(module: ConfigModule) { - generatedModules.put(module.javaClass, module) + generatedModules[module.javaClass] = module } fun registerModules(vararg modules: ConfigModule) { diff --git a/server/src/main/kotlin/ir/armor/tachidesk/server/ServerSetup.kt b/server/src/main/kotlin/ir/armor/tachidesk/server/ServerSetup.kt index 49567428..935eec64 100644 --- a/server/src/main/kotlin/ir/armor/tachidesk/server/ServerSetup.kt +++ b/server/src/main/kotlin/ir/armor/tachidesk/server/ServerSetup.kt @@ -37,19 +37,21 @@ class ApplicationDirs( val mangaRoot = "$dataRoot/manga" } -val serverConfig: ServerConfig by DI.global.instance() +val serverConfig: ServerConfig by lazy { GlobalConfigManager.module() } val systemTray by lazy { systemTray() } val androidCompat by lazy { AndroidCompat() } -fun applicationSetup(rootDir: String? = null, config: Config = GlobalConfigManager.config) { +fun applicationSetup(rootDir: String? = null) { val dirs = if (rootDir != null) { ApplicationDirs(rootDir) } else { ApplicationDirs() } + System.setProperty("ir.armor.tachidesk.rootDir", dirs.dataRoot) + // make dirs we need listOf( dirs.dataRoot, @@ -60,10 +62,13 @@ fun applicationSetup(rootDir: String? = null, config: Config = GlobalConfigManag File(it).mkdirs() } + GlobalConfigManager.registerModule( + ServerConfig.register(GlobalConfigManager.config) + ) + // Application dirs DI.global.addImport(DI.Module("Server") { bind() with singleton { dirs } - bind() with singleton { ServerConfig.register(config) } }) // Load config API DI.global.addImport(ConfigKodeinModule().create()) diff --git a/server/src/test/kotlin/ir/armor/tachidesk/TestExtensions.kt b/server/src/test/kotlin/ir/armor/tachidesk/TestExtensions.kt index 000ad356..574bb9dd 100644 --- a/server/src/test/kotlin/ir/armor/tachidesk/TestExtensions.kt +++ b/server/src/test/kotlin/ir/armor/tachidesk/TestExtensions.kt @@ -41,7 +41,7 @@ class TestExtensions { @BeforeAll fun setup() { val dataRoot = File("tmp/TestDesk").absolutePath - applicationSetup(dataRoot, loadConfigs(dataRoot)) + applicationSetup(dataRoot) setLoggingEnabled(false) runBlocking { extensions = getExtensionList() diff --git a/server/src/test/kotlin/ir/armor/tachidesk/TestUtils.kt b/server/src/test/kotlin/ir/armor/tachidesk/TestUtils.kt index dab9e378..685a6ff0 100644 --- a/server/src/test/kotlin/ir/armor/tachidesk/TestUtils.kt +++ b/server/src/test/kotlin/ir/armor/tachidesk/TestUtils.kt @@ -1,40 +1,8 @@ package ir.armor.tachidesk import ch.qos.logback.classic.Level -import com.typesafe.config.Config -import com.typesafe.config.ConfigFactory -import com.typesafe.config.ConfigRenderOptions import mu.KotlinLogging import org.slf4j.Logger -import java.io.File - -/** - * Load configs - */ -fun loadConfigs(dataRoot: String): Config { - val logger = KotlinLogging.logger {} - //Load reference configs - val compatConfig = ConfigFactory.parseResources("compat-reference.conf") - val serverConfig = ConfigFactory.parseResources("server-reference.conf") - - //Load user config - val userConfig = - File(dataRoot, "server.conf").let { - ConfigFactory.parseFile(it) - } - - val config = ConfigFactory.empty() - .withFallback(userConfig) - .withFallback(compatConfig) - .withFallback(serverConfig) - .resolve() - - logger.debug { - "Loaded config:\n" + config.root().render(ConfigRenderOptions.concise().setFormatted(true)) - } - - return config -} fun setLoggingEnabled(enabled: Boolean = true) { val logger = (KotlinLogging.logger(Logger.ROOT_LOGGER_NAME).underlyingLogger as ch.qos.logback.classic.Logger) From c11887fada4bb4d619b5a734610f679162b1f907 Mon Sep 17 00:00:00 2001 From: Syer10 Date: Sat, 3 Apr 2021 18:35:30 -0400 Subject: [PATCH 5/5] Allow rootdir to be used as a argument --- .../main/kotlin/ir/armor/tachidesk/server/ServerSetup.kt | 9 +++++---- .../src/test/kotlin/ir/armor/tachidesk/TestExtensions.kt | 3 ++- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/server/src/main/kotlin/ir/armor/tachidesk/server/ServerSetup.kt b/server/src/main/kotlin/ir/armor/tachidesk/server/ServerSetup.kt index 935eec64..516b5418 100644 --- a/server/src/main/kotlin/ir/armor/tachidesk/server/ServerSetup.kt +++ b/server/src/main/kotlin/ir/armor/tachidesk/server/ServerSetup.kt @@ -43,15 +43,16 @@ val systemTray by lazy { systemTray() } val androidCompat by lazy { AndroidCompat() } -fun applicationSetup(rootDir: String? = null) { +fun applicationSetup() { + val rootDir: String? = System.getProperty("ir.armor.tachidesk.rootDir") val dirs = if (rootDir != null) { ApplicationDirs(rootDir) } else { - ApplicationDirs() + ApplicationDirs().also { + System.setProperty("ir.armor.tachidesk.rootDir", it.dataRoot) + } } - System.setProperty("ir.armor.tachidesk.rootDir", dirs.dataRoot) - // make dirs we need listOf( dirs.dataRoot, diff --git a/server/src/test/kotlin/ir/armor/tachidesk/TestExtensions.kt b/server/src/test/kotlin/ir/armor/tachidesk/TestExtensions.kt index 574bb9dd..500309ea 100644 --- a/server/src/test/kotlin/ir/armor/tachidesk/TestExtensions.kt +++ b/server/src/test/kotlin/ir/armor/tachidesk/TestExtensions.kt @@ -41,7 +41,8 @@ class TestExtensions { @BeforeAll fun setup() { val dataRoot = File("tmp/TestDesk").absolutePath - applicationSetup(dataRoot) + System.setProperty("ir.armor.tachidesk.rootDir", dataRoot) + applicationSetup() setLoggingEnabled(false) runBlocking { extensions = getExtensionList()