From d76620634385356335f3e8e3692a19e14b86f979 Mon Sep 17 00:00:00 2001 From: Aria Moradi Date: Thu, 4 Feb 2021 00:32:01 +0330 Subject: [PATCH] fix sqlite locking fuckery by replacing it with h2 --- server/build.gradle.kts | 2 + .../tachiyomi/network/MemoryCookieJar.kt | 10 +- .../main/kotlin/ir/armor/tachidesk/Main.kt | 4 +- .../ir/armor/tachidesk/database/DBMangaer.kt | 6 +- .../kotlin/ir/armor/tachidesk/util/Manga.kt | 226 +++++++----------- .../ir/armor/tachidesk/util/MangaDexHelper.kt | 54 +++-- 6 files changed, 130 insertions(+), 172 deletions(-) diff --git a/server/build.gradle.kts b/server/build.gradle.kts index b078d812..a7d1e9e7 100644 --- a/server/build.gradle.kts +++ b/server/build.gradle.kts @@ -80,6 +80,8 @@ dependencies { implementation ("org.jetbrains.exposed:exposed-dao:$exposed_version") implementation ("org.jetbrains.exposed:exposed-jdbc:$exposed_version") implementation ("org.xerial:sqlite-jdbc:3.30.1") + implementation ("com.h2database:h2:1.4.199") + // AndroidCompat implementation(project(":AndroidCompat")) diff --git a/server/src/main/kotlin/eu/kanade/tachiyomi/network/MemoryCookieJar.kt b/server/src/main/kotlin/eu/kanade/tachiyomi/network/MemoryCookieJar.kt index 3ea7a031..100c66e6 100644 --- a/server/src/main/kotlin/eu/kanade/tachiyomi/network/MemoryCookieJar.kt +++ b/server/src/main/kotlin/eu/kanade/tachiyomi/network/MemoryCookieJar.kt @@ -50,10 +50,10 @@ class WrappedCookie private constructor(val cookie: Cookie) { if (other !is WrappedCookie) return false return other.cookie.name == cookie.name && - other.cookie.domain == cookie.domain && - other.cookie.path == cookie.path && - other.cookie.secure == cookie.secure && - other.cookie.hostOnly == cookie.hostOnly + other.cookie.domain == cookie.domain && + other.cookie.path == cookie.path && + other.cookie.secure == cookie.secure && + other.cookie.hostOnly == cookie.hostOnly } override fun hashCode(): Int { @@ -69,4 +69,4 @@ class WrappedCookie private constructor(val cookie: Cookie) { companion object { fun wrap(cookie: Cookie) = WrappedCookie(cookie) } -} \ No newline at end of file +} diff --git a/server/src/main/kotlin/ir/armor/tachidesk/Main.kt b/server/src/main/kotlin/ir/armor/tachidesk/Main.kt index 819656e4..30260c9f 100644 --- a/server/src/main/kotlin/ir/armor/tachidesk/Main.kt +++ b/server/src/main/kotlin/ir/armor/tachidesk/Main.kt @@ -11,7 +11,7 @@ import ir.armor.tachidesk.util.getChapterList import ir.armor.tachidesk.util.getExtensionList import ir.armor.tachidesk.util.getManga import ir.armor.tachidesk.util.getMangaList -import ir.armor.tachidesk.util.getMangaUpdateQueueThread +// import ir.armor.tachidesk.util.getMangaUpdateQueueThread import ir.armor.tachidesk.util.getPages import ir.armor.tachidesk.util.getSource import ir.armor.tachidesk.util.getSourceList @@ -57,7 +57,7 @@ class Main { // start app androidCompat.startApp(App()) - Thread(getMangaUpdateQueueThread).start() +// Thread(getMangaUpdateQueueThread).start() val app = Javalin.create { config -> try { diff --git a/server/src/main/kotlin/ir/armor/tachidesk/database/DBMangaer.kt b/server/src/main/kotlin/ir/armor/tachidesk/database/DBMangaer.kt index b6a3215b..4d9af4ae 100644 --- a/server/src/main/kotlin/ir/armor/tachidesk/database/DBMangaer.kt +++ b/server/src/main/kotlin/ir/armor/tachidesk/database/DBMangaer.kt @@ -15,13 +15,15 @@ import org.jetbrains.exposed.sql.transactions.transaction object DBMangaer { val db by lazy { - Database.connect("jdbc:sqlite:${Config.dataRoot}/database.db", "org.sqlite.JDBC") + Database.connect("jdbc:h2:${Config.dataRoot}/database.h2", "org.h2.Driver") +// Database.connect("jdbc:sqlite:${Config.dataRoot}/database.sqlite3", "org.sqlite.JDBC") } } fun makeDataBaseTables() { // mention db object to connect - DBMangaer.db + val db = DBMangaer.db + db.useNestedTransactions = true transaction { SchemaUtils.create(ExtensionsTable) diff --git a/server/src/main/kotlin/ir/armor/tachidesk/util/Manga.kt b/server/src/main/kotlin/ir/armor/tachidesk/util/Manga.kt index c5dbb62d..274260ca 100644 --- a/server/src/main/kotlin/ir/armor/tachidesk/util/Manga.kt +++ b/server/src/main/kotlin/ir/armor/tachidesk/util/Manga.kt @@ -12,164 +12,116 @@ import ir.armor.tachidesk.database.table.MangaStatus import ir.armor.tachidesk.database.table.MangaTable import ir.armor.tachidesk.database.table.SourceTable import org.jetbrains.exposed.sql.select -import org.jetbrains.exposed.sql.transactions.experimental.newSuspendedTransaction import org.jetbrains.exposed.sql.transactions.transaction import org.jetbrains.exposed.sql.update import java.io.File import java.io.InputStream -import java.util.concurrent.ArrayBlockingQueue - -val getMangaUpdateQueue = ArrayBlockingQueue>(1000) -@Volatile -var getMangaCount = 0 - -val getMangaUpdateQueueThread = Runnable { - while (true) { - val p = getMangaUpdateQueue.take() - println("took ${p.first}") - while (getMangaCount > 0) { - println("count is $getMangaCount") - Thread.sleep(1000) - } - val mangaId = p.first - println("working on $mangaId") - val fetchedManga = p.second!! - try { - transaction { - println("transaction start $mangaId") - MangaTable.update({ MangaTable.id eq mangaId }) { - - it[MangaTable.initialized] = true - - it[MangaTable.artist] = fetchedManga.artist - it[MangaTable.author] = fetchedManga.author - it[MangaTable.description] = fetchedManga.description - it[MangaTable.genre] = fetchedManga.genre - it[MangaTable.status] = fetchedManga.status - if (fetchedManga.thumbnail_url != null && fetchedManga.thumbnail_url!!.isNotEmpty()) - it[MangaTable.thumbnail_url] = fetchedManga.thumbnail_url - } - println("transaction end $mangaId") - } - } catch (e: Exception) { - println(e) - } - } -} fun getManga(mangaId: Int, proxyThumbnail: Boolean = true): MangaDataClass { - synchronized(getMangaCount) { - getMangaCount++ - } - return try { + var mangaEntry = transaction { MangaTable.select { MangaTable.id eq mangaId }.firstOrNull()!! } + + return if (mangaEntry[MangaTable.initialized]) { + MangaDataClass( + mangaId, + mangaEntry[MangaTable.sourceReference].value, + + mangaEntry[MangaTable.url], + mangaEntry[MangaTable.title], + if (proxyThumbnail) proxyThumbnailUrl(mangaId) else mangaEntry[MangaTable.thumbnail_url], + + true, + + mangaEntry[MangaTable.artist], + mangaEntry[MangaTable.author], + mangaEntry[MangaTable.description], + mangaEntry[MangaTable.genre], + MangaStatus.valueOf(mangaEntry[MangaTable.status]).name, + ) + } else { // initialize manga + val source = getHttpSource(mangaEntry[MangaTable.sourceReference].value) + val fetchedManga = source.fetchMangaDetails( + SManga.create().apply { + url = mangaEntry[MangaTable.url] + title = mangaEntry[MangaTable.title] + } + ).toBlocking().first() + transaction { - var mangaEntry = MangaTable.select { MangaTable.id eq mangaId }.firstOrNull()!! + MangaTable.update({ MangaTable.id eq mangaId }) { - return@transaction if (mangaEntry[MangaTable.initialized]) { - println("${mangaEntry[MangaTable.title]} is initialized") - println("${mangaEntry[MangaTable.thumbnail_url]}") - MangaDataClass( - mangaId, - mangaEntry[MangaTable.sourceReference].value, + it[MangaTable.initialized] = true - mangaEntry[MangaTable.url], - mangaEntry[MangaTable.title], - if (proxyThumbnail) proxyThumbnailUrl(mangaId) else mangaEntry[MangaTable.thumbnail_url], - - true, - - mangaEntry[MangaTable.artist], - mangaEntry[MangaTable.author], - mangaEntry[MangaTable.description], - mangaEntry[MangaTable.genre], - MangaStatus.valueOf(mangaEntry[MangaTable.status]).name, - ) - } else { // initialize manga - val source = getHttpSource(mangaEntry[MangaTable.sourceReference].value) - val fetchedManga = source.fetchMangaDetails( - SManga.create().apply { - url = mangaEntry[MangaTable.url] - title = mangaEntry[MangaTable.title] - } - ).toBlocking().first() - - // update database - // TODO: sqlite gets fucked here - println("putting $mangaId") - getMangaUpdateQueue.put(Pair(mangaId, fetchedManga)) - -// mangaEntry = MangaTable.select { MangaTable.id eq mangaId }.firstOrNull()!! - val newThumbnail = - if (fetchedManga.thumbnail_url != null && fetchedManga.thumbnail_url!!.isNotEmpty()) { - fetchedManga.thumbnail_url - } else mangaEntry[MangaTable.thumbnail_url] - - MangaDataClass( - mangaId, - mangaEntry[MangaTable.sourceReference].value, - - mangaEntry[MangaTable.url], - mangaEntry[MangaTable.title], - if (proxyThumbnail) proxyThumbnailUrl(mangaId) else newThumbnail, - - true, - - fetchedManga.artist, - fetchedManga.author, - fetchedManga.description, - fetchedManga.genre, - MangaStatus.valueOf(fetchedManga.status).name, - ) + it[MangaTable.artist] = fetchedManga.artist + it[MangaTable.author] = fetchedManga.author + it[MangaTable.description] = fetchedManga.description + it[MangaTable.genre] = fetchedManga.genre + it[MangaTable.status] = fetchedManga.status + if (fetchedManga.thumbnail_url != null && fetchedManga.thumbnail_url!!.isNotEmpty()) + it[MangaTable.thumbnail_url] = fetchedManga.thumbnail_url } } - } finally { - synchronized(getMangaCount) { - getMangaCount-- - } + + mangaEntry = transaction { MangaTable.select { MangaTable.id eq mangaId }.firstOrNull()!! } + val newThumbnail = mangaEntry[MangaTable.thumbnail_url] + + MangaDataClass( + mangaId, + mangaEntry[MangaTable.sourceReference].value, + + mangaEntry[MangaTable.url], + mangaEntry[MangaTable.title], + if (proxyThumbnail) proxyThumbnailUrl(mangaId) else newThumbnail, + + true, + + fetchedManga.artist, + fetchedManga.author, + fetchedManga.description, + fetchedManga.genre, + MangaStatus.valueOf(fetchedManga.status).name, + ) } } fun getThumbnail(mangaId: Int): Pair { - return transaction { - var mangaEntry = MangaTable.select { MangaTable.id eq mangaId }.firstOrNull()!! - var filePath = Config.thumbnailsRoot + "/$mangaId" + val mangaEntry = transaction { MangaTable.select { MangaTable.id eq mangaId }.firstOrNull()!! } + var filePath = Config.thumbnailsRoot + "/$mangaId" - val potentialCache = findFileNameStartingWith(Config.thumbnailsRoot, mangaId.toString()) - if (potentialCache != null) { - println("using cached thumbnail file") - return@transaction Pair( - pathToInputStream(potentialCache), - "image/${potentialCache.substringAfter("$mangaId.")}" - ) - } + val potentialCache = findFileNameStartingWith(Config.thumbnailsRoot, mangaId.toString()) + if (potentialCache != null) { + println("using cached thumbnail file") + return Pair( + pathToInputStream(potentialCache), + "image/${potentialCache.substringAfter("$mangaId.")}" + ) + } - val sourceId = mangaEntry[MangaTable.sourceReference].value - println("getting source for $mangaId") - val source = getHttpSource(sourceId) - var thumbnailUrl = mangaEntry[MangaTable.thumbnail_url] - if (thumbnailUrl == null || thumbnailUrl.isEmpty()) { - thumbnailUrl = getManga(mangaId, proxyThumbnail = false).thumbnailUrl!! - } - println(thumbnailUrl) - val response = source.client.newCall( - GET(thumbnailUrl, source.headers) - ).execute() + val sourceId = mangaEntry[MangaTable.sourceReference].value + println("getting source for $mangaId") + val source = getHttpSource(sourceId) + var thumbnailUrl = mangaEntry[MangaTable.thumbnail_url] + if (thumbnailUrl == null || thumbnailUrl.isEmpty()) { + thumbnailUrl = getManga(mangaId, proxyThumbnail = false).thumbnailUrl!! + } + println(thumbnailUrl) + val response = source.client.newCall( + GET(thumbnailUrl, source.headers) + ).execute() - println(response.code) + println(response.code) - if (response.code == 200) { - val contentType = response.headers["content-type"]!! - filePath += "." + contentType.substringAfter("image/") + if (response.code == 200) { + val contentType = response.headers["content-type"]!! + filePath += "." + contentType.substringAfter("image/") - writeStream(response.body!!.byteStream(), filePath) + writeStream(response.body!!.byteStream(), filePath) - return@transaction Pair( - pathToInputStream(filePath), - contentType - ) - } else { - throw Exception("request error! ${response.code}") - } + return Pair( + pathToInputStream(filePath), + contentType + ) + } else { + throw Exception("request error! ${response.code}") } } diff --git a/server/src/main/kotlin/ir/armor/tachidesk/util/MangaDexHelper.kt b/server/src/main/kotlin/ir/armor/tachidesk/util/MangaDexHelper.kt index 061ab0b8..1cdb2300 100644 --- a/server/src/main/kotlin/ir/armor/tachidesk/util/MangaDexHelper.kt +++ b/server/src/main/kotlin/ir/armor/tachidesk/util/MangaDexHelper.kt @@ -1,30 +1,32 @@ package ir.armor.tachidesk.util -import com.android.dx.util.ExceptionWithContext.withContext import eu.kanade.tachiyomi.network.POST import eu.kanade.tachiyomi.source.online.HttpSource -import kotlinx.coroutines.Dispatchers import okhttp3.FormBody import okhttp3.OkHttpClient import java.net.URLEncoder +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + class MangaDexHelper(private val mangaDexSource: HttpSource) { private fun clientBuilder(): OkHttpClient = clientBuilder(0) private fun clientBuilder( - r18Toggle: Int, - okHttpClient: OkHttpClient = mangaDexSource.network.client + r18Toggle: Int, + okHttpClient: OkHttpClient = mangaDexSource.network.client ): OkHttpClient = okHttpClient.newBuilder() - .addNetworkInterceptor { chain -> - val originalCookies = chain.request().header("Cookie") ?: "" - val newReq = chain - .request() - .newBuilder() - .header("Cookie", "$originalCookies; ${cookiesHeader(r18Toggle)}") - .build() - chain.proceed(newReq) - }.build() + .addNetworkInterceptor { chain -> + val originalCookies = chain.request().header("Cookie") ?: "" + val newReq = chain + .request() + .newBuilder() + .header("Cookie", "$originalCookies; ${cookiesHeader(r18Toggle)}") + .build() + chain.proceed(newReq) + }.build() private fun cookiesHeader(r18Toggle: Int): String { val cookies = mutableMapOf() @@ -33,9 +35,9 @@ class MangaDexHelper(private val mangaDexSource: HttpSource) { } private fun buildCookies(cookies: Map) = - cookies.entries.joinToString(separator = "; ", postfix = ";") { - "${URLEncoder.encode(it.key, "UTF-8")}=${URLEncoder.encode(it.value, "UTF-8")}" - } + cookies.entries.joinToString(separator = "; ", postfix = ";") { + "${URLEncoder.encode(it.key, "UTF-8")}=${URLEncoder.encode(it.value, "UTF-8")}" + } // fun isLogged(): Boolean { // val httpUrl = mangaDexSource.baseUrl.toHttpUrlOrNull()!! @@ -44,21 +46,21 @@ class MangaDexHelper(private val mangaDexSource: HttpSource) { fun login(username: String, password: String, twoFactorCode: String = ""): Boolean { val formBody = FormBody.Builder() - .add("login_username", username) - .add("login_password", password) - .add("no_js", "1") - .add("remember_me", "1") + .add("login_username", username) + .add("login_password", password) + .add("no_js", "1") + .add("remember_me", "1") twoFactorCode.let { formBody.add("two_factor", it) } val response = clientBuilder().newCall( - POST( - "${mangaDexSource.baseUrl}/ajax/actions.ajax.php?function=login", - mangaDexSource.headers, - formBody.build() - ) + POST( + "${mangaDexSource.baseUrl}/ajax/actions.ajax.php?function=login", + mangaDexSource.headers, + formBody.build() + ) ).execute() return response.body!!.string().isEmpty() } @@ -90,4 +92,4 @@ class MangaDexHelper(private val mangaDexSource: HttpSource) { // false // } // } -} \ No newline at end of file +}