diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index cc0def43..f6d671b5 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -12,6 +12,7 @@ settings = "1.0.0-RC" twelvemonkeys = "3.9.4" playwright = "1.28.0" graphqlkotlin = "6.5.3" +xmlserialization = "0.86.1" [libraries] # Kotlin @@ -27,6 +28,8 @@ coroutines-test = { module = "org.jetbrains.kotlinx:kotlinx-coroutines-test", ve # Serialization serialization-json = { module = "org.jetbrains.kotlinx:kotlinx-serialization-json", version.ref = "serialization" } serialization-protobuf = { module = "org.jetbrains.kotlinx:kotlinx-serialization-protobuf", version.ref = "serialization" } +serialization-xml-core = { module = "io.github.pdvrieze.xmlutil:core-jvm", version.ref = "xmlserialization" } +serialization-xml = { module = "io.github.pdvrieze.xmlutil:serialization-jvm", version.ref = "xmlserialization" } # Logging slf4japi = "org.slf4j:slf4j-api:2.0.7" @@ -100,7 +103,7 @@ xmlpull = "xmlpull:xmlpull:1.1.3.4a" appdirs = "net.harawata:appdirs:1.2.1" zip4j = "net.lingala.zip4j:zip4j:2.11.5" commonscompress = "org.apache.commons:commons-compress:1.23.0" -junrar = "com.github.junrar:junrar:7.5.4" +junrar = "com.github.junrar:junrar:7.5.5" # CloudflareInterceptor playwright = { module = "com.microsoft.playwright:playwright", version.ref = "playwright" } diff --git a/server/build.gradle.kts b/server/build.gradle.kts index 51e7d8d3..5bd9ecb0 100644 --- a/server/build.gradle.kts +++ b/server/build.gradle.kts @@ -45,6 +45,10 @@ dependencies { implementation(libs.rxjava) implementation(libs.jsoup) + // ComicInfo + implementation(libs.serialization.xml.core) + implementation(libs.serialization.xml) + // Sort implementation(libs.sort) @@ -114,6 +118,7 @@ buildConfig { tasks { shadowJar { + isZip64 = true manifest { attributes( "Main-Class" to MainClass, diff --git a/server/src/main/kotlin/eu/kanade/tachiyomi/source/Source.kt b/server/src/main/kotlin/eu/kanade/tachiyomi/source/Source.kt index f6436476..0856cf8c 100644 --- a/server/src/main/kotlin/eu/kanade/tachiyomi/source/Source.kt +++ b/server/src/main/kotlin/eu/kanade/tachiyomi/source/Source.kt @@ -4,6 +4,7 @@ import eu.kanade.tachiyomi.source.model.Page import eu.kanade.tachiyomi.source.model.SChapter import eu.kanade.tachiyomi.source.model.SManga import rx.Observable +import suwayomi.tachidesk.manga.impl.util.lang.awaitSingle /** * A basic interface for creating a source. It could be an online source, a local source, etc... @@ -25,21 +26,59 @@ interface Source { * * @param manga the manga to update. */ - fun fetchMangaDetails(manga: SManga): Observable + @Deprecated( + "Use the 1.x API instead", + ReplaceWith("getMangaDetails") + ) + fun fetchMangaDetails(manga: SManga): Observable = throw IllegalStateException("Not used") /** * Returns an observable with all the available chapters for a manga. * * @param manga the manga to update. */ - fun fetchChapterList(manga: SManga): Observable> + @Deprecated( + "Use the 1.x API instead", + ReplaceWith("getChapterList") + ) + fun fetchChapterList(manga: SManga): Observable> = throw IllegalStateException("Not used") /** - * Returns an observable with the list of pages a chapter has. + * Returns an observable with the list of pages a chapter has. Pages should be returned + * in the expected order; the index is ignored. * * @param chapter the chapter. */ - fun fetchPageList(chapter: SChapter): Observable> + @Deprecated( + "Use the 1.x API instead", + ReplaceWith("getPageList") + ) + fun fetchPageList(chapter: SChapter): Observable> = Observable.empty() + + /** + * [1.x API] Get the updated details for a manga. + */ + @Suppress("DEPRECATION") + suspend fun getMangaDetails(manga: SManga): SManga { + return fetchMangaDetails(manga).awaitSingle() + } + + /** + * [1.x API] Get all the available chapters for a manga. + */ + @Suppress("DEPRECATION") + suspend fun getChapterList(manga: SManga): List { + return fetchChapterList(manga).awaitSingle() + } + + /** + * [1.x API] Get the list of pages a chapter has. Pages should be returned + * in the expected order; the index is ignored. + */ + @Suppress("DEPRECATION") + suspend fun getPageList(chapter: SChapter): List { + return fetchPageList(chapter).awaitSingle() + } } // fun Source.icon(): Drawable? = Injekt.get().getAppIconForSource(this) diff --git a/server/src/main/kotlin/eu/kanade/tachiyomi/source/local/LocalSource.kt b/server/src/main/kotlin/eu/kanade/tachiyomi/source/local/LocalSource.kt index 08b5e159..5d0bfe82 100644 --- a/server/src/main/kotlin/eu/kanade/tachiyomi/source/local/LocalSource.kt +++ b/server/src/main/kotlin/eu/kanade/tachiyomi/source/local/LocalSource.kt @@ -1,15 +1,21 @@ package eu.kanade.tachiyomi.source.local -import com.github.junrar.Archive import eu.kanade.tachiyomi.source.CatalogueSource -import eu.kanade.tachiyomi.source.local.LocalSource.Format.Directory -import eu.kanade.tachiyomi.source.local.LocalSource.Format.Epub -import eu.kanade.tachiyomi.source.local.LocalSource.Format.Rar -import eu.kanade.tachiyomi.source.local.LocalSource.Format.Zip +import eu.kanade.tachiyomi.source.UnmeteredSource +import eu.kanade.tachiyomi.source.local.filter.OrderBy +import eu.kanade.tachiyomi.source.local.image.LocalCoverManager +import eu.kanade.tachiyomi.source.local.io.Archive +import eu.kanade.tachiyomi.source.local.io.Format +import eu.kanade.tachiyomi.source.local.io.LocalSourceFileSystem import eu.kanade.tachiyomi.source.local.loader.EpubPageLoader import eu.kanade.tachiyomi.source.local.loader.RarPageLoader import eu.kanade.tachiyomi.source.local.loader.ZipPageLoader -import eu.kanade.tachiyomi.source.model.Filter +import eu.kanade.tachiyomi.source.local.metadata.COMIC_INFO_FILE +import eu.kanade.tachiyomi.source.local.metadata.ComicInfo +import eu.kanade.tachiyomi.source.local.metadata.MangaDetails +import eu.kanade.tachiyomi.source.local.metadata.copyFromComicInfo +import eu.kanade.tachiyomi.source.local.metadata.fillChapterMetadata +import eu.kanade.tachiyomi.source.local.metadata.fillMangaMetadata import eu.kanade.tachiyomi.source.model.FilterList import eu.kanade.tachiyomi.source.model.MangasPage import eu.kanade.tachiyomi.source.model.Page @@ -18,14 +24,14 @@ import eu.kanade.tachiyomi.source.model.SManga import eu.kanade.tachiyomi.util.chapter.ChapterRecognition import eu.kanade.tachiyomi.util.lang.compareToCaseInsensitiveNaturalOrder import eu.kanade.tachiyomi.util.storage.EpubFile +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withContext import kotlinx.serialization.json.Json -import kotlinx.serialization.json.JsonObject -import kotlinx.serialization.json.contentOrNull import kotlinx.serialization.json.decodeFromStream -import kotlinx.serialization.json.intOrNull -import kotlinx.serialization.json.jsonArray -import kotlinx.serialization.json.jsonPrimitive import mu.KotlinLogging +import nl.adaptivity.xmlutil.core.KtXmlReader +import nl.adaptivity.xmlutil.serialization.XML import org.apache.commons.compress.archivers.zip.ZipFile import org.jetbrains.exposed.sql.insert import org.jetbrains.exposed.sql.insertAndGetId @@ -44,10 +50,348 @@ import uy.kohesive.injekt.injectLazy import java.io.File import java.io.FileInputStream import java.io.InputStream -import java.util.Locale -import java.util.concurrent.TimeUnit +import java.nio.charset.StandardCharsets +import kotlin.time.Duration.Companion.days +import com.github.junrar.Archive as JunrarArchive + +class LocalSource( + private val fileSystem: LocalSourceFileSystem, + private val coverManager: LocalCoverManager +) : CatalogueSource, UnmeteredSource { + + private val json: Json by injectLazy() + private val xml: XML by injectLazy() + + private val POPULAR_FILTERS = FilterList(OrderBy.Popular()) + private val LATEST_FILTERS = FilterList(OrderBy.Latest()) + + override val name: String = NAME + + override val id: Long = ID + + override val lang: String = LANG + + override fun toString() = name + + override val supportsLatest: Boolean = true + + // Browse related + override fun fetchPopularManga(page: Int) = fetchSearchManga(page, "", POPULAR_FILTERS) + + override fun fetchLatestUpdates(page: Int) = fetchSearchManga(page, "", LATEST_FILTERS) + + override fun fetchSearchManga(page: Int, query: String, filters: FilterList): Observable { + val baseDirsFiles = fileSystem.getFilesInBaseDirectories() + val lastModifiedLimit by lazy { if (filters === LATEST_FILTERS) System.currentTimeMillis() - LATEST_THRESHOLD else 0L } + var mangaDirs = baseDirsFiles + // Filter out files that are hidden and is not a folder + .filter { it.isDirectory && !it.name.startsWith('.') } + .distinctBy { it.name } + .filter { // Filter by query or last modified + if (lastModifiedLimit == 0L) { + it.name.contains(query, ignoreCase = true) + } else { + it.lastModified() >= lastModifiedLimit + } + } + + filters.forEach { filter -> + when (filter) { + is OrderBy.Popular -> { + mangaDirs = if (filter.state!!.ascending) { + mangaDirs.sortedWith(compareBy(String.CASE_INSENSITIVE_ORDER) { it.name }) + } else { + mangaDirs.sortedWith(compareByDescending(String.CASE_INSENSITIVE_ORDER) { it.name }) + } + } + is OrderBy.Latest -> { + mangaDirs = if (filter.state!!.ascending) { + mangaDirs.sortedBy(File::lastModified) + } else { + mangaDirs.sortedByDescending(File::lastModified) + } + } + + else -> { + /* Do nothing */ + } + } + } + + // Transform mangaDirs to list of SManga + val mangas = mangaDirs.map { mangaDir -> + SManga.create().apply { + title = mangaDir.name + url = mangaDir.name + + // Try to find the cover + coverManager.find(mangaDir.name) + ?.takeIf(File::exists) + ?.let { thumbnail_url = it.absolutePath } + } + } + + // Fetch chapters of all the manga + mangas.forEach { manga -> + runBlocking { + val chapters = getChapterList(manga) + if (chapters.isNotEmpty()) { + val chapter = chapters.last() + val format = getFormat(chapter) + + if (format is Format.Epub) { + EpubFile(format.file).use { epub -> + epub.fillMangaMetadata(manga) + } + } + + // Copy the cover from the first chapter found if not available + if (manga.thumbnail_url == null) { + updateCover(chapter, manga) + } + } + } + } + + return Observable.just(MangasPage(mangas.toList(), false)) + } + + // Manga details related + override suspend fun getMangaDetails(manga: SManga): SManga = withContext(Dispatchers.IO) { + coverManager.find(manga.url)?.let { + manga.thumbnail_url = it.absolutePath + } + + // Augment manga details based on metadata files + try { + val mangaDirFiles = fileSystem.getFilesInMangaDirectory(manga.url).toList() + + val comicInfoFile = mangaDirFiles + .firstOrNull { it.name == COMIC_INFO_FILE } + val noXmlFile = mangaDirFiles + .firstOrNull { it.name == ".noxml" } + val legacyJsonDetailsFile = mangaDirFiles + .firstOrNull { it.extension == "json" } + + when { + // Top level ComicInfo.xml + comicInfoFile != null -> { + noXmlFile?.delete() + setMangaDetailsFromComicInfoFile(comicInfoFile.inputStream(), manga) + } + + // TODO: automatically convert these to ComicInfo.xml + legacyJsonDetailsFile != null -> { + json.decodeFromStream(legacyJsonDetailsFile.inputStream()).run { + title?.let { manga.title = it } + author?.let { manga.author = it } + artist?.let { manga.artist = it } + description?.let { manga.description = it } + genre?.let { manga.genre = it.joinToString() } + status?.let { manga.status = it } + } + } + + // Copy ComicInfo.xml from chapter archive to top level if found + noXmlFile == null -> { + val chapterArchives = mangaDirFiles + .filter(Archive::isSupported) + .toList() + + val mangaDir = fileSystem.getMangaDirectory(manga.url) + val folderPath = mangaDir?.absolutePath + + val copiedFile = copyComicInfoFileFromArchive(chapterArchives, folderPath) + if (copiedFile != null) { + setMangaDetailsFromComicInfoFile(copiedFile.inputStream(), manga) + } else { + // Avoid re-scanning + File("$folderPath/.noxml").createNewFile() + } + } + } + } catch (e: Throwable) { + logger.error(e) { "Error setting manga details from local metadata for ${manga.title}" } + } + + return@withContext manga + } + + private fun copyComicInfoFileFromArchive(chapterArchives: List, folderPath: String?): File? { + for (chapter in chapterArchives) { + when (Format.valueOf(chapter)) { + is Format.Zip -> { + ZipFile(chapter).use { zip: ZipFile -> + zip.getEntry(COMIC_INFO_FILE)?.let { comicInfoFile -> + zip.getInputStream(comicInfoFile).buffered().use { stream -> + return copyComicInfoFile(stream, folderPath) + } + } + } + } + is Format.Rar -> { + JunrarArchive(chapter).use { rar -> + rar.fileHeaders.firstOrNull { it.fileName == COMIC_INFO_FILE }?.let { comicInfoFile -> + rar.getInputStream(comicInfoFile).buffered().use { stream -> + return copyComicInfoFile(stream, folderPath) + } + } + } + } + else -> {} + } + } + return null + } + + private fun copyComicInfoFile(comicInfoFileStream: InputStream, folderPath: String?): File { + return File("$folderPath/$COMIC_INFO_FILE").apply { + outputStream().use { outputStream -> + comicInfoFileStream.use { it.copyTo(outputStream) } + } + } + } + + private fun setMangaDetailsFromComicInfoFile(stream: InputStream, manga: SManga) { + val comicInfo = KtXmlReader(stream, StandardCharsets.UTF_8.name()).use { + xml.decodeFromReader(it) + } + + manga.copyFromComicInfo(comicInfo) + } + + // Chapters + override suspend fun getChapterList(manga: SManga): List { + return fileSystem.getFilesInMangaDirectory(manga.url) + // Only keep supported formats + .filter { it.isDirectory || Archive.isSupported(it) } + .map { chapterFile -> + SChapter.create().apply { + url = "${manga.url}/${chapterFile.name}" + name = if (chapterFile.isDirectory) { + chapterFile.name + } else { + chapterFile.nameWithoutExtension + } + date_upload = chapterFile.lastModified() + chapter_number = ChapterRecognition + .parseChapterNumber(manga.title, this.name, this.chapter_number.toDouble()) + .toFloat() + + val format = Format.valueOf(chapterFile) + if (format is Format.Epub) { + EpubFile(format.file).use { epub -> + epub.fillChapterMetadata(this) + } + } + } + } + .sortedWith { c1, c2 -> + val c = c2.chapter_number.compareTo(c1.chapter_number) + if (c == 0) c2.name.compareToCaseInsensitiveNaturalOrder(c1.name) else c + } + .toList() + } + + // Filters + override fun getFilterList() = FilterList(OrderBy.Popular()) + + // TODO Fix Memory Leak + override suspend fun getPageList(chapter: SChapter): List { + return when (val format = getFormat(chapter)) { + is Format.Directory -> { + format.file.listFiles().orEmpty() + .sortedBy { it.name } + .filter { !it.isDirectory && ImageUtil.isImage(it.name, it::inputStream) } + .mapIndexed { index, page -> + Page( + index, + imageUrl = applicationDirs.localMangaRoot + "/" + chapter.url + "/" + page.name + ) + } + } + is Format.Zip -> { + val loader = ZipPageLoader(format.file) + val pages = loader.getPages() + pageCache[chapter.url] = pages.map { it.stream!! } + + pages + } + is Format.Rar -> { + val loader = RarPageLoader(format.file) + val pages = loader.getPages() + pageCache[chapter.url] = pages.map { it.stream!! } + + pages + } + is Format.Epub -> { + val loader = EpubPageLoader(format.file) + val pages = loader.getPages() + pageCache[chapter.url] = pages.map { it.stream!! } + + pages + } + } + } + + fun getFormat(chapter: SChapter): Format { + try { + return fileSystem.getBaseDirectories() + .map { dir -> File(dir, chapter.url) } + .find { it.exists() } + ?.let(Format.Companion::valueOf) + ?: throw Exception("Chapter not found") + } catch (e: Format.UnknownFormatException) { + throw Exception("Invalid chapter format") + } catch (e: Exception) { + throw e + } + } + + private fun updateCover(chapter: SChapter, manga: SManga): File? { + return try { + when (val format = getFormat(chapter)) { + is Format.Directory -> { + val entry = format.file.listFiles() + ?.sortedWith { f1, f2 -> f1.name.compareToCaseInsensitiveNaturalOrder(f2.name) } + ?.find { !it.isDirectory && ImageUtil.isImage(it.name) { FileInputStream(it) } } + + entry?.let { coverManager.update(manga, it.inputStream()) } + } + is Format.Zip -> { + ZipFile(format.file).use { zip -> + val entry = zip.entries.toList() + .sortedWith { f1, f2 -> f1.name.compareToCaseInsensitiveNaturalOrder(f2.name) } + .find { !it.isDirectory && ImageUtil.isImage(it.name) { zip.getInputStream(it) } } + + entry?.let { coverManager.update(manga, zip.getInputStream(it)) } + } + } + is Format.Rar -> { + JunrarArchive(format.file).use { archive -> + val entry = archive.fileHeaders + .sortedWith { f1, f2 -> f1.fileName.compareToCaseInsensitiveNaturalOrder(f2.fileName) } + .find { !it.isDirectory && ImageUtil.isImage(it.fileName) { archive.getInputStream(it) } } + + entry?.let { coverManager.update(manga, archive.getInputStream(it)) } + } + } + is Format.Epub -> { + EpubFile(format.file).use { epub -> + val entry = epub.getImagesFromPages() + .firstOrNull() + ?.let { epub.getEntry(it) } + + entry?.let { coverManager.update(manga, epub.getInputStream(it)) } + } + } + } + } catch (e: Throwable) { + logger.error(e) { "Error updating cover for ${manga.title}" } + null + } + } -class LocalSource : CatalogueSource { companion object { const val ID = 0L const val LANG = "localsourcelang" @@ -55,11 +399,7 @@ class LocalSource : CatalogueSource { const val EXTENSION_NAME = "Local Source fake extension" - const val HELP_URL = "https://tachiyomi.org/help/guides/local-manga/" - - private val SUPPORTED_ARCHIVE_TYPES = setOf("zip", "rar", "cbr", "cbz", "epub") - - private val LATEST_THRESHOLD = TimeUnit.MILLISECONDS.convert(7, TimeUnit.DAYS) + private val LATEST_THRESHOLD = 7.days.inWholeMilliseconds private val logger = KotlinLogging.logger {} @@ -67,29 +407,6 @@ class LocalSource : CatalogueSource { val pageCache: MutableMap InputStream>> = mutableMapOf() - fun updateCover(manga: SManga, input: InputStream): File? { - val cover = getCoverFile(File("${applicationDirs.localMangaRoot}/${manga.url}")) - ?: File("${applicationDirs.localMangaRoot}/${manga.url}/cover.jpg") - - cover.parentFile?.mkdirs() - input.use { - cover.outputStream().use { - input.copyTo(it) - } - } - - return cover - } - - /** - * Returns valid cover file inside [parent] directory. - */ - private fun getCoverFile(parent: File): File? { - return parent.listFiles()?.find { it.nameWithoutExtension == "cover" }?.takeIf { - it.isFile && ImageUtil.isImage(it.name) { it.inputStream() } - } - } - fun register() { transaction { val sourceRecord = SourceTable.select { SourceTable.id eq ID }.firstOrNull() @@ -117,294 +434,8 @@ class LocalSource : CatalogueSource { } } - registerCatalogueSource(ID to LocalSource()) + val fs = LocalSourceFileSystem(applicationDirs) + registerCatalogueSource(ID to LocalSource(fs, LocalCoverManager(fs))) } } - - override val id = ID - override val name = NAME - override val lang = LANG - override val supportsLatest = true - - private val json: Json by injectLazy() - - override fun toString() = name - - override fun fetchPopularManga(page: Int) = fetchSearchManga(page, "", POPULAR_FILTERS) - - override fun fetchSearchManga(page: Int, query: String, filters: FilterList): Observable { - val time = if (filters === LATEST_FILTERS) System.currentTimeMillis() - LATEST_THRESHOLD else 0L - - var mangaDirs = File(applicationDirs.localMangaRoot).listFiles().orEmpty().toList() - .filter { it.isDirectory } - .filterNot { it.name.startsWith('.') } - .filter { if (time == 0L) it.name.contains(query, ignoreCase = true) else it.lastModified() >= time } - .distinctBy { it.name } - - val state = ((if (filters.isEmpty()) POPULAR_FILTERS else filters)[0] as OrderBy).state - when (state?.index) { - 0 -> { - mangaDirs = if (state.ascending) { - mangaDirs.sortedBy { it.name.lowercase(Locale.ENGLISH) } - } else { - mangaDirs.sortedByDescending { it.name.lowercase(Locale.ENGLISH) } - } - } - 1 -> { - mangaDirs = if (state.ascending) { - mangaDirs.sortedBy(File::lastModified) - } else { - mangaDirs.sortedByDescending(File::lastModified) - } - } - } - - val mangas = mangaDirs.map { mangaDir -> - SManga.create().apply { - title = mangaDir.name - url = mangaDir.name - - // Try to find the cover - val cover = getCoverFile(File("${applicationDirs.localMangaRoot}/$url")) - if (cover != null && cover.exists()) { - thumbnail_url = cover.absolutePath - } - - val chapters = fetchChapterList(this).toBlocking().first() - if (chapters.isNotEmpty()) { - val chapter = chapters.last() - val format = getFormat(chapter) - if (format is Format.Epub) { - EpubFile(format.file).use { epub -> - epub.fillMangaMetadata(this) - } - } - - // Copy the cover from the first chapter found. - if (thumbnail_url == null) { - try { - thumbnail_url = updateCover(chapter, this)?.absolutePath - } catch (e: Exception) { - logger.error { e } - } - } - } - } - } - - return Observable.just(MangasPage(mangas.toList(), false)) - } - - override fun fetchLatestUpdates(page: Int) = fetchSearchManga(page, "", LATEST_FILTERS) - - override fun fetchMangaDetails(manga: SManga): Observable { - File(applicationDirs.localMangaRoot, manga.url).listFiles().orEmpty().toList() - .firstOrNull { it.extension == "json" } - ?.apply { - val obj = json.decodeFromStream(inputStream()) - - manga.title = obj["title"]?.jsonPrimitive?.contentOrNull ?: manga.title - manga.author = obj["author"]?.jsonPrimitive?.contentOrNull ?: manga.author - manga.artist = obj["artist"]?.jsonPrimitive?.contentOrNull ?: manga.artist - manga.description = obj["description"]?.jsonPrimitive?.contentOrNull ?: manga.description - manga.genre = obj["genre"]?.jsonArray?.joinToString(", ") { it.jsonPrimitive.content } - ?: manga.genre - manga.status = obj["status"]?.jsonPrimitive?.intOrNull ?: manga.status - } - - // update the cover - val cover = getCoverFile(File("${applicationDirs.localMangaRoot}/${manga.url}")) - if (cover != null && cover.exists()) { - manga.thumbnail_url = cover.absolutePath - } - - return Observable.just(manga) - } - - override fun fetchChapterList(manga: SManga): Observable> { - val chapters = File(applicationDirs.localMangaRoot, manga.url).listFiles().orEmpty().toList() - .filter { it.isDirectory || isSupportedFile(it.extension) } - .map { chapterFile -> - SChapter.create().apply { - url = "${manga.url}/${chapterFile.name}" - name = if (chapterFile.isDirectory) { - chapterFile.name - } else { - chapterFile.nameWithoutExtension - } - date_upload = chapterFile.lastModified() - - val format = getFormat(this) - if (format is Format.Epub) { - EpubFile(format.file).use { epub -> - epub.fillChapterMetadata(this) - } - } - - val chapNameCut = stripMangaTitle(name, manga.title) - if (chapNameCut.isNotEmpty()) name = chapNameCut - ChapterRecognition.parseChapterNumber(this, manga) - } - } - .sortedWith { c1, c2 -> - val c = c2.chapter_number.compareTo(c1.chapter_number) - if (c == 0) c2.name.compareToCaseInsensitiveNaturalOrder(c1.name) else c - } - .toList() - - return Observable.just(chapters) - } - - /** - * Strips the manga title from a chapter name, matching only based on alphanumeric and whitespace - * characters. - */ - private fun stripMangaTitle(chapterName: String, mangaTitle: String): String { - var chapterNameIndex = 0 - var mangaTitleIndex = 0 - while (chapterNameIndex < chapterName.length && mangaTitleIndex < mangaTitle.length) { - val chapterChar = chapterName[chapterNameIndex] - val mangaChar = mangaTitle[mangaTitleIndex] - if (!chapterChar.equals(mangaChar, true)) { - val invalidChapterChar = !chapterChar.isLetterOrDigit() && !chapterChar.isWhitespace() - val invalidMangaChar = !mangaChar.isLetterOrDigit() && !mangaChar.isWhitespace() - - if (!invalidChapterChar && !invalidMangaChar) { - return chapterName - } - - if (invalidChapterChar) { - chapterNameIndex++ - } - - if (invalidMangaChar) { - mangaTitleIndex++ - } - } else { - chapterNameIndex++ - mangaTitleIndex++ - } - } - - return chapterName.substring(chapterNameIndex).trimStart(' ', '-', '_', ',', ':') - } - - private fun isSupportedFile(extension: String): Boolean { - return extension.lowercase() in SUPPORTED_ARCHIVE_TYPES - } - - override fun fetchPageList(chapter: SChapter): Observable> { - val chapterFile = File(applicationDirs.localMangaRoot + "/" + chapter.url) - - return when (getFormat(chapterFile)) { - is Directory -> { - Observable.just( - chapterFile.listFiles().orEmpty() - .sortedBy { it.name } - .filter { !it.isDirectory && ImageUtil.isImage(it.name, it::inputStream) } - .mapIndexed { index, page -> - Page( - index, - imageUrl = applicationDirs.localMangaRoot + "/" + chapter.url + "/" + page.name - ) - } - ) - } - is Zip -> { - val pages = ZipPageLoader(chapterFile).getPages() - pageCache[chapter.url] = pages.map { it.stream!! } - - Observable.just(pages) - } - is Rar -> { - val pages = RarPageLoader(chapterFile).getPages() - pageCache[chapter.url] = pages.map { it.stream!! } - - Observable.just(pages) - } - is Epub -> { - val pages = EpubPageLoader(chapterFile).getPages() - pageCache[chapter.url] = pages.map { it.stream!! } - - Observable.just(pages) - } - } - } - - fun getFormat(chapter: SChapter): Format { - val chapFile = File(applicationDirs.localMangaRoot, chapter.url) - if (chapFile.exists()) { - return getFormat(chapFile) - } - - throw Exception("Chapter not found") - } - - private fun getFormat(file: File): Format = with(file) { - when { - isDirectory -> Format.Directory(this) - extension.equals("zip", true) || extension.equals("cbz", true) -> Format.Zip(this) - extension.equals("rar", true) || extension.equals("cbr", true) -> Format.Rar(this) - extension.equals("epub", true) -> Format.Epub(this) - - else -> throw Exception("Invalid chapter format") - } - } - - private fun updateCover(chapter: SChapter, manga: SManga): File? { - return when (val format = getFormat(chapter)) { - is Format.Directory -> { - val entry = format.file.listFiles() - ?.sortedWith { f1, f2 -> f1.name.compareToCaseInsensitiveNaturalOrder(f2.name) } - ?.find { !it.isDirectory && ImageUtil.isImage(it.name) { FileInputStream(it) } } - - entry?.let { updateCover(manga, it.inputStream()) } - } - is Format.Zip -> { - ZipFile(format.file).use { zip -> - val entry = zip.entries.toList() - .sortedWith { f1, f2 -> f1.name.compareToCaseInsensitiveNaturalOrder(f2.name) } - .find { !it.isDirectory && ImageUtil.isImage(it.name) { zip.getInputStream(it) } } - - entry?.let { updateCover(manga, zip.getInputStream(it)) } - } - } - is Format.Rar -> { - Archive(format.file).use { archive -> - val entry = archive.fileHeaders - .sortedWith { f1, f2 -> f1.fileName.compareToCaseInsensitiveNaturalOrder(f2.fileName) } - .find { !it.isDirectory && ImageUtil.isImage(it.fileName) { archive.getInputStream(it) } } - - entry?.let { updateCover(manga, archive.getInputStream(it)) } - } - } - is Format.Epub -> { - EpubFile(format.file).use { epub -> - val entry = epub.getImagesFromPages() - .firstOrNull() - ?.let { epub.getEntry(it) } - - entry?.let { updateCover(manga, epub.getInputStream(it)) } - } - } - } - } - - override fun getFilterList() = POPULAR_FILTERS - - private val POPULAR_FILTERS = FilterList(OrderBy()) - private val LATEST_FILTERS = FilterList(OrderBy().apply { state = Filter.Sort.Selection(1, false) }) - - private class OrderBy : Filter.Sort( - "Order by", - arrayOf("Title", "Date"), - Selection(0, true) - ) - - sealed class Format { - data class Directory(val file: File) : Format() - data class Zip(val file: File) : Format() - data class Rar(val file: File) : Format() - data class Epub(val file: File) : Format() - } } diff --git a/server/src/main/kotlin/eu/kanade/tachiyomi/source/local/filter/OrderBy.kt b/server/src/main/kotlin/eu/kanade/tachiyomi/source/local/filter/OrderBy.kt new file mode 100644 index 00000000..15b557d6 --- /dev/null +++ b/server/src/main/kotlin/eu/kanade/tachiyomi/source/local/filter/OrderBy.kt @@ -0,0 +1,12 @@ +package eu.kanade.tachiyomi.source.local.filter + +import eu.kanade.tachiyomi.source.model.Filter + +sealed class OrderBy(selection: Selection) : Filter.Sort( + "Order by", + arrayOf("Title", "Date"), + selection +) { + class Popular() : OrderBy(Selection(0, true)) + class Latest() : OrderBy(Selection(1, false)) +} diff --git a/server/src/main/kotlin/eu/kanade/tachiyomi/source/local/image/LocalCoverManager.kt b/server/src/main/kotlin/eu/kanade/tachiyomi/source/local/image/LocalCoverManager.kt new file mode 100644 index 00000000..829d7f45 --- /dev/null +++ b/server/src/main/kotlin/eu/kanade/tachiyomi/source/local/image/LocalCoverManager.kt @@ -0,0 +1,52 @@ +package eu.kanade.tachiyomi.source.local.image + +import eu.kanade.tachiyomi.source.local.io.LocalSourceFileSystem +import eu.kanade.tachiyomi.source.model.SManga +import suwayomi.tachidesk.manga.impl.util.storage.ImageUtil +import java.io.File +import java.io.InputStream + +private const val DEFAULT_COVER_NAME = "cover.jpg" + +class LocalCoverManager( + private val fileSystem: LocalSourceFileSystem +) { + + fun find(mangaUrl: String): File? { + return fileSystem.getFilesInMangaDirectory(mangaUrl) + // Get all file whose names start with 'cover' + .filter { it.isFile && it.nameWithoutExtension.equals("cover", ignoreCase = true) } + // Get the first actual image + .firstOrNull { + ImageUtil.isImage(it.name) { it.inputStream() } + } + } + + fun update( + manga: SManga, + inputStream: InputStream + ): File? { + val directory = fileSystem.getMangaDirectory(manga.url) + if (directory == null) { + inputStream.close() + return null + } + + var targetFile = find(manga.url) + if (targetFile == null) { + targetFile = File(directory.absolutePath, DEFAULT_COVER_NAME) + targetFile.createNewFile() + } + + // It might not exist at this point + targetFile.parentFile?.mkdirs() + inputStream.use { input -> + targetFile.outputStream().use { output -> + input.copyTo(output) + } + } + + manga.thumbnail_url = targetFile.absolutePath + return targetFile + } +} diff --git a/server/src/main/kotlin/eu/kanade/tachiyomi/source/local/io/Archive.kt b/server/src/main/kotlin/eu/kanade/tachiyomi/source/local/io/Archive.kt new file mode 100644 index 00000000..f3d061e8 --- /dev/null +++ b/server/src/main/kotlin/eu/kanade/tachiyomi/source/local/io/Archive.kt @@ -0,0 +1,12 @@ +package eu.kanade.tachiyomi.source.local.io + +import java.io.File + +object Archive { + + private val SUPPORTED_ARCHIVE_TYPES = listOf("zip", "cbz", "rar", "cbr", "epub") + + fun isSupported(file: File): Boolean = with(file) { + return extension.lowercase() in SUPPORTED_ARCHIVE_TYPES + } +} diff --git a/server/src/main/kotlin/eu/kanade/tachiyomi/source/local/io/Format.kt b/server/src/main/kotlin/eu/kanade/tachiyomi/source/local/io/Format.kt new file mode 100644 index 00000000..906a7f2f --- /dev/null +++ b/server/src/main/kotlin/eu/kanade/tachiyomi/source/local/io/Format.kt @@ -0,0 +1,25 @@ +package eu.kanade.tachiyomi.source.local.io + +import java.io.File + +sealed interface Format { + data class Directory(val file: File) : Format + data class Zip(val file: File) : Format + data class Rar(val file: File) : Format + data class Epub(val file: File) : Format + + class UnknownFormatException : Exception() + + companion object { + + fun valueOf(file: File) = with(file) { + when { + isDirectory -> Directory(this) + extension.equals("zip", true) || extension.equals("cbz", true) -> Zip(this) + extension.equals("rar", true) || extension.equals("cbr", true) -> Rar(this) + extension.equals("epub", true) -> Epub(this) + else -> throw UnknownFormatException() + } + } + } +} diff --git a/server/src/main/kotlin/eu/kanade/tachiyomi/source/local/io/LocalSourceFileSystem.kt b/server/src/main/kotlin/eu/kanade/tachiyomi/source/local/io/LocalSourceFileSystem.kt new file mode 100644 index 00000000..6ff42d19 --- /dev/null +++ b/server/src/main/kotlin/eu/kanade/tachiyomi/source/local/io/LocalSourceFileSystem.kt @@ -0,0 +1,33 @@ +package eu.kanade.tachiyomi.source.local.io + +import suwayomi.tachidesk.server.ApplicationDirs +import java.io.File + +class LocalSourceFileSystem( + private val applicationDirs: ApplicationDirs +) { + + fun getBaseDirectories(): Sequence { + return sequenceOf(File(applicationDirs.localMangaRoot)) + } + + fun getFilesInBaseDirectories(): Sequence { + return getBaseDirectories() + // Get all the files inside all baseDir + .flatMap { it.listFiles().orEmpty().toList() } + } + + fun getMangaDirectory(name: String): File? { + return getFilesInBaseDirectories() + // Get the first mangaDir or null + .firstOrNull { it.isDirectory && it.name == name } + } + + fun getFilesInMangaDirectory(name: String): Sequence { + return getFilesInBaseDirectories() + // Filter out ones that are not related to the manga and is not a directory + .filter { it.isDirectory && it.name == name } + // Get all the files inside the filtered folders + .flatMap { it.listFiles().orEmpty().toList() } + } +} diff --git a/server/src/main/kotlin/eu/kanade/tachiyomi/source/local/loader/EpubPageLoader.kt b/server/src/main/kotlin/eu/kanade/tachiyomi/source/local/loader/EpubPageLoader.kt index 2e15eb04..eda28932 100644 --- a/server/src/main/kotlin/eu/kanade/tachiyomi/source/local/loader/EpubPageLoader.kt +++ b/server/src/main/kotlin/eu/kanade/tachiyomi/source/local/loader/EpubPageLoader.kt @@ -8,16 +8,9 @@ import java.io.File */ class EpubPageLoader(file: File) : PageLoader { - /** - * The epub file. - */ private val epub = EpubFile(file) - /** - * Returns an observable containing the pages found on this zip archive ordered with a natural - * comparator. - */ - override fun getPages(): List { + override suspend fun getPages(): List { return epub.getImagesFromPages() .mapIndexed { i, path -> val streamFn = { epub.getInputStream(epub.getEntry(path)!!) } @@ -26,4 +19,8 @@ class EpubPageLoader(file: File) : PageLoader { } } } + + override fun recycle() { + epub.close() + } } diff --git a/server/src/main/kotlin/eu/kanade/tachiyomi/source/local/loader/PageLoader.kt b/server/src/main/kotlin/eu/kanade/tachiyomi/source/local/loader/PageLoader.kt index c06be987..29b7d9c9 100644 --- a/server/src/main/kotlin/eu/kanade/tachiyomi/source/local/loader/PageLoader.kt +++ b/server/src/main/kotlin/eu/kanade/tachiyomi/source/local/loader/PageLoader.kt @@ -6,5 +6,7 @@ interface PageLoader { * Returns an observable containing the list of pages of a chapter. Only the first emission * will be used. */ - fun getPages(): List + suspend fun getPages(): List + + fun recycle() } diff --git a/server/src/main/kotlin/eu/kanade/tachiyomi/source/local/loader/RarPageLoader.kt b/server/src/main/kotlin/eu/kanade/tachiyomi/source/local/loader/RarPageLoader.kt index ea559bbf..298c6eab 100644 --- a/server/src/main/kotlin/eu/kanade/tachiyomi/source/local/loader/RarPageLoader.kt +++ b/server/src/main/kotlin/eu/kanade/tachiyomi/source/local/loader/RarPageLoader.kt @@ -4,59 +4,48 @@ import com.github.junrar.Archive import com.github.junrar.rarfile.FileHeader import eu.kanade.tachiyomi.util.lang.compareToCaseInsensitiveNaturalOrder import suwayomi.tachidesk.manga.impl.util.storage.ImageUtil -import java.io.ByteArrayInputStream -import java.io.ByteArrayOutputStream import java.io.File import java.io.InputStream +import java.io.PipedInputStream +import java.io.PipedOutputStream /** * Loader used to load a chapter from a .rar or .cbr file. */ class RarPageLoader(file: File) : PageLoader { - /** - * The rar archive to load pages from. - */ - private val archive = Archive(file) + private val rar = Archive(file) - /** - * The fully uncompressed files, to be used in case archive is solid. - */ - private var archiveMap = mutableMapOf() - - /** - * Returns an observable containing the pages found on this rar archive ordered with a natural - * comparator. - */ - override fun getPages(): List { - if (archive.mainHeader.isSolid) { - // Solid means that we need to read all the file sequentially - for (header in archive.fileHeaders) { - val baos = ByteArrayOutputStream() - archive.extractFile(header, baos) - archiveMap[header] = ByteArrayInputStream(baos.toByteArray()) - } - // After reading the full archive, proceed to filter and transform - return archive.fileHeaders - .filter { !it.isDirectory && ImageUtil.isImage(it.fileName) { archiveMap.getValue(it) } } - .sortedWith { f1, f2 -> f1.fileName.compareToCaseInsensitiveNaturalOrder(f2.fileName) } - .mapIndexed { i, header -> - val streamFn = { archiveMap.getValue(header) } - - ReaderPage(i).apply { - stream = streamFn - } - } - } - return archive.fileHeaders - .filter { !it.isDirectory && ImageUtil.isImage(it.fileName) { archive.getInputStream(it) } } + override suspend fun getPages(): List { + return rar.fileHeaders.asSequence() + .filter { !it.isDirectory && ImageUtil.isImage(it.fileName) { rar.getInputStream(it) } } .sortedWith { f1, f2 -> f1.fileName.compareToCaseInsensitiveNaturalOrder(f2.fileName) } .mapIndexed { i, header -> - val streamFn = { archive.getInputStream(header) } - ReaderPage(i).apply { - stream = streamFn + stream = { getStream(rar, header) } } } + .toList() + } + + override fun recycle() { + rar.close() + } + + /** + * Returns an input stream for the given [header]. + */ + private fun getStream(rar: Archive, header: FileHeader): InputStream { + val pipeIn = PipedInputStream() + val pipeOut = PipedOutputStream(pipeIn) + synchronized(this) { + try { + pipeOut.use { + rar.extractFile(header, it) + } + } catch (_: Exception) { + } + } + return pipeIn } } diff --git a/server/src/main/kotlin/eu/kanade/tachiyomi/source/local/loader/ZipPageLoader.kt b/server/src/main/kotlin/eu/kanade/tachiyomi/source/local/loader/ZipPageLoader.kt index 04c97ea5..4fcdbe1b 100644 --- a/server/src/main/kotlin/eu/kanade/tachiyomi/source/local/loader/ZipPageLoader.kt +++ b/server/src/main/kotlin/eu/kanade/tachiyomi/source/local/loader/ZipPageLoader.kt @@ -5,25 +5,26 @@ import org.apache.commons.compress.archivers.zip.ZipFile import suwayomi.tachidesk.manga.impl.util.storage.ImageUtil import java.io.File +/** + * Loader used to load a chapter from a .zip or .cbz file. + */ class ZipPageLoader(file: File) : PageLoader { - /** - * The zip file to load pages from. - */ + private val zip = ZipFile(file) - /** - * Returns an observable containing the pages found on this zip archive ordered with a natural - * comparator. - */ - override fun getPages(): List { - return zip.entries.toList() + override suspend fun getPages(): List { + return zip.entries.asSequence() .filter { !it.isDirectory && ImageUtil.isImage(it.name) { zip.getInputStream(it) } } .sortedWith { f1, f2 -> f1.name.compareToCaseInsensitiveNaturalOrder(f2.name) } .mapIndexed { i, entry -> - val streamFn = { zip.getInputStream(entry) } ReaderPage(i).apply { - stream = streamFn + stream = { zip.getInputStream(entry) } } } + .toList() + } + + override fun recycle() { + zip.close() } } diff --git a/server/src/main/kotlin/eu/kanade/tachiyomi/source/local/metadata/ComicInfo.kt b/server/src/main/kotlin/eu/kanade/tachiyomi/source/local/metadata/ComicInfo.kt new file mode 100644 index 00000000..f2ce77fe --- /dev/null +++ b/server/src/main/kotlin/eu/kanade/tachiyomi/source/local/metadata/ComicInfo.kt @@ -0,0 +1,162 @@ +package eu.kanade.tachiyomi.source.local.metadata + +import eu.kanade.tachiyomi.source.model.SManga +import kotlinx.serialization.Serializable +import nl.adaptivity.xmlutil.serialization.XmlElement +import nl.adaptivity.xmlutil.serialization.XmlSerialName +import nl.adaptivity.xmlutil.serialization.XmlValue + +const val COMIC_INFO_FILE = "ComicInfo.xml" + +fun SManga.copyFromComicInfo(comicInfo: ComicInfo) { + comicInfo.series?.let { title = it.value } + comicInfo.writer?.let { author = it.value } + comicInfo.summary?.let { description = it.value } + + listOfNotNull( + comicInfo.genre?.value, + comicInfo.tags?.value, + comicInfo.categories?.value + ) + .distinct() + .joinToString(", ") { it.trim() } + .takeIf { it.isNotEmpty() } + ?.let { genre = it } + + listOfNotNull( + comicInfo.penciller?.value, + comicInfo.inker?.value, + comicInfo.colorist?.value, + comicInfo.letterer?.value, + comicInfo.coverArtist?.value + ) + .flatMap { it.split(", ") } + .distinct() + .joinToString(", ") { it.trim() } + .takeIf { it.isNotEmpty() } + ?.let { artist = it } + + status = ComicInfoPublishingStatus.toSMangaValue(comicInfo.publishingStatus?.value) +} + +@Serializable +@XmlSerialName("ComicInfo", "", "") +data class ComicInfo( + val title: Title?, + val series: Series?, + val number: Number?, + val summary: Summary?, + val writer: Writer?, + val penciller: Penciller?, + val inker: Inker?, + val colorist: Colorist?, + val letterer: Letterer?, + val coverArtist: CoverArtist?, + val translator: Translator?, + val genre: Genre?, + val tags: Tags?, + val web: Web?, + val publishingStatus: PublishingStatusTachiyomi?, + val categories: CategoriesTachiyomi? +) { + @Suppress("UNUSED") + @XmlElement(false) + @XmlSerialName("xmlns:xsd", "", "") + val xmlSchema: String = "http://www.w3.org/2001/XMLSchema" + + @Suppress("UNUSED") + @XmlElement(false) + @XmlSerialName("xmlns:xsi", "", "") + val xmlSchemaInstance: String = "http://www.w3.org/2001/XMLSchema-instance" + + @Serializable + @XmlSerialName("Title", "", "") + data class Title(@XmlValue(true) val value: String = "") + + @Serializable + @XmlSerialName("Series", "", "") + data class Series(@XmlValue(true) val value: String = "") + + @Serializable + @XmlSerialName("Number", "", "") + data class Number(@XmlValue(true) val value: String = "") + + @Serializable + @XmlSerialName("Summary", "", "") + data class Summary(@XmlValue(true) val value: String = "") + + @Serializable + @XmlSerialName("Writer", "", "") + data class Writer(@XmlValue(true) val value: String = "") + + @Serializable + @XmlSerialName("Penciller", "", "") + data class Penciller(@XmlValue(true) val value: String = "") + + @Serializable + @XmlSerialName("Inker", "", "") + data class Inker(@XmlValue(true) val value: String = "") + + @Serializable + @XmlSerialName("Colorist", "", "") + data class Colorist(@XmlValue(true) val value: String = "") + + @Serializable + @XmlSerialName("Letterer", "", "") + data class Letterer(@XmlValue(true) val value: String = "") + + @Serializable + @XmlSerialName("CoverArtist", "", "") + data class CoverArtist(@XmlValue(true) val value: String = "") + + @Serializable + @XmlSerialName("Translator", "", "") + data class Translator(@XmlValue(true) val value: String = "") + + @Serializable + @XmlSerialName("Genre", "", "") + data class Genre(@XmlValue(true) val value: String = "") + + @Serializable + @XmlSerialName("Tags", "", "") + data class Tags(@XmlValue(true) val value: String = "") + + @Serializable + @XmlSerialName("Web", "", "") + data class Web(@XmlValue(true) val value: String = "") + + // The spec doesn't have a good field for this + @Serializable + @XmlSerialName("PublishingStatusTachiyomi", "http://www.w3.org/2001/XMLSchema", "ty") + data class PublishingStatusTachiyomi(@XmlValue(true) val value: String = "") + + @Serializable + @XmlSerialName("Categories", "http://www.w3.org/2001/XMLSchema", "ty") + data class CategoriesTachiyomi(@XmlValue(true) val value: String = "") +} + +enum class ComicInfoPublishingStatus( + val comicInfoValue: String, + val sMangaModelValue: Int +) { + ONGOING("Ongoing", SManga.ONGOING), + COMPLETED("Completed", SManga.COMPLETED), + LICENSED("Licensed", SManga.LICENSED), + PUBLISHING_FINISHED("Publishing finished", SManga.PUBLISHING_FINISHED), + CANCELLED("Cancelled", SManga.CANCELLED), + ON_HIATUS("On hiatus", SManga.ON_HIATUS), + UNKNOWN("Unknown", SManga.UNKNOWN) + ; + + companion object { + fun toComicInfoValue(value: Long): String { + return entries.firstOrNull { it.sMangaModelValue == value.toInt() }?.comicInfoValue + ?: UNKNOWN.comicInfoValue + } + + fun toSMangaValue(value: String?): Int { + return entries.firstOrNull { it.comicInfoValue == value }?.sMangaModelValue + ?: UNKNOWN.sMangaModelValue + } + } +} diff --git a/server/src/main/kotlin/eu/kanade/tachiyomi/source/local/metadata/EpubFile.kt b/server/src/main/kotlin/eu/kanade/tachiyomi/source/local/metadata/EpubFile.kt new file mode 100644 index 00000000..e2a0de85 --- /dev/null +++ b/server/src/main/kotlin/eu/kanade/tachiyomi/source/local/metadata/EpubFile.kt @@ -0,0 +1,60 @@ +package eu.kanade.tachiyomi.source.local.metadata + +import eu.kanade.tachiyomi.source.model.SChapter +import eu.kanade.tachiyomi.source.model.SManga +import eu.kanade.tachiyomi.util.storage.EpubFile +import java.text.ParseException +import java.text.SimpleDateFormat +import java.util.Locale + +/** + * Fills manga metadata using this epub file's metadata. + */ +fun EpubFile.fillMangaMetadata(manga: SManga) { + val ref = getPackageHref() + val doc = getPackageDocument(ref) + + val creator = doc.getElementsByTag("dc:creator").first() + val description = doc.getElementsByTag("dc:description").first() + + manga.author = creator?.text() + manga.description = description?.text() +} + +/** + * Fills chapter metadata using this epub file's metadata. + */ +fun EpubFile.fillChapterMetadata(chapter: SChapter) { + val ref = getPackageHref() + val doc = getPackageDocument(ref) + + val title = doc.getElementsByTag("dc:title").first() + val publisher = doc.getElementsByTag("dc:publisher").first() + val creator = doc.getElementsByTag("dc:creator").first() + var date = doc.getElementsByTag("dc:date").first() + if (date == null) { + date = doc.select("meta[property=dcterms:modified]").first() + } + + if (title != null) { + chapter.name = title.text() + } + + if (publisher != null) { + chapter.scanlator = publisher.text() + } else if (creator != null) { + chapter.scanlator = creator.text() + } + + if (date != null) { + val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ", Locale.getDefault()) + try { + val parsedDate = dateFormat.parse(date.text()) + if (parsedDate != null) { + chapter.date_upload = parsedDate.time + } + } catch (e: ParseException) { + // Empty + } + } +} diff --git a/server/src/main/kotlin/eu/kanade/tachiyomi/source/local/metadata/MangaDetails.kt b/server/src/main/kotlin/eu/kanade/tachiyomi/source/local/metadata/MangaDetails.kt new file mode 100644 index 00000000..6e2f1d0d --- /dev/null +++ b/server/src/main/kotlin/eu/kanade/tachiyomi/source/local/metadata/MangaDetails.kt @@ -0,0 +1,13 @@ +package eu.kanade.tachiyomi.source.local.metadata + +import kotlinx.serialization.Serializable + +@Serializable +class MangaDetails( + val title: String? = null, + val author: String? = null, + val artist: String? = null, + val description: String? = null, + val genre: List? = null, + val status: Int? = null +) diff --git a/server/src/main/kotlin/eu/kanade/tachiyomi/util/chapter/ChapterRecognition.kt b/server/src/main/kotlin/eu/kanade/tachiyomi/util/chapter/ChapterRecognition.kt index dd44f42b..5d935adb 100644 --- a/server/src/main/kotlin/eu/kanade/tachiyomi/util/chapter/ChapterRecognition.kt +++ b/server/src/main/kotlin/eu/kanade/tachiyomi/util/chapter/ChapterRecognition.kt @@ -1,111 +1,78 @@ package eu.kanade.tachiyomi.util.chapter -import eu.kanade.tachiyomi.source.model.SChapter -import eu.kanade.tachiyomi.source.model.SManga - /** * -R> = regex conversion. */ object ChapterRecognition { + + private const val NUMBER_PATTERN = """([0-9]+)(\.[0-9]+)?(\.?[a-z]+)?""" + /** * All cases with Ch.xx * Mokushiroku Alice Vol.1 Ch. 4: Misrepresentation -R> 4 */ - private val basic = Regex("""(?<=ch\.) *([0-9]+)(\.[0-9]+)?(\.?[a-z]+)?""") + private val basic = Regex("""(?<=ch\.) *$NUMBER_PATTERN""") /** - * Regex used when only one number occurrence * Example: Bleach 567: Down With Snowwhite -R> 567 */ - private val occurrence = Regex("""([0-9]+)(\.[0-9]+)?(\.?[a-z]+)?""") - - /** - * Regex used when manga title removed - * Example: Solanin 028 Vol. 2 -> 028 Vol.2 -> 028Vol.2 -R> 028 - */ - private val withoutManga = Regex("""^([0-9]+)(\.[0-9]+)?(\.?[a-z]+)?""") + private val number = Regex(NUMBER_PATTERN) /** * Regex used to remove unwanted tags * Example Prison School 12 v.1 vol004 version1243 volume64 -R> Prison School 12 */ - private val unwanted = Regex("""(? One Piece 12special */ - private val unwantedWhiteSpace = Regex("""(\s)(extra|special|omake)""") + private val unwantedWhiteSpace = Regex("""\s(?=extra|special|omake)""") - fun parseChapterNumber(chapter: SChapter, manga: SManga) { + fun parseChapterNumber(mangaTitle: String, chapterName: String, chapterNumber: Double? = null): Double { // If chapter number is known return. - if (chapter.chapter_number == -2f || chapter.chapter_number > -1f) { - return + if (chapterNumber != null && (chapterNumber == -2.0 || chapterNumber > -1.0)) { + return chapterNumber } // Get chapter title with lower case - var name = chapter.name.lowercase() - - // Remove comma's from chapter. - name = name.replace(',', '.') - - // Remove unwanted white spaces. - unwantedWhiteSpace.findAll(name).let { - it.forEach { occurrence -> name = name.replace(occurrence.value, occurrence.value.trim()) } - } - - // Remove unwanted tags. - unwanted.findAll(name).let { - it.forEach { occurrence -> name = name.replace(occurrence.value, "") } - } - - // Check base case ch.xx - if (updateChapter(basic.find(name), chapter)) { - return - } - - // Check one number occurrence. - val occurrences: MutableList = arrayListOf() - occurrence.findAll(name).let { - it.forEach { occurrence -> occurrences.add(occurrence) } - } - - if (occurrences.size == 1) { - if (updateChapter(occurrences[0], chapter)) { - return - } - } + var name = chapterName.lowercase() // Remove manga title from chapter title. - val nameWithoutManga = name.replace(manga.title.lowercase(), "").trim() + name = name.replace(mangaTitle.lowercase(), "").trim() - // Check if first value is number after title remove. - if (updateChapter(withoutManga.find(nameWithoutManga), chapter)) { - return - } + // Remove comma's or hyphens. + name = name.replace(',', '.').replace('-', '.') + + // Remove unwanted white spaces. + name = unwantedWhiteSpace.replace(name, "") + + // Remove unwanted tags. + name = unwanted.replace(name, "") + + // Check base case ch.xx + basic.find(name)?.let { return getChapterNumberFromMatch(it) } // Take the first number encountered. - if (updateChapter(occurrence.find(nameWithoutManga), chapter)) { - return - } + number.find(name)?.let { return getChapterNumberFromMatch(it) } + + return chapterNumber ?: -1.0 } /** - * Check if volume is found and update chapter + * Check if chapter number is found and return it * @param match result of regex - * @param chapter chapter object - * @return true if volume is found + * @return chapter number if found else null */ - private fun updateChapter(match: MatchResult?, chapter: SChapter): Boolean { - match?.let { - val initial = it.groups[1]?.value?.toFloat()!! + private fun getChapterNumberFromMatch(match: MatchResult): Double { + return match.let { + val initial = it.groups[1]?.value?.toDouble()!! val subChapterDecimal = it.groups[2]?.value val subChapterAlpha = it.groups[3]?.value val addition = checkForDecimal(subChapterDecimal, subChapterAlpha) - chapter.chapter_number = initial.plus(addition) - return true + initial.plus(addition) } - return false } /** @@ -114,39 +81,39 @@ object ChapterRecognition { * @param alpha alpha value of regex * @return decimal/alpha float value */ - private fun checkForDecimal(decimal: String?, alpha: String?): Float { + private fun checkForDecimal(decimal: String?, alpha: String?): Double { if (!decimal.isNullOrEmpty()) { - return decimal.toFloat() + return decimal.toDouble() } if (!alpha.isNullOrEmpty()) { if (alpha.contains("extra")) { - return .99f + return 0.99 } if (alpha.contains("omake")) { - return .98f + return 0.98 } if (alpha.contains("special")) { - return .97f + return 0.97 } - return if (alpha[0] == '.') { - // Take value after (.) - parseAlphaPostFix(alpha[1]) - } else { - parseAlphaPostFix(alpha[0]) + val trimmedAlpha = alpha.trimStart('.') + if (trimmedAlpha.length == 1) { + return parseAlphaPostFix(trimmedAlpha[0]) } } - return .0f + return 0.0 } /** * x.a -> x.1, x.b -> x.2, etc */ - private fun parseAlphaPostFix(alpha: Char): Float { - return ("0." + (alpha.code - 96).toString()).toFloat() + private fun parseAlphaPostFix(alpha: Char): Double { + val number = alpha.code - ('a'.code - 1) + if (number >= 10) return 0.0 + return number / 10.0 } } diff --git a/server/src/main/kotlin/eu/kanade/tachiyomi/util/storage/EpubFile.kt b/server/src/main/kotlin/eu/kanade/tachiyomi/util/storage/EpubFile.kt index 149d9f9a..d73d98fd 100644 --- a/server/src/main/kotlin/eu/kanade/tachiyomi/util/storage/EpubFile.kt +++ b/server/src/main/kotlin/eu/kanade/tachiyomi/util/storage/EpubFile.kt @@ -1,7 +1,5 @@ package eu.kanade.tachiyomi.util.storage -import eu.kanade.tachiyomi.source.model.SChapter -import eu.kanade.tachiyomi.source.model.SManga import org.apache.commons.compress.archivers.zip.ZipArchiveEntry import org.apache.commons.compress.archivers.zip.ZipFile import org.jsoup.Jsoup @@ -9,9 +7,6 @@ import org.jsoup.nodes.Document import java.io.Closeable import java.io.File import java.io.InputStream -import java.text.ParseException -import java.text.SimpleDateFormat -import java.util.Locale /** * Wrapper over ZipFile to load files in epub format. @@ -49,58 +44,6 @@ class EpubFile(file: File) : Closeable { return zip.getEntry(name) } - /** - * Fills manga metadata using this epub file's metadata. - */ - fun fillMangaMetadata(manga: SManga) { - val ref = getPackageHref() - val doc = getPackageDocument(ref) - - val creator = doc.getElementsByTag("dc:creator").first() - val description = doc.getElementsByTag("dc:description").first() - - manga.author = creator?.text() - manga.description = description?.text() - } - - /** - * Fills chapter metadata using this epub file's metadata. - */ - fun fillChapterMetadata(chapter: SChapter) { - val ref = getPackageHref() - val doc = getPackageDocument(ref) - - val title = doc.getElementsByTag("dc:title").first() - val publisher = doc.getElementsByTag("dc:publisher").first() - val creator = doc.getElementsByTag("dc:creator").first() - var date = doc.getElementsByTag("dc:date").first() - if (date == null) { - date = doc.select("meta[property=dcterms:modified]").first() - } - - if (title != null) { - chapter.name = title.text() - } - - if (publisher != null) { - chapter.scanlator = publisher.text() - } else if (creator != null) { - chapter.scanlator = creator.text() - } - - if (date != null) { - val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ", Locale.getDefault()) - try { - val parsedDate = dateFormat.parse(date.text()) - if (parsedDate != null) { - chapter.date_upload = parsedDate.time - } - } catch (e: ParseException) { - // Empty - } - } - } - /** * Returns the path of all the images found in the epub file. */ @@ -114,7 +57,7 @@ class EpubFile(file: File) : Closeable { /** * Returns the path to the package document. */ - private fun getPackageHref(): String { + fun getPackageHref(): String { val meta = zip.getEntry(resolveZipPath("META-INF", "container.xml")) if (meta != null) { val metaDoc = zip.getInputStream(meta).use { Jsoup.parse(it, null, "") } @@ -129,7 +72,7 @@ class EpubFile(file: File) : Closeable { /** * Returns the package document where all the files are listed. */ - private fun getPackageDocument(ref: String): Document { + fun getPackageDocument(ref: String): Document { val entry = zip.getEntry(ref) return zip.getInputStream(entry).use { Jsoup.parse(it, null, "") } } @@ -137,9 +80,9 @@ class EpubFile(file: File) : Closeable { /** * Returns all the pages from the epub. */ - private fun getPagesFromDocument(document: Document): List { + fun getPagesFromDocument(document: Document): List { val pages = document.select("manifest > item") - .filter { element -> "application/xhtml+xml" == element.attr("media-type") } + .filter { node -> "application/xhtml+xml" == node.attr("media-type") } .associateBy { it.attr("id") } val spine = document.select("spine > itemref").map { it.attr("idref") } diff --git a/server/src/main/kotlin/suwayomi/tachidesk/graphql/mutations/ChapterMutation.kt b/server/src/main/kotlin/suwayomi/tachidesk/graphql/mutations/ChapterMutation.kt index 5361830c..b42cab68 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/graphql/mutations/ChapterMutation.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/graphql/mutations/ChapterMutation.kt @@ -11,7 +11,6 @@ import org.jetbrains.exposed.sql.update import suwayomi.tachidesk.graphql.types.ChapterMetaType import suwayomi.tachidesk.graphql.types.ChapterType import suwayomi.tachidesk.manga.impl.Chapter -import suwayomi.tachidesk.manga.impl.util.lang.awaitSingle import suwayomi.tachidesk.manga.impl.util.source.GetCatalogueSource.getCatalogueSourceOrNull import suwayomi.tachidesk.manga.model.table.ChapterMetaTable import suwayomi.tachidesk.manga.model.table.ChapterTable @@ -213,12 +212,12 @@ class ChapterMutation { val source = getCatalogueSourceOrNull(manga[MangaTable.sourceReference])!! return future { - source.fetchPageList( + source.getPageList( SChapter.create().apply { url = chapter[ChapterTable.url] name = chapter[ChapterTable.name] } - ).awaitSingle() + ) }.thenApply { pageList -> transaction { PageTable.deleteWhere { PageTable.chapter eq chapterId } diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Chapter.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Chapter.kt index d4acd712..68e83f2c 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Chapter.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Chapter.kt @@ -28,7 +28,6 @@ import org.jetbrains.exposed.sql.update import suwayomi.tachidesk.manga.impl.Manga.getManga import suwayomi.tachidesk.manga.impl.download.DownloadManager import suwayomi.tachidesk.manga.impl.download.DownloadManager.EnqueueInput -import suwayomi.tachidesk.manga.impl.util.lang.awaitSingle import suwayomi.tachidesk.manga.impl.util.source.GetCatalogueSource.getCatalogueSourceOrStub import suwayomi.tachidesk.manga.model.dataclass.ChapterDataClass import suwayomi.tachidesk.manga.model.dataclass.MangaChapterDataClass @@ -118,12 +117,13 @@ object Chapter { } val numberOfCurrentChapters = getCountOfMangaChapters(mangaId) - val chapterList = source.fetchChapterList(sManga).awaitSingle() + val chapterList = source.getChapterList(sManga) // Recognize number for new chapters. - chapterList.forEach { - (source as? HttpSource)?.prepareNewChapter(it, sManga) - ChapterRecognition.parseChapterNumber(it, sManga) + chapterList.forEach { chapter -> + (source as? HttpSource)?.prepareNewChapter(chapter, sManga) + val chapterNumber = ChapterRecognition.parseChapterNumber(manga.title, chapter.name, chapter.chapter_number.toDouble()) + chapter.chapter_number = chapterNumber.toFloat() } var now = Instant.now().epochSecond diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Manga.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Manga.kt index f09b728a..fa05127f 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Manga.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Manga.kt @@ -25,7 +25,6 @@ import org.kodein.di.conf.global import org.kodein.di.instance import suwayomi.tachidesk.manga.impl.MangaList.proxyThumbnailUrl import suwayomi.tachidesk.manga.impl.Source.getSource -import suwayomi.tachidesk.manga.impl.util.lang.awaitSingle import suwayomi.tachidesk.manga.impl.util.network.await import suwayomi.tachidesk.manga.impl.util.source.GetCatalogueSource.getCatalogueSourceOrNull import suwayomi.tachidesk.manga.impl.util.source.GetCatalogueSource.getCatalogueSourceOrStub @@ -105,7 +104,7 @@ object Manga { url = mangaEntry[MangaTable.url] title = mangaEntry[MangaTable.title] } - val networkManga = source.fetchMangaDetails(sManga).awaitSingle() + val networkManga = source.getMangaDetails(sManga) sManga.copyFrom(networkManga) transaction { diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/chapter/ChapterForDownload.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/chapter/ChapterForDownload.kt index a8881bc2..622e4d91 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/chapter/ChapterForDownload.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/chapter/ChapterForDownload.kt @@ -18,7 +18,6 @@ import org.jetbrains.exposed.sql.update import suwayomi.tachidesk.manga.impl.Page.getPageName import suwayomi.tachidesk.manga.impl.util.getChapterCbzPath import suwayomi.tachidesk.manga.impl.util.getChapterDownloadPath -import suwayomi.tachidesk.manga.impl.util.lang.awaitSingle import suwayomi.tachidesk.manga.impl.util.source.GetCatalogueSource.getCatalogueSourceOrStub import suwayomi.tachidesk.manga.impl.util.storage.ImageResponse import suwayomi.tachidesk.manga.model.dataclass.ChapterDataClass @@ -64,12 +63,12 @@ private class ChapterForDownload( val mangaEntry = transaction { MangaTable.select { MangaTable.id eq mangaId }.first() } val source = getCatalogueSourceOrStub(mangaEntry[MangaTable.sourceReference]) - return source.fetchPageList( + return source.getPageList( SChapter.create().apply { url = chapterEntry[ChapterTable.url] name = chapterEntry[ChapterTable.name] } - ).awaitSingle() + ) } private fun markAsNotDownloaded() {