Libarchive refactor (#1249)

* Refactor archive support with libarchive

* Refactor archive support with libarchive

* Revert string resource changs

* Only mark archive formats as supported

Comic book archives should not be compressed.

* Fixup

* Remove epub from archive format list

* Move to mihon package

* Format

* Cleanup

Co-authored-by: Shamicen <84282253+Shamicen@users.noreply.github.com>
(cherry picked from commit 239c38982c4fd55d4d86b37fd9c3c51c3b47d098)

* handle incorrect passwords

* lint

* fixed broken encryption detection + small tweaks

* Add safeguard to prevent ArchiveInputStream from being closed twice (#967)

* fix: Add safeguard to prevent ArchiveInputStream from being closed twice

* detekt

* lint: Make detekt happy

---------

Co-authored-by: AntsyLich <59261191+AntsyLich@users.noreply.github.com>

(cherry picked from commit e620665dda9eb5cc39f09e6087ea4f60a3cbe150)

* fixed ArchiveReaderMode CACHE_TO_DISK

* Added some missing SY --> comments

---------

Co-authored-by: FooIbar <118464521+fooibar@users.noreply.github.com>
Co-authored-by: Ahmad Ansori Palembani <46041660+null2264@users.noreply.github.com>
This commit is contained in:
Shamicen
2024-08-18 02:25:25 +02:00
committed by GitHub
parent 71f2daf8f3
commit 95c834581b
27 changed files with 477 additions and 749 deletions
@@ -1,5 +1,6 @@
package eu.kanade.tachiyomi.data.coil
import android.app.Application
import android.graphics.Bitmap
import android.os.Build
import coil3.ImageLoader
@@ -11,37 +12,37 @@ import coil3.decode.ImageSource
import coil3.fetch.SourceFetchResult
import coil3.request.Options
import coil3.request.bitmapConfig
import com.hippo.unifile.UniFile
import eu.kanade.tachiyomi.util.storage.CbzCrypto
import eu.kanade.tachiyomi.util.storage.CbzCrypto.getCoverStream
import eu.kanade.tachiyomi.util.system.GLUtil
import net.lingala.zip4j.ZipFile
import net.lingala.zip4j.model.FileHeader
import mihon.core.common.archive.archiveReader
import okio.BufferedSource
import tachiyomi.core.common.util.system.ImageUtil
import tachiyomi.decoder.ImageDecoder
import uy.kohesive.injekt.Injekt
import uy.kohesive.injekt.api.get
import java.io.BufferedInputStream
/**
* A [Decoder] that uses built-in [ImageDecoder] to decode images that is not supported by the system.
*/
class TachiyomiImageDecoder(private val resources: ImageSource, private val options: Options) : Decoder {
private val context = Injekt.get<Application>()
override suspend fun decode(): DecodeResult {
// SY -->
var zip4j: ZipFile? = null
var entry: FileHeader? = null
var coverStream: BufferedInputStream? = null
if (resources.sourceOrNull()?.peek()?.use { CbzCrypto.detectCoverImageArchive(it.inputStream()) } == true) {
if (resources.source().peek().use { ImageUtil.findImageType(it.inputStream()) == null }) {
zip4j = ZipFile(resources.file().toFile().absolutePath)
entry = zip4j.fileHeaders.firstOrNull {
it.fileName.equals(CbzCrypto.DEFAULT_COVER_NAME, ignoreCase = true)
}
if (zip4j.isEncrypted) zip4j.setPassword(CbzCrypto.getDecryptedPasswordCbz())
coverStream = UniFile.fromFile(resources.file().toFile())
?.archiveReader(context = context)
?.getCoverStream()
}
}
val decoder = resources.sourceOrNull()?.use {
zip4j.use { zipFile ->
ImageDecoder.newInstance(zipFile?.getInputStream(entry) ?: it.inputStream(), options.cropBorders, displayProfile)
coverStream.use { coverStream ->
ImageDecoder.newInstance(coverStream ?: it.inputStream(), options.cropBorders, displayProfile)
}
}
// SY <--
@@ -44,10 +44,10 @@ import kotlinx.coroutines.flow.update
import kotlinx.coroutines.launch
import kotlinx.coroutines.supervisorScope
import logcat.LogPriority
import mihon.core.common.archive.ZipWriter
import nl.adaptivity.xmlutil.serialization.XML
import okhttp3.Response
import tachiyomi.core.common.i18n.stringResource
import tachiyomi.core.common.storage.addFilesToZip
import tachiyomi.core.common.storage.extension
import tachiyomi.core.common.util.lang.launchIO
import tachiyomi.core.common.util.lang.launchNow
@@ -65,12 +65,8 @@ import tachiyomi.domain.track.interactor.GetTracks
import tachiyomi.i18n.MR
import uy.kohesive.injekt.Injekt
import uy.kohesive.injekt.api.get
import java.io.BufferedOutputStream
import java.io.File
import java.util.Locale
import java.util.zip.CRC32
import java.util.zip.ZipEntry
import java.util.zip.ZipOutputStream
/**
* This class is the one in charge of downloading chapters.
@@ -619,70 +615,19 @@ class Downloader(
tmpDir: UniFile,
) {
// SY -->
if (CbzCrypto.getPasswordProtectDlPref() && CbzCrypto.isPasswordSet()) {
archiveEncryptedChapter(mangaDir, dirname, tmpDir)
return
}
val encrypt = CbzCrypto.getPasswordProtectDlPref() && CbzCrypto.isPasswordSet()
// SY <--
val zip = mangaDir.createFile("$dirname.cbz$TMP_DIR_SUFFIX")!!
ZipOutputStream(BufferedOutputStream(zip.openOutputStream())).use { zipOut ->
zipOut.setMethod(ZipEntry.STORED)
tmpDir.listFiles()?.forEach { img ->
img.openInputStream().use { input ->
val data = input.readBytes()
val size = img.length()
val entry = ZipEntry(img.name).apply {
val crc = CRC32().apply {
update(data)
}
setCrc(crc.value)
compressedSize = size
setSize(size)
}
zipOut.putNextEntry(entry)
zipOut.write(data)
}
ZipWriter(context, zip, /* SY --> */ encrypt /* SY <-- */).use { writer ->
tmpDir.listFiles()?.forEach { file ->
writer.write(file)
}
}
zip.renameTo("$dirname.cbz")
tmpDir.delete()
}
// SY -->
private fun archiveEncryptedChapter(
mangaDir: UniFile,
dirname: String,
tmpDir: UniFile,
) {
tmpDir.filePath?.let { addPaddingToImage(File(it)) }
tmpDir.listFiles()?.toList()?.let { files ->
mangaDir.createFile("$dirname.cbz$TMP_DIR_SUFFIX")
?.addFilesToZip(files, CbzCrypto.getDecryptedPasswordCbz())
}
mangaDir.findFile("$dirname.cbz$TMP_DIR_SUFFIX")?.renameTo("$dirname.cbz")
tmpDir.delete()
}
private fun addPaddingToImage(imageDir: File) {
imageDir.listFiles()
// using ImageUtils isImage and findImageType functions causes IO errors when deleting files to set Exif Metadata
// it should be safe to assume that all files with image extensions are actual images at this point
?.filter {
it.extension.equals("jpg", true) ||
it.extension.equals("jpeg", true) ||
it.extension.equals("png", true) ||
it.extension.equals("webp", true)
}
?.forEach { ImageUtil.addPaddingToImageExif(it) }
}
// SY <--
/**
* Creates a ComicInfo.xml file inside the given directory.
*/
@@ -1,9 +1,6 @@
package eu.kanade.tachiyomi.ui.reader.loader
import android.app.Application
import android.os.Build
import com.github.junrar.Archive
import com.github.junrar.rarfile.FileHeader
import com.hippo.unifile.UniFile
import eu.kanade.tachiyomi.source.model.Page
import eu.kanade.tachiyomi.ui.reader.model.ReaderPage
@@ -16,80 +13,69 @@ import kotlinx.coroutines.async
import kotlinx.coroutines.runBlocking
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
import tachiyomi.core.common.storage.UniFileTempFileManager
import mihon.core.common.archive.ArchiveReader
import tachiyomi.core.common.util.system.ImageUtil
import uy.kohesive.injekt.injectLazy
import java.io.File
import java.io.InputStream
import java.io.PipedInputStream
import java.io.PipedOutputStream
import java.util.concurrent.Executors
/**
* Loader used to load a chapter from a .rar or .cbr file.
* Loader used to load a chapter from an archive file.
*/
internal class RarPageLoader(file: UniFile) : PageLoader() {
internal class ArchivePageLoader(private val reader: ArchiveReader) : PageLoader() {
// SY -->
private val tempFileManager: UniFileTempFileManager by injectLazy()
private val rar = if (Build.VERSION.SDK_INT < Build.VERSION_CODES.N) {
Archive(tempFileManager.createTempFile(file))
} else {
Archive(file.openInputStream())
}
private val mutex = Mutex()
private val context: Application by injectLazy()
private val readerPreferences: ReaderPreferences by injectLazy()
private val tmpDir = File(context.externalCacheDir, "reader_${file.hashCode()}").also {
private val tmpDir = File(context.externalCacheDir, "reader_${reader.archiveHashCode}").also {
it.deleteRecursively()
}
init {
reader.wrongPassword?.let { wrongPassword ->
if (wrongPassword) {
error("Incorrect archive password")
}
}
if (readerPreferences.archiveReaderMode().get() == ReaderPreferences.ArchiveReaderMode.CACHE_TO_DISK) {
tmpDir.mkdirs()
rar.fileHeaders.asSequence()
.filter { !it.isDirectory && ImageUtil.isImage(it.fileName) { rar.getInputStream(it) } }
.sortedWith { f1, f2 -> f1.fileName.compareToCaseInsensitiveNaturalOrder(f2.fileName) }
.forEach { header ->
File(tmpDir, header.fileName.substringAfterLast("/"))
.also { it.createNewFile() }
.outputStream()
.use { output ->
rar.getInputStream(header).use { input ->
input.copyTo(output)
reader.useEntries { entries ->
entries
.filter { it.isFile && ImageUtil.isImage(it.name) { reader.getInputStream(it.name)!! } }
.sortedWith { f1, f2 -> f1.name.compareToCaseInsensitiveNaturalOrder(f2.name) }
.forEach { entry ->
File(tmpDir, entry.name.substringAfterLast("/"))
.also { it.createNewFile() }
.outputStream()
.use { output ->
reader.getInputStream(entry.name)?.use { input ->
input.copyTo(output)
}
}
}
}
}
}
}
}
// SY <--
override var isLocal: Boolean = true
/**
* Pool for copying compressed files to an input stream.
*/
private val pool = Executors.newFixedThreadPool(1)
override suspend fun getPages(): List<ReaderPage> {
override suspend fun getPages(): List<ReaderPage> = reader.useEntries { entries ->
// SY -->
if (readerPreferences.archiveReaderMode().get() == ReaderPreferences.ArchiveReaderMode.CACHE_TO_DISK) {
return DirectoryPageLoader(UniFile.fromFile(tmpDir)!!).getPages()
}
val mutex = Mutex()
// SY <--
return rar.fileHeaders.asSequence()
.filter { !it.isDirectory && ImageUtil.isImage(it.fileName) { rar.getInputStream(it) } }
.sortedWith { f1, f2 -> f1.fileName.compareToCaseInsensitiveNaturalOrder(f2.fileName) }
.mapIndexed { i, header ->
entries
.filter { it.isFile && ImageUtil.isImage(it.name) { reader.getInputStream(it.name)!! } }
.sortedWith { f1, f2 -> f1.name.compareToCaseInsensitiveNaturalOrder(f2.name) }
.mapIndexed { i, entry ->
// SY -->
val imageBytesDeferred: Deferred<ByteArray>? =
when (readerPreferences.archiveReaderMode().get()) {
ReaderPreferences.ArchiveReaderMode.LOAD_INTO_MEMORY -> {
CoroutineScope(Dispatchers.IO).async {
mutex.withLock {
getStream(header).buffered().use { stream ->
reader.getInputStream(entry.name)!!.buffered().use { stream ->
stream.readBytes()
}
}
@@ -98,12 +84,11 @@ internal class RarPageLoader(file: UniFile) : PageLoader() {
else -> null
}
val imageBytes by lazy { runBlocking { imageBytesDeferred?.await() } }
// SY <--
ReaderPage(i).apply {
// SY -->
stream = { imageBytes?.copyOf()?.inputStream() ?: getStream(header) }
stream = { imageBytes?.copyOf()?.inputStream() ?: reader.getInputStream(entry.name)!! }
// SY <--
status = Page.State.READY
}
@@ -117,27 +102,9 @@ internal class RarPageLoader(file: UniFile) : PageLoader() {
override fun recycle() {
super.recycle()
rar.close()
reader.close()
// SY -->
tmpDir.deleteRecursively()
// SY <--
pool.shutdown()
}
/**
* Returns an input stream for the given [header].
*/
private fun getStream(header: FileHeader): InputStream {
val pipeIn = PipedInputStream()
val pipeOut = PipedOutputStream(pipeIn)
pool.execute {
try {
pipeOut.use {
rar.extractFile(header, it)
}
} catch (e: Exception) {
}
}
return pipeIn
}
}
@@ -1,7 +1,6 @@
package eu.kanade.tachiyomi.ui.reader.loader
import android.content.Context
import com.github.junrar.exception.UnsupportedRarV5Exception
import eu.kanade.tachiyomi.data.download.DownloadManager
import eu.kanade.tachiyomi.data.download.DownloadProvider
import eu.kanade.tachiyomi.source.Source
@@ -9,6 +8,7 @@ import eu.kanade.tachiyomi.source.online.HttpSource
import eu.kanade.tachiyomi.source.online.all.MergedSource
import eu.kanade.tachiyomi.ui.reader.model.ReaderChapter
import eu.kanade.tachiyomi.ui.reader.setting.ReaderPreferences
import mihon.core.common.archive.archiveReader
import tachiyomi.core.common.i18n.stringResource
import tachiyomi.core.common.util.lang.withIOContext
import tachiyomi.core.common.util.system.logcat
@@ -124,34 +124,26 @@ class ChapterLoader(
source is LocalSource -> source.getFormat(chapter.chapter).let { format ->
when (format) {
is Format.Directory -> DirectoryPageLoader(format.file)
is Format.Zip -> ZipPageLoader(format.file, context)
is Format.Rar -> try {
RarPageLoader(format.file)
} catch (e: UnsupportedRarV5Exception) {
error(context.stringResource(MR.strings.loader_rar5_error))
}
is Format.Epub -> EpubPageLoader(format.file, context)
is Format.Archive -> ArchivePageLoader(format.file.archiveReader(context))
is Format.Epub -> EpubPageLoader(format.file.archiveReader(context))
}
}
else -> error(context.stringResource(MR.strings.loader_not_implemented_error))
}
}
// SY <--
isDownloaded -> DownloadPageLoader(chapter, manga, source, downloadManager, downloadProvider)
isDownloaded -> DownloadPageLoader(
chapter,
manga,
source,
downloadManager,
downloadProvider,
)
source is LocalSource -> source.getFormat(chapter.chapter).let { format ->
when (format) {
is Format.Directory -> DirectoryPageLoader(format.file)
// SY -->
is Format.Zip -> ZipPageLoader(format.file, context)
is Format.Rar -> try {
RarPageLoader(format.file)
// SY <--
} catch (e: UnsupportedRarV5Exception) {
error(context.stringResource(MR.strings.loader_rar5_error))
}
// SY -->
is Format.Epub -> EpubPageLoader(format.file, context)
// SY <--
is Format.Archive -> ArchivePageLoader(format.file.archiveReader(context))
is Format.Epub -> EpubPageLoader(format.file.archiveReader(context))
}
}
source is HttpSource -> HttpPageLoader(chapter, source)
@@ -10,6 +10,7 @@ import eu.kanade.tachiyomi.source.Source
import eu.kanade.tachiyomi.source.model.Page
import eu.kanade.tachiyomi.ui.reader.model.ReaderChapter
import eu.kanade.tachiyomi.ui.reader.model.ReaderPage
import mihon.core.common.archive.archiveReader
import tachiyomi.domain.manga.model.Manga
import uy.kohesive.injekt.injectLazy
@@ -26,7 +27,7 @@ internal class DownloadPageLoader(
private val context: Application by injectLazy()
private var zipPageLoader: ZipPageLoader? = null
private var archivePageLoader: ArchivePageLoader? = null
override var isLocal: Boolean = true
@@ -42,13 +43,11 @@ internal class DownloadPageLoader(
override fun recycle() {
super.recycle()
zipPageLoader?.recycle()
archivePageLoader?.recycle()
}
private suspend fun getPagesFromArchive(file: UniFile): List<ReaderPage> {
// SY -->
val loader = ZipPageLoader(file, context).also { zipPageLoader = it }
// SY <--
val loader = ArchivePageLoader(file.archiveReader(context)).also { archivePageLoader = it }
return loader.getPages()
}
@@ -64,6 +63,6 @@ internal class DownloadPageLoader(
}
override suspend fun loadPage(page: ReaderPage) {
zipPageLoader?.loadPage(page)
archivePageLoader?.loadPage(page)
}
}
@@ -1,26 +1,23 @@
package eu.kanade.tachiyomi.ui.reader.loader
import android.content.Context
import com.hippo.unifile.UniFile
import eu.kanade.tachiyomi.source.model.Page
import eu.kanade.tachiyomi.ui.reader.model.ReaderPage
import eu.kanade.tachiyomi.util.storage.EpubFile
import mihon.core.common.archive.ArchiveReader
/**
* Loader used to load a chapter from a .epub file.
*/
// SY -->
internal class EpubPageLoader(file: UniFile, context: Context) : PageLoader() {
internal class EpubPageLoader(reader: ArchiveReader) : PageLoader() {
private val epub = EpubFile(file, context)
// SY <--
private val epub = EpubFile(reader)
override var isLocal: Boolean = true
override suspend fun getPages(): List<ReaderPage> {
return epub.getImagesFromPages()
.mapIndexed { i, path ->
val streamFn = { epub.getInputStream(epub.getEntry(path)!!) }
val streamFn = { epub.getInputStream(path)!! }
ReaderPage(i).apply {
stream = streamFn
status = Page.State.READY
@@ -1,168 +0,0 @@
package eu.kanade.tachiyomi.ui.reader.loader
import android.content.Context
import android.os.Build
import com.hippo.unifile.UniFile
import eu.kanade.tachiyomi.source.model.Page
import eu.kanade.tachiyomi.ui.reader.model.ReaderPage
import eu.kanade.tachiyomi.ui.reader.setting.ReaderPreferences
import eu.kanade.tachiyomi.util.lang.compareToCaseInsensitiveNaturalOrder
import eu.kanade.tachiyomi.util.storage.CbzCrypto
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Deferred
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.async
import kotlinx.coroutines.runBlocking
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
import org.apache.commons.compress.archivers.zip.ZipFile
import tachiyomi.core.common.i18n.stringResource
import tachiyomi.core.common.storage.UniFileTempFileManager
import tachiyomi.core.common.storage.isEncryptedZip
import tachiyomi.core.common.storage.openReadOnlyChannel
import tachiyomi.core.common.storage.testCbzPassword
import tachiyomi.core.common.storage.unzip
import tachiyomi.core.common.util.system.ImageUtil
import tachiyomi.i18n.sy.SYMR
import uy.kohesive.injekt.injectLazy
import java.io.File
import java.nio.channels.SeekableByteChannel
import net.lingala.zip4j.ZipFile as Zip4jFile
/**
* Loader used to load a chapter from a .zip or .cbz file.
*/
internal class ZipPageLoader(file: UniFile, context: Context) : PageLoader() {
// SY -->
private val channel: SeekableByteChannel = file.openReadOnlyChannel(context)
private val tempFileManager: UniFileTempFileManager by injectLazy()
private val readerPreferences: ReaderPreferences by injectLazy()
private val tmpDir = File(context.externalCacheDir, "reader_${file.hashCode()}").also {
it.deleteRecursively()
}
private val apacheZip: ZipFile? = if (!file.isEncryptedZip() && Build.VERSION.SDK_INT > Build.VERSION_CODES.N) {
ZipFile.Builder()
.setSeekableByteChannel(channel)
.get()
} else {
null
}
private val tmpFile =
if (
apacheZip == null &&
readerPreferences.archiveReaderMode().get() != ReaderPreferences.ArchiveReaderMode.CACHE_TO_DISK
) {
tempFileManager.createTempFile(file)
} else {
null
}
private val zip4j =
if (apacheZip == null && tmpFile != null) {
Zip4jFile(tmpFile)
} else {
null
}
init {
if (file.isEncryptedZip()) {
if (!file.testCbzPassword()) {
this.recycle()
throw IllegalStateException(context.stringResource(SYMR.strings.wrong_cbz_archive_password))
}
zip4j?.setPassword(CbzCrypto.getDecryptedPasswordCbz())
}
if (readerPreferences.archiveReaderMode().get() == ReaderPreferences.ArchiveReaderMode.CACHE_TO_DISK) {
file.unzip(tmpDir, onlyCopyImages = true)
}
}
// SY <--
override fun recycle() {
super.recycle()
apacheZip?.close()
// SY -->
zip4j?.close()
tmpDir.deleteRecursively()
}
override var isLocal: Boolean = true
override suspend fun getPages(): List<ReaderPage> {
if (readerPreferences.archiveReaderMode().get() == ReaderPreferences.ArchiveReaderMode.CACHE_TO_DISK) {
return DirectoryPageLoader(UniFile.fromFile(tmpDir)!!).getPages()
}
return if (apacheZip == null) {
loadZip4j()
} else {
loadApacheZip(apacheZip)
}
}
private fun loadZip4j(): List<ReaderPage> {
val mutex = Mutex()
return zip4j!!.fileHeaders.asSequence()
.filter { !it.isDirectory && ImageUtil.isImage(it.fileName) { zip4j.getInputStream(it) } }
.sortedWith { f1, f2 -> f1.fileName.compareToCaseInsensitiveNaturalOrder(f2.fileName) }
.mapIndexed { i, entry ->
val imageBytesDeferred: Deferred<ByteArray>? =
when (readerPreferences.archiveReaderMode().get()) {
ReaderPreferences.ArchiveReaderMode.LOAD_INTO_MEMORY -> {
CoroutineScope(Dispatchers.IO).async {
mutex.withLock {
zip4j.getInputStream(entry).buffered().use { stream ->
stream.readBytes()
}
}
}
}
else -> null
}
val imageBytes by lazy { runBlocking { imageBytesDeferred?.await() } }
ReaderPage(i).apply {
stream = { imageBytes?.copyOf()?.inputStream() ?: zip4j.getInputStream(entry) }
status = Page.State.READY
}
}.toList()
}
private fun loadApacheZip(zip: ZipFile): List<ReaderPage> {
val mutex = Mutex()
return zip.entries.asSequence()
.filter { !it.isDirectory && ImageUtil.isImage(it.name) { zip.getInputStream(it) } }
.sortedWith { f1, f2 -> f1.name.compareToCaseInsensitiveNaturalOrder(f2.name) }
.mapIndexed { i, entry ->
val imageBytesDeferred: Deferred<ByteArray>? =
when (readerPreferences.archiveReaderMode().get()) {
ReaderPreferences.ArchiveReaderMode.LOAD_INTO_MEMORY -> {
CoroutineScope(Dispatchers.IO).async {
mutex.withLock {
zip.getInputStream(entry).buffered().use { stream ->
stream.readBytes()
}
}
}
}
else -> null
}
val imageBytes by lazy { runBlocking { imageBytesDeferred?.await() } }
ReaderPage(i).apply {
stream = { imageBytes?.copyOf()?.inputStream() ?: zip.getInputStream(entry) }
status = Page.State.READY
}
}.toList()
}
// SY <--
/**
* No additional action required to load the page
*/
override suspend fun loadPage(page: ReaderPage) {
check(!isRecycled)
}
}