diff --git a/app/build.gradle.kts b/app/build.gradle.kts index 05c1417c0..7bb58571d 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -291,9 +291,6 @@ dependencies { testImplementation(kotlinx.coroutines.test) // SY --> - // Text distance (EH) - implementation(sylibs.simularity) - // Firebase (EH) implementation(platform(libs.firebase.bom)) implementation(libs.firebase.analytics) diff --git a/app/src/main/java/exh/smartsearch/BaseSmartSearchEngine.kt b/app/src/main/java/exh/smartsearch/BaseSmartSearchEngine.kt deleted file mode 100644 index 1c975fbf7..000000000 --- a/app/src/main/java/exh/smartsearch/BaseSmartSearchEngine.kt +++ /dev/null @@ -1,182 +0,0 @@ -package exh.smartsearch - -import info.debatty.java.stringsimilarity.NormalizedLevenshtein -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.async -import kotlinx.coroutines.supervisorScope -import java.util.Locale - -typealias SearchAction = suspend (String) -> List - -abstract class BaseSmartSearchEngine( - private val extraSearchParams: String? = null, - private val eligibleThreshold: Double = MIN_ELIGIBLE_THRESHOLD, -) { - private val normalizedLevenshtein = NormalizedLevenshtein() - - protected abstract fun getTitle(result: T): String - - protected suspend fun smartSearch(searchAction: SearchAction, title: String): T? { - val cleanedTitle = cleanSmartSearchTitle(title) - - val queries = getSmartSearchQueries(cleanedTitle) - - val eligibleManga = supervisorScope { - queries.map { query -> - async(Dispatchers.Default) { - val builtQuery = if (extraSearchParams != null) { - "$query ${extraSearchParams.trim()}" - } else { - query - } - - searchAction(builtQuery).map { - val cleanedMangaTitle = cleanSmartSearchTitle(getTitle(it)) - val normalizedDistance = normalizedLevenshtein.similarity(cleanedTitle, cleanedMangaTitle) - SearchEntry(it, normalizedDistance) - }.filter { (_, normalizedDistance) -> - normalizedDistance >= eligibleThreshold - } - } - }.flatMap { it.await() } - } - - return eligibleManga.maxByOrNull { it.dist }?.manga - } - - protected suspend fun normalSearch(searchAction: SearchAction, title: String): T? { - val eligibleManga = supervisorScope { - val searchQuery = if (extraSearchParams != null) { - "$title ${extraSearchParams.trim()}" - } else { - title - } - val searchResults = searchAction(searchQuery) - - if (searchResults.size == 1) { - return@supervisorScope listOf(SearchEntry(searchResults.first(), 0.0)) - } - - searchResults.map { - val normalizedDistance = normalizedLevenshtein.similarity(title, getTitle(it)) - SearchEntry(it, normalizedDistance) - }.filter { (_, normalizedDistance) -> - normalizedDistance >= eligibleThreshold - } - } - - return eligibleManga.maxByOrNull { it.dist }?.manga - } - - private fun cleanSmartSearchTitle(title: String): String { - val preTitle = title.lowercase(Locale.getDefault()) - - // Remove text in brackets - var cleanedTitle = removeTextInBrackets(preTitle, true) - if (cleanedTitle.length <= 5) { // Title is suspiciously short, try parsing it backwards - cleanedTitle = removeTextInBrackets(preTitle, false) - } - - // Strip chapter reference RU - cleanedTitle = cleanedTitle.replace(chapterRefCyrillicRegexp, " ").trim() - - // Strip non-special characters - val cleanedTitleEng = cleanedTitle.replace(titleRegex, " ") - - // Do not strip foreign language letters if cleanedTitle is too short - if (cleanedTitleEng.length <= 5) { - cleanedTitle = cleanedTitle.replace(titleCyrillicRegex, " ") - } else { - cleanedTitle = cleanedTitleEng - } - - // Strip splitters and consecutive spaces - cleanedTitle = cleanedTitle.trim().replace(" - ", " ").replace(consecutiveSpacesRegex, " ").trim() - - return cleanedTitle - } - - private fun removeTextInBrackets(text: String, readForward: Boolean): String { - val bracketPairs = listOf( - '(' to ')', - '[' to ']', - '<' to '>', - '{' to '}', - ) - var openingBracketPairs = bracketPairs.mapIndexed { index, (opening, _) -> - opening to index - }.toMap() - var closingBracketPairs = bracketPairs.mapIndexed { index, (_, closing) -> - closing to index - }.toMap() - - // Reverse pairs if reading backwards - if (!readForward) { - val tmp = openingBracketPairs - openingBracketPairs = closingBracketPairs - closingBracketPairs = tmp - } - - val depthPairs = bracketPairs.map { 0 }.toMutableList() - - val result = StringBuilder() - for (c in if (readForward) text else text.reversed()) { - val openingBracketDepthIndex = openingBracketPairs[c] - if (openingBracketDepthIndex != null) { - depthPairs[openingBracketDepthIndex]++ - } else { - val closingBracketDepthIndex = closingBracketPairs[c] - if (closingBracketDepthIndex != null) { - depthPairs[closingBracketDepthIndex]-- - } else { - @Suppress("ControlFlowWithEmptyBody") - if (depthPairs.all { it <= 0 }) { - result.append(c) - } else { - // In brackets, do not append to result - } - } - } - } - - return result.toString() - } - - private fun getSmartSearchQueries(cleanedTitle: String): List { - val splitCleanedTitle = cleanedTitle.split(" ") - val splitSortedByLargest = splitCleanedTitle.sortedByDescending { it.length } - - if (splitCleanedTitle.isEmpty()) { - return emptyList() - } - - // Search cleaned title - // Search two largest words - // Search largest word - // Search first two words - // Search first word - - val searchQueries = listOf( - listOf(cleanedTitle), - splitSortedByLargest.take(2), - splitSortedByLargest.take(1), - splitCleanedTitle.take(2), - splitCleanedTitle.take(1), - ) - - return searchQueries.map { - it.joinToString(" ").trim() - }.distinct() - } - - companion object { - const val MIN_ELIGIBLE_THRESHOLD = 0.4 - - private val titleRegex = Regex("[^a-zA-Z0-9- ]") - private val titleCyrillicRegex = Regex("[^\\p{L}0-9- ]") - private val consecutiveSpacesRegex = Regex(" +") - private val chapterRefCyrillicRegexp = Regex("""((- часть|- глава) \d*)""") - } -} - -data class SearchEntry(val manga: T, val dist: Double) diff --git a/app/src/main/java/exh/smartsearch/SmartLibrarySearchEngine.kt b/app/src/main/java/exh/smartsearch/SmartLibrarySearchEngine.kt index c0e0b956e..1da279bf1 100644 --- a/app/src/main/java/exh/smartsearch/SmartLibrarySearchEngine.kt +++ b/app/src/main/java/exh/smartsearch/SmartLibrarySearchEngine.kt @@ -1,5 +1,6 @@ package exh.smartsearch +import mihon.feature.migration.list.search.BaseSmartSearchEngine import tachiyomi.domain.library.model.LibraryManga class SmartLibrarySearchEngine( @@ -9,7 +10,7 @@ class SmartLibrarySearchEngine( override fun getTitle(result: LibraryManga) = result.manga.ogTitle suspend fun smartSearch(library: List, title: String): LibraryManga? = - smartSearch( + deepSearch( { query -> library.filter { it.manga.ogTitle.contains(query, true) } }, diff --git a/app/src/main/java/exh/smartsearch/SmartSourceSearchEngine.kt b/app/src/main/java/exh/smartsearch/SmartSourceSearchEngine.kt deleted file mode 100644 index 766fb146d..000000000 --- a/app/src/main/java/exh/smartsearch/SmartSourceSearchEngine.kt +++ /dev/null @@ -1,27 +0,0 @@ -package exh.smartsearch - -import eu.kanade.tachiyomi.source.CatalogueSource -import eu.kanade.tachiyomi.source.model.FilterList -import eu.kanade.tachiyomi.source.model.SManga -import mihon.domain.manga.model.toDomainManga -import tachiyomi.domain.manga.model.Manga - -class SmartSourceSearchEngine( - extraSearchParams: String? = null, -) : BaseSmartSearchEngine(extraSearchParams) { - - override fun getTitle(result: SManga) = result.originalTitle - - suspend fun smartSearch(source: CatalogueSource, title: String): Manga? = - smartSearch(makeSearchAction(source), title).let { - it?.toDomainManga(source.id) - } - - suspend fun normalSearch(source: CatalogueSource, title: String): Manga? = - normalSearch(makeSearchAction(source), title).let { - it?.toDomainManga(source.id) - } - - private fun makeSearchAction(source: CatalogueSource): SearchAction = - { query -> source.getSearchManga(1, query, FilterList()).mangas } -} diff --git a/app/src/main/java/exh/ui/smartsearch/SmartSearchScreenModel.kt b/app/src/main/java/exh/ui/smartsearch/SmartSearchScreenModel.kt index d332f4c8b..b0ffaad29 100644 --- a/app/src/main/java/exh/ui/smartsearch/SmartSearchScreenModel.kt +++ b/app/src/main/java/exh/ui/smartsearch/SmartSearchScreenModel.kt @@ -4,8 +4,8 @@ import cafe.adriel.voyager.core.model.StateScreenModel import cafe.adriel.voyager.core.model.screenModelScope import eu.kanade.tachiyomi.source.CatalogueSource import eu.kanade.tachiyomi.ui.browse.source.SourcesScreen -import exh.smartsearch.SmartSourceSearchEngine import kotlinx.coroutines.CancellationException +import mihon.feature.migration.list.search.SmartSourceSearchEngine import tachiyomi.core.common.util.lang.launchIO import tachiyomi.domain.manga.interactor.NetworkToLocalManga import tachiyomi.domain.manga.model.Manga @@ -14,19 +14,19 @@ import uy.kohesive.injekt.Injekt import uy.kohesive.injekt.api.get class SmartSearchScreenModel( - private val sourceId: Long, + sourceId: Long, private val config: SourcesScreen.SmartSearchConfig, private val networkToLocalManga: NetworkToLocalManga = Injekt.get(), - private val sourceManager: SourceManager = Injekt.get(), + sourceManager: SourceManager = Injekt.get(), ) : StateScreenModel(null) { - private val smartSearchEngine = SmartSourceSearchEngine() + private val smartSearchEngine = SmartSourceSearchEngine(null) val source = sourceManager.get(sourceId) as CatalogueSource init { screenModelScope.launchIO { val result = try { - val resultManga = smartSearchEngine.smartSearch(source, config.origTitle) + val resultManga = smartSearchEngine.deepSearch(source, config.origTitle) if (resultManga != null) { val localManga = networkToLocalManga(resultManga) SearchResults.Found(localManga) diff --git a/gradle/sy.versions.toml b/gradle/sy.versions.toml index 81d92ee03..fff9970b1 100644 --- a/gradle/sy.versions.toml +++ b/gradle/sy.versions.toml @@ -2,7 +2,6 @@ koin = "4.0.4" [libraries] -simularity = "info.debatty:java-string-similarity:2.0.0" xlog = "com.elvishew:xlog:1.11.1" ratingbar = "me.zhanghai.android.materialratingbar:library:1.4.0"