Implement FlareSolverr (#844)

* Implement FlareSolverr * Oops
2024-01-23 18:48:55 -05:00
parent 9121a6341c
commit d658e07583
8 changed files with 207 additions and 191 deletions
@@ -7,30 +7,29 @@ package eu.kanade.tachiyomi.network
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at https://mozilla.org/MPL/2.0/. */

-// import android.content.Context
-// import eu.kanade.tachiyomi.BuildConfig
-// import eu.kanade.tachiyomi.data.preference.PreferencesHelper
-// import okhttp3.HttpUrl.Companion.toHttpUrl
-// import okhttp3.dnsoverhttps.DnsOverHttps
-// import okhttp3.logging.HttpLoggingInterceptor
-// import uy.kohesive.injekt.injectLazy
 import android.content.Context
 import eu.kanade.tachiyomi.network.interceptor.CloudflareInterceptor
 import eu.kanade.tachiyomi.network.interceptor.IgnoreGzipInterceptor
 import eu.kanade.tachiyomi.network.interceptor.UncaughtExceptionInterceptor
 import eu.kanade.tachiyomi.network.interceptor.UserAgentInterceptor
+import kotlinx.coroutines.DelicateCoroutinesApi
+import kotlinx.coroutines.GlobalScope
+import kotlinx.coroutines.flow.MutableStateFlow
+import kotlinx.coroutines.flow.drop
+import kotlinx.coroutines.flow.launchIn
+import kotlinx.coroutines.flow.onEach
 import mu.KotlinLogging
 import okhttp3.Cache
 import okhttp3.OkHttpClient
 import okhttp3.brotli.BrotliInterceptor
 import okhttp3.logging.HttpLoggingInterceptor
+import suwayomi.tachidesk.manga.impl.util.source.GetCatalogueSource
 import java.io.File
 import java.net.CookieHandler
 import java.net.CookieManager
 import java.net.CookiePolicy
 import java.util.concurrent.TimeUnit

-@Suppress("UNUSED_PARAMETER")
 class NetworkHelper(context: Context) {
    //    private val preferences: PreferencesHelper by injectLazy()

@@ -48,6 +47,26 @@ class NetworkHelper(context: Context) {
    }
    // Tachidesk <--

+    private val userAgent =
+        MutableStateFlow(
+            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " +
+                "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+        )
+
+    fun defaultUserAgentProvider(): String {
+        return userAgent.value
+    }
+
+    init {
+        @OptIn(DelicateCoroutinesApi::class)
+        userAgent
+            .drop(1)
+            .onEach {
+                GetCatalogueSource.unregisterAllCatalogueSources() // need to reset the headers
+            }
+            .launchIn(GlobalScope)
+    }
+
    private val baseClientBuilder: OkHttpClient.Builder
        get() {
            val builder =
@@ -63,7 +82,7 @@ class NetworkHelper(context: Context) {
                        ),
                    )
                    .addInterceptor(UncaughtExceptionInterceptor())
-                    .addInterceptor(UserAgentInterceptor())
+                    .addInterceptor(UserAgentInterceptor(::defaultUserAgentProvider))
                    .addNetworkInterceptor(IgnoreGzipInterceptor())
                    .addNetworkInterceptor(BrotliInterceptor)

@@ -78,14 +97,14 @@ class NetworkHelper(context: Context) {
                        }
                    },
                ).apply {
-                    level = HttpLoggingInterceptor.Level.BASIC
+                    level = HttpLoggingInterceptor.Level.HEADERS
                }
            builder.addNetworkInterceptor(httpLoggingInterceptor)
            // }

-            // builder.addInterceptor(
-            //     CloudflareInterceptor(context, cookieJar, ::defaultUserAgentProvider),
-            // )
+            builder.addInterceptor(
+                CloudflareInterceptor(setUserAgent = { userAgent.value = it }),
+            )

            // when (preferences.dohProvider().get()) {
            //     PREF_DOH_CLOUDFLARE -> builder.dohCloudflare()
@@ -108,9 +127,5 @@ class NetworkHelper(context: Context) {
 //    val client by lazy { baseClientBuilder.cache(Cache(cacheDir, cacheSize)).build() }
    val client by lazy { baseClientBuilder.build() }

-    val cloudflareClient by lazy {
-        client.newBuilder()
-            .addInterceptor(CloudflareInterceptor())
-            .build()
-    }
+    val cloudflareClient by lazy { client }
 }
@@ -21,16 +21,18 @@ class PersistentCookieStore(context: Context) : CookieStore {
    private val lock = ReentrantLock()

    init {
-        for ((key, value) in prefs.all) {
-            @Suppress("UNCHECKED_CAST")
-            val cookies = value as? Set<String>
-            if (cookies != null) {
+        val domains =
+            prefs.all.keys.map { it.substringBeforeLast(".") }
+                .toSet()
+        domains.forEach { domain ->
+            val cookies = prefs.getStringSet(domain, emptySet())
+            if (!cookies.isNullOrEmpty()) {
                try {
-                    val url = "http://$key".toHttpUrlOrNull() ?: continue
+                    val url = "http://$domain".toHttpUrlOrNull() ?: return@forEach
                    val nonExpiredCookies =
                        cookies.mapNotNull { Cookie.parse(url, it) }
                            .filter { !it.hasExpired() }
-                    cookieMap.put(key, nonExpiredCookies)
+                    cookieMap[domain] = nonExpiredCookies
                } catch (e: Exception) {
                    // Ignore
                }
@@ -1,39 +1,62 @@
 package eu.kanade.tachiyomi.network.interceptor

 import eu.kanade.tachiyomi.network.NetworkHelper
+import eu.kanade.tachiyomi.network.POST
+import eu.kanade.tachiyomi.network.awaitSuccess
+import eu.kanade.tachiyomi.network.parseAs
+import kotlinx.coroutines.runBlocking
+import kotlinx.coroutines.sync.Mutex
+import kotlinx.coroutines.sync.withLock
+import kotlinx.serialization.SerialName
+import kotlinx.serialization.Serializable
+import kotlinx.serialization.encodeToString
+import kotlinx.serialization.json.Json
 import mu.KotlinLogging
+import okhttp3.Cookie
+import okhttp3.HttpUrl
 import okhttp3.Interceptor
+import okhttp3.MediaType.Companion.toMediaType
+import okhttp3.Request
+import okhttp3.RequestBody.Companion.toRequestBody
 import okhttp3.Response
+import suwayomi.tachidesk.server.serverConfig
 import uy.kohesive.injekt.injectLazy
 import java.io.IOException
+import kotlin.time.Duration.Companion.seconds

-class CloudflareInterceptor : Interceptor {
+class CloudflareInterceptor(
+    private val setUserAgent: (String) -> Unit,
+) : Interceptor {
    private val logger = KotlinLogging.logger {}

    private val network: NetworkHelper by injectLazy()

-    @Suppress("UNUSED_VARIABLE", "UNREACHABLE_CODE")
    override fun intercept(chain: Interceptor.Chain): Response {
        val originalRequest = chain.request()

        logger.trace { "CloudflareInterceptor is being used." }

-        val originalResponse = chain.proceed(chain.request())
+        val originalResponse = chain.proceed(originalRequest)

        // Check if Cloudflare anti-bot is on
        if (!(originalResponse.code in ERROR_CODES && originalResponse.header("Server") in SERVER_CHECK)) {
            return originalResponse
        }

-        throw IOException("Cloudflare bypass currently disabled ")
+        if (!serverConfig.flareSolverrEnabled.value) {
+            throw IOException("Cloudflare bypass currently disabled")
+        }

        logger.debug { "Cloudflare anti-bot is on, CloudflareInterceptor is kicking in..." }

        return try {
            originalResponse.close()
-            network.cookieStore.remove(originalRequest.url.toUri())
+            // network.cookieStore.remove(originalRequest.url.toUri())

-            val request = originalRequest // resolveWithWebView(originalRequest)
+            val request =
+                runBlocking {
+                    CFClearance.resolveWithFlareSolverr(setUserAgent, originalRequest)
+                }

            chain.proceed(request)
        } catch (e: Exception) {
@@ -57,172 +80,140 @@ class CloudflareInterceptor : Interceptor {
 object CFClearance {
    private val logger = KotlinLogging.logger {}
    private val network: NetworkHelper by injectLazy()
+    private val json: Json by injectLazy()
+    private val jsonMediaType = "application/json".toMediaType()
+    private val mutex = Mutex()

-    /*init {
-        // Fix the default DriverJar issue by providing our own implementation
-        // ref: https://github.com/microsoft/playwright-java/issues/1138
-        System.setProperty("playwright.driver.impl", "suwayomi.tachidesk.server.util.DriverJar")
-    }
+    @Serializable
+    data class FlareSolverCookie(
+        val name: String,
+        val value: String,
+    )

-    fun resolveWithWebView(originalRequest: Request): Request {
-        val url = originalRequest.url.toString()
+    @Serializable
+    data class FlareSolverRequest(
+        val cmd: String,
+        val url: String,
+        val maxTimeout: Int? = null,
+        val session: List<String>? = null,
+        @SerialName("session_ttl_minutes")
+        val sessionTtlMinutes: Int? = null,
+        val cookies: List<FlareSolverCookie>? = null,
+        val returnOnlyCookies: Boolean? = null,
+        val proxy: String? = null,
+        val postData: String? = null, // only used with cmd 'request.post'
+    )

-        logger.debug { "resolveWithWebView($url)" }
+    @Serializable
+    data class FlareSolverSolutionCookie(
+        val name: String,
+        val value: String,
+        val domain: String,
+        val path: String,
+        val expires: Double? = null,
+        val size: Int? = null,
+        val httpOnly: Boolean,
+        val secure: Boolean,
+        val session: Boolean? = null,
+        val sameSite: String,
+    )

-        val cookies =
-            Playwright.create().use { playwright ->
-                playwright.chromium().launch(
-                    LaunchOptions()
-                        .setHeadless(false)
-                        .apply {
-                            if (serverConfig.socksProxyEnabled.value) {
-                                setProxy("socks5://${serverConfig.socksProxyHost.value}:${serverConfig.socksProxyPort.value}")
+    @Serializable
+    data class FlareSolverSolution(
+        val url: String,
+        val status: Int,
+        val headers: Map<String, String>? = null,
+        val response: String? = null,
+        val cookies: List<FlareSolverSolutionCookie>,
+        val userAgent: String,
+    )
+
+    @Serializable
+    data class FlareSolverResponse(
+        val solution: FlareSolverSolution,
+        val status: String,
+        val message: String,
+        val startTimestamp: Long,
+        val endTimestamp: Long,
+        val version: String,
+    )
+
+    suspend fun resolveWithFlareSolverr(
+        setUserAgent: (String) -> Unit,
+        originalRequest: Request,
+    ): Request {
+        val flareSolverResponse =
+            with(json) {
+                mutex.withLock {
+                    network.client.newCall(
+                        POST(
+                            url = serverConfig.flareSolverrUrl.value.removeSuffix("/") + "/v1",
+                            body =
+                                Json.encodeToString(
+                                    FlareSolverRequest(
+                                        "request.get",
+                                        originalRequest.url.toString(),
+                                        cookies =
+                                            network.cookieStore.get(originalRequest.url).map {
+                                                FlareSolverCookie(it.name, it.value)
+                                            },
+                                        returnOnlyCookies = true,
+                                        maxTimeout =
+                                            serverConfig.flareSolverrTimeout.value
+                                                .seconds
+                                                .inWholeMilliseconds
+                                                .toInt(),
+                                    ),
+                                ).toRequestBody(jsonMediaType),
+                        ),
+                    ).awaitSuccess().parseAs<FlareSolverResponse>()
+                }
+            }
+
+        if (flareSolverResponse.solution.status in 200..299) {
+            setUserAgent(flareSolverResponse.solution.userAgent)
+            val cookies =
+                flareSolverResponse.solution.cookies
+                    .map { cookie ->
+                        Cookie.Builder()
+                            .name(cookie.name)
+                            .value(cookie.value)
+                            .domain(cookie.domain)
+                            .path(cookie.path)
+                            .expiresAt(cookie.expires?.takeUnless { it < 0.0 }?.toLong() ?: Long.MAX_VALUE)
+                            .also {
+                                if (cookie.httpOnly) it.httpOnly()
+                                if (cookie.secure) it.secure()
                            }
-                        },
-                ).use { browser ->
-                    val userAgent = originalRequest.header("User-Agent")
-                    if (userAgent != null) {
-                        browser.newContext(Browser.NewContextOptions().setUserAgent(userAgent)).use { browserContext ->
-                            browserContext.newPage().use { getCookies(it, url) }
-                        }
-                    } else {
-                        browser.newPage().use { getCookies(it, url) }
+                            .build()
                    }
-                }
-            }
+                    .groupBy { it.domain }
+                    .flatMap { (domain, cookies) ->
+                        network.cookieStore.addAll(
+                            HttpUrl.Builder()
+                                .scheme("http")
+                                .host(domain.removePrefix("."))
+                                .build(),
+                            cookies,
+                        )

-        // Copy cookies to cookie store
-        cookies.groupBy { it.domain }.forEach { (domain, cookies) ->
-            network.cookieStore.addAll(
-                url =
-                    HttpUrl.Builder()
-                        .scheme("http")
-                        .host(domain)
-                        .build(),
-                cookies = cookies,
-            )
-        }
-        // Merge new and existing cookies for this request
-        // Find the cookies that we need to merge into this request
-        val convertedForThisRequest =
-            cookies.filter {
-                it.matches(originalRequest.url)
-            }
-        // Extract cookies from current request
-        val existingCookies =
-            Cookie.parseAll(
-                originalRequest.url,
-                originalRequest.headers,
-            )
-        // Filter out existing values of cookies that we are about to merge in
-        val filteredExisting =
-            existingCookies.filter { existing ->
-                convertedForThisRequest.none { converted -> converted.name == existing.name }
-            }
-        logger.trace { "Existing cookies" }
-        logger.trace { existingCookies.joinToString("; ") }
-        val newCookies = filteredExisting + convertedForThisRequest
-        logger.trace { "New cookies" }
-        logger.trace { newCookies.joinToString("; ") }
-        return originalRequest.newBuilder()
-            .header("Cookie", newCookies.joinToString("; ") { "${it.name}=${it.value}" })
-            .build()
-    }*/
-
-    fun getWebViewUserAgent(): String {
-        return "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
-        /*return try {
-            throw PlaywrightException("playwrite is diabled for v0.6.7")
-
-            Playwright.create().use { playwright ->
-                playwright.chromium().launch(
-                    LaunchOptions()
-                        .setHeadless(true),
-                ).use { browser ->
-                    browser.newPage().use { page ->
-                        val userAgent = page.evaluate("() => {return navigator.userAgent}") as String
-                        logger.debug { "WebView User-Agent is $userAgent" }
-                        return userAgent
+                        cookies
                    }
+            logger.trace { "New cookies\n${cookies.joinToString("; ")}" }
+            val finalCookies =
+                network.cookieStore.get(originalRequest.url).joinToString("; ", postfix = "; ") {
+                    "${it.name}=${it.value}"
                }
-            }
-        } catch (e: PlaywrightException) {
-            // Playwright might fail on headless environments like docker
-            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36"
-        }*/
-    }
-
-    /*private fun getCookies(
-        page: Page,
-        url: String,
-    ): List<Cookie> {
-        applyStealthInitScripts(page)
-        page.navigate(url)
-        val challengeResolved = waitForChallengeResolve(page)
-
-        return if (challengeResolved) {
-            val cookies = page.context().cookies()
-
-            logger.debug {
-                val userAgent = page.evaluate("() => {return navigator.userAgent}")
-                "Playwright User-Agent is $userAgent"
-            }
-
-            // Convert PlayWright cookies to OkHttp cookies
-            cookies.map {
-                Cookie.Builder()
-                    .domain(it.domain.removePrefix("."))
-                    .expiresAt(it.expires?.times(1000)?.toLong() ?: Long.MAX_VALUE)
-                    .name(it.name)
-                    .path(it.path)
-                    .value(it.value).apply {
-                        if (it.httpOnly) httpOnly()
-                        if (it.secure) secure()
-                    }.build()
-            }
+            logger.trace { "Final cookies\n$finalCookies" }
+            return originalRequest.newBuilder()
+                .header("Cookie", finalCookies)
+                .header("User-Agent", flareSolverResponse.solution.userAgent)
+                .build()
        } else {
            logger.debug { "Cloudflare challenge failed to resolve" }
            throw CloudflareBypassException()
        }
    }

-    // ref: https://github.com/vvanglro/cf-clearance/blob/44124a8f06d8d0ecf2bf558a027082ff88dab435/cf_clearance/stealth.py#L18
-    private val stealthInitScripts by lazy {
-        arrayOf(
-            ServerConfig::class.java.getResource("/cloudflare-js/canvas.fingerprinting.js")!!.readText(),
-            ServerConfig::class.java.getResource("/cloudflare-js/chrome.global.js")!!.readText(),
-            ServerConfig::class.java.getResource("/cloudflare-js/emulate.touch.js")!!.readText(),
-            ServerConfig::class.java.getResource("/cloudflare-js/navigator.permissions.js")!!.readText(),
-            ServerConfig::class.java.getResource("/cloudflare-js/navigator.webdriver.js")!!.readText(),
-            ServerConfig::class.java.getResource("/cloudflare-js/chrome.runtime.js")!!.readText(),
-            ServerConfig::class.java.getResource("/cloudflare-js/chrome.plugin.js")!!.readText(),
-        )
-    }
-
-    // ref: https://github.com/vvanglro/cf-clearance/blob/44124a8f06d8d0ecf2bf558a027082ff88dab435/cf_clearance/stealth.py#L76
-    private fun applyStealthInitScripts(page: Page) {
-        for (script in stealthInitScripts) {
-            page.addInitScript(script)
-        }
-    }
-
-    // ref: https://github.com/vvanglro/cf-clearance/blob/44124a8f06d8d0ecf2bf558a027082ff88dab435/cf_clearance/retry.py#L21
-    private fun waitForChallengeResolve(page: Page): Boolean {
-        // sometimes the user has to solve the captcha challenge manually, potentially wait a long time
-        val timeoutSeconds = 120
-        repeat(timeoutSeconds) {
-            page.waitForTimeout(1.seconds.toDouble(DurationUnit.MILLISECONDS))
-            val success =
-                try {
-                    page.querySelector("#challenge-form") == null
-                } catch (e: Exception) {
-                    logger.debug(e) { "query Error" }
-                    false
-                }
-            if (success) return true
-        }
-        return false
-    }*/
-
    private class CloudflareBypassException : Exception()
 }
@@ -1,10 +1,9 @@
 package eu.kanade.tachiyomi.network.interceptor

-import eu.kanade.tachiyomi.source.online.HttpSource
 import okhttp3.Interceptor
 import okhttp3.Response

-class UserAgentInterceptor : Interceptor {
+class UserAgentInterceptor(private val userAgentProvider: () -> String) : Interceptor {
    override fun intercept(chain: Interceptor.Chain): Response {
        val originalRequest = chain.request()

@@ -13,7 +12,7 @@ class UserAgentInterceptor : Interceptor {
                originalRequest
                    .newBuilder()
                    .removeHeader("User-Agent")
-                    .addHeader("User-Agent", HttpSource.DEFAULT_USER_AGENT)
+                    .addHeader("User-Agent", userAgentProvider())
                    .build()
            chain.proceed(newRequest)
        } else {
@@ -4,7 +4,6 @@ import eu.kanade.tachiyomi.network.GET
 import eu.kanade.tachiyomi.network.NetworkHelper
 import eu.kanade.tachiyomi.network.asObservableSuccess
 import eu.kanade.tachiyomi.network.awaitSuccess
-import eu.kanade.tachiyomi.network.interceptor.CFClearance.getWebViewUserAgent
 import eu.kanade.tachiyomi.network.newCachelessCallWithProgress
 import eu.kanade.tachiyomi.source.CatalogueSource
 import eu.kanade.tachiyomi.source.model.FilterList
@@ -107,7 +106,7 @@ abstract class HttpSource : CatalogueSource {
     */
    protected open fun headersBuilder() =
        Headers.Builder().apply {
-            add("User-Agent", DEFAULT_USER_AGENT)
+            add("User-Agent", network.defaultUserAgentProvider())
        }

    /**
@@ -480,10 +479,6 @@ abstract class HttpSource : CatalogueSource {
     * Returns the list of filters for the source.
     */
    override fun getFilterList() = FilterList()
-
-    companion object {
-        val DEFAULT_USER_AGENT by lazy { getWebViewUserAgent() }
-    }
 }

 class LicensedMangaChaptersException : Exception("Licensed - No chapters to show")
@@ -81,4 +81,8 @@ object GetCatalogueSource {
    fun unregisterCatalogueSource(sourceId: Long) {
        sourceCache.remove(sourceId)
    }
+
+    fun unregisterAllCatalogueSources() {
+        sourceCache.clear()
+    }
 }
@@ -131,6 +131,11 @@ class ServerConfig(getConfig: () -> Config, val moduleName: String = SERVER_CONF
    // local source
    val localSourcePath: MutableStateFlow<String> by OverrideConfigValue(StringConfigAdapter)

+    // cloudflare bypass
+    val flareSolverrEnabled: MutableStateFlow<Boolean> by OverrideConfigValue(BooleanConfigAdapter)
+    val flareSolverrUrl: MutableStateFlow<String> by OverrideConfigValue(StringConfigAdapter)
+    val flareSolverrTimeout: MutableStateFlow<Int> by OverrideConfigValue(IntConfigAdapter)
+
    @OptIn(ExperimentalCoroutinesApi::class)
    fun <T> subscribeTo(
        flow: Flow<T>,
@@ -56,3 +56,8 @@ server.backupTTL = 14 # time in days - 0 to disable it - range: 1 <= n < ∞ - d

 # local source
 server.localSourcePath = ""
+
+# Cloudflare bypass
+server.flareSolverrEnabled = false
+server.flareSolverrUrl = "http://localhost:8191"
+server.flareSolverrTimeout = 60 # time in seconds