Background captcha resolving

app/build.gradle

@@ -87,6 +87,7 @@ android {
 			'-opt-in=coil3.annotation.InternalCoilApi',
 			'-opt-in=kotlinx.serialization.ExperimentalSerializationApi',
 			'-Xjspecify-annotations=strict',
+			'-Xannotation-default-target=first-only',
 			'-Xtype-enhancement-improvements-strict-mode'
 		]
 	}

app/src/main/kotlin/org/koitharu/kotatsu/core/exceptions/resolve/CaptchaHandler.kt

@@ -7,6 +7,7 @@ import android.content.Context
 import android.content.Intent
 import android.os.Build
 import android.provider.Settings
+import androidx.annotation.CheckResult
 import androidx.annotation.RequiresPermission
 import androidx.collection.MutableScatterMap
 import androidx.core.app.NotificationChannelCompat
@@ -43,6 +44,7 @@ import org.koitharu.kotatsu.core.model.UnknownMangaSource
 import org.koitharu.kotatsu.core.model.getTitle
 import org.koitharu.kotatsu.core.model.isNsfw
 import org.koitharu.kotatsu.core.nav.AppRouter
+import org.koitharu.kotatsu.core.network.webview.WebViewExecutor
 import org.koitharu.kotatsu.core.parser.favicon.faviconUri
 import org.koitharu.kotatsu.core.prefs.SourceSettings
 import org.koitharu.kotatsu.core.util.ext.checkNotificationPermission
@@ -65,11 +67,13 @@ class CaptchaHandler @Inject constructor(
 	@LocalizedAppContext private val context: Context,
 	private val databaseProvider: Provider<MangaDatabase>,
 	private val coilProvider: Provider<ImageLoader>,
+	private val webViewExecutor: WebViewExecutor,
 ) : EventListener() {
 
 	private val exceptionMap = MutableScatterMap<MangaSource, CloudFlareProtectedException>()
 	private val mutex = Mutex()
 
+	@CheckResult
 	suspend fun handle(exception: CloudFlareException): Boolean = handleException(exception.source, exception, true)
 
 	suspend fun discard(source: MangaSource) {
@@ -79,10 +83,18 @@ class CaptchaHandler @Inject constructor(
 	override fun onError(request: ImageRequest, result: ErrorResult) {
 		super.onError(request, result)
 		val e = result.throwable
-		if (e is CloudFlareException && request.extras[ignoreCaptchaKey] != true) {
+		if (e is CloudFlareException) {
 			val scope = request.lifecycle?.coroutineScope ?: processLifecycleScope
 			scope.launch {
-				handleException(e.source, e, true)
+				if (
+					handleException(
+						source = e.source,
+						exception = e,
+						notify = request.extras[suppressCaptchaKey] != true,
+					)
+				) {
+					coilProvider.get().enqueue(request) // TODO check if ok
+				}
 			}
 		}
 	}
@@ -90,11 +102,14 @@ class CaptchaHandler @Inject constructor(
 	private suspend fun handleException(
 		source: MangaSource,
 		exception: CloudFlareException?,
-		notify: Boolean
+		notify: Boolean,
 	): Boolean = withContext(Dispatchers.Default) {
 		if (source == UnknownMangaSource) {
 			return@withContext false
 		}
+		if (exception != null && webViewExecutor.tryResolveCaptcha(exception, RESOLVE_TIMEOUT)) {
+			return@withContext true
+		}
 		mutex.withLock {
 			var removedException: CloudFlareProtectedException? = null
 			if (exception is CloudFlareProtectedException) {
@@ -119,7 +134,7 @@ class CaptchaHandler @Inject constructor(
 				notify(exceptions)
 			}
 		}
-		true
+		false
 	}
 
 	@RequiresPermission(Manifest.permission.POST_NOTIFICATIONS)
@@ -234,7 +249,7 @@ class CaptchaHandler @Inject constructor(
 				.data(source.faviconUri())
 				.allowHardware(false)
 				.allowConversionToBitmap(true)
-				.ignoreCaptchaErrors()
+				.suppressCaptchaErrors()
 				.mangaSourceExtra(source)
 				.size(context.resources.getNotificationIconSize())
 				.scale(Scale.FILL)
@@ -260,11 +275,11 @@ class CaptchaHandler @Inject constructor(
 
 	companion object {
 
-		fun ImageRequest.Builder.ignoreCaptchaErrors() = apply {
-			extras[ignoreCaptchaKey] = true
+		fun ImageRequest.Builder.suppressCaptchaErrors() = apply {
+			extras[suppressCaptchaKey] = true
 		}
 
-		val ignoreCaptchaKey = Extras.Key(false)
+		private val suppressCaptchaKey = Extras.Key(false)
 
 		private const val CHANNEL_ID = "captcha"
 		private const val TAG = CHANNEL_ID
@@ -272,5 +287,6 @@ class CaptchaHandler @Inject constructor(
 		private const val GROUP_NOTIFICATION_ID = 34
 		private const val SETTINGS_ACTION_CODE = 3
 		private const val ACTION_DISCARD = "org.koitharu.kotatsu.CAPTCHA_DISCARD"
+		private const val RESOLVE_TIMEOUT = 20_000L
 	}
 }

app/src/main/kotlin/org/koitharu/kotatsu/core/network/webview/CaptchaContinuationClient.kt

@@ -0,0 +1,30 @@
+package org.koitharu.kotatsu.core.network.webview
+
+import android.graphics.Bitmap
+import android.webkit.WebView
+import org.koitharu.kotatsu.core.network.cookies.MutableCookieJar
+import org.koitharu.kotatsu.parsers.network.CloudFlareHelper
+import kotlin.coroutines.Continuation
+
+class CaptchaContinuationClient(
+	private val cookieJar: MutableCookieJar,
+	private val targetUrl: String,
+	continuation: Continuation<Unit>,
+) : ContinuationResumeWebViewClient(continuation) {
+
+	private val oldClearance = CloudFlareHelper.getClearanceCookie(cookieJar, targetUrl)
+
+	override fun onPageFinished(view: WebView?, url: String?) = Unit
+
+	override fun onPageStarted(view: WebView?, url: String?, favicon: Bitmap?) {
+		super.onPageStarted(view, url, favicon)
+		checkClearance(view)
+	}
+
+	private fun checkClearance(view: WebView?) {
+		val clearance = CloudFlareHelper.getClearanceCookie(cookieJar, targetUrl)
+		if (clearance != null && clearance != oldClearance) {
+			resumeContinuation(view)
+		}
+	}
+}

app/src/main/kotlin/org/koitharu/kotatsu/core/network/webview/ContinuationResumeWebViewClient.kt

@@ -5,11 +5,15 @@ import android.webkit.WebViewClient
 import kotlin.coroutines.Continuation
 import kotlin.coroutines.resume
 
-class ContinuationResumeWebViewClient(
+open class ContinuationResumeWebViewClient(
 	private val continuation: Continuation<Unit>,
 ) : WebViewClient() {
 
 	override fun onPageFinished(view: WebView?, url: String?) {
+		resumeContinuation(view)
+	}
+
+	protected fun resumeContinuation(view: WebView?) {
 		view?.webViewClient = WebViewClient() // reset to default
 		continuation.resume(Unit)
 	}

app/src/main/kotlin/org/koitharu/kotatsu/core/network/webview/WebViewExecutor.kt

@@ -1,31 +1,48 @@
 package org.koitharu.kotatsu.core.network.webview
 
 import android.content.Context
+import android.webkit.WebSettings
 import android.webkit.WebView
-import androidx.annotation.MainThread
 import dagger.hilt.android.qualifiers.ApplicationContext
 import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.coroutineScope
+import kotlinx.coroutines.suspendCancellableCoroutine
 import kotlinx.coroutines.sync.Mutex
 import kotlinx.coroutines.sync.withLock
 import kotlinx.coroutines.withContext
+import kotlinx.coroutines.withTimeout
+import org.koitharu.kotatsu.core.exceptions.CloudFlareException
+import org.koitharu.kotatsu.core.network.CommonHeaders
+import org.koitharu.kotatsu.core.network.cookies.MutableCookieJar
+import org.koitharu.kotatsu.core.network.proxy.ProxyProvider
+import org.koitharu.kotatsu.core.parser.MangaRepository
+import org.koitharu.kotatsu.core.parser.ParserMangaRepository
 import org.koitharu.kotatsu.core.util.ext.configureForParser
 import org.koitharu.kotatsu.core.util.ext.printStackTraceDebug
-import org.koitharu.kotatsu.core.util.ext.sanitizeHeaderValue
-import org.koitharu.kotatsu.parsers.util.nullIfEmpty
+import org.koitharu.kotatsu.parsers.model.MangaSource
+import org.koitharu.kotatsu.parsers.util.runCatchingCancellable
 import java.lang.ref.WeakReference
 import javax.inject.Inject
+import javax.inject.Provider
 import javax.inject.Singleton
 import kotlin.coroutines.resume
 import kotlin.coroutines.suspendCoroutine
 
 @Singleton
 class WebViewExecutor @Inject constructor(
-	@ApplicationContext private val context: Context
+	@ApplicationContext private val context: Context,
+	private val proxyProvider: ProxyProvider,
+	private val cookieJar: MutableCookieJar,
+	private val mangaRepositoryFactoryProvider: Provider<MangaRepository.Factory>,
 ) {
 
 	private var webViewCached: WeakReference<WebView>? = null
 	private val mutex = Mutex()
 
+	val defaultUserAgent: String? by lazy {
+		WebSettings.getDefaultUserAgent(context)
+	}
+
 	suspend fun evaluateJs(baseUrl: String?, script: String): String? = mutex.withLock {
 		withContext(Dispatchers.Main.immediate) {
 			val webView = obtainWebView()
@@ -43,16 +60,59 @@ class WebViewExecutor @Inject constructor(
 		}
 	}
 
-	@MainThread
-	fun getDefaultUserAgent() = runCatching {
-		obtainWebView().settings.userAgentString.sanitizeHeaderValue().trim().nullIfEmpty()
-	}.onFailure { e ->
-		e.printStackTraceDebug()
-	}.getOrNull()
-
-	@MainThread
-	private fun obtainWebView(): WebView = webViewCached?.get() ?: WebView(context).also {
-		it.configureForParser(null)
-		webViewCached = WeakReference(it)
+	suspend fun tryResolveCaptcha(exception: CloudFlareException, timeout: Long): Boolean = mutex.withLock {
+		runCatchingCancellable {
+			withContext(Dispatchers.Main.immediate) {
+				val webView = obtainWebView()
+				try {
+					exception.source.getUserAgent()?.let {
+						webView.settings.userAgentString = it
+					}
+					coroutineScope {
+						withTimeout(timeout) {
+							suspendCancellableCoroutine { cont ->
+								webView.webViewClient = CaptchaContinuationClient(
+									cookieJar = cookieJar,
+									targetUrl = exception.url,
+									continuation = cont,
+								)
+								cont.invokeOnCancellation {
+									webView.stopLoading()
+								}
+								webView.loadUrl(exception.url)
+							}
+						}
+					}
+				} finally {
+					webView.settings.userAgentString = defaultUserAgent
+				}
+			}
+		}.onFailure { e ->
+			exception.addSuppressed(e)
+			e.printStackTraceDebug()
+		}.isSuccess
+	}
+
+	private suspend fun obtainWebView(): WebView {
+		webViewCached?.get()?.let {
+			return it
+		}
+		return withContext(Dispatchers.Main.immediate) {
+			webViewCached?.get()?.let {
+				return@withContext it
+			}
+			WebView(context).also {
+				it.configureForParser(null)
+				webViewCached = WeakReference(it)
+				proxyProvider.applyWebViewConfig()
+				it.onResume()
+				it.resumeTimers()
+			}
+		}
+	}
+
+	private fun MangaSource.getUserAgent(): String? {
+		val repository = mangaRepositoryFactoryProvider.get().create(this) as? ParserMangaRepository
+		return repository?.getRequestHeaders()?.get(CommonHeaders.USER_AGENT)
 	}
 }

app/src/main/kotlin/org/koitharu/kotatsu/core/parser/MangaLoaderContextImpl.kt

@@ -5,8 +5,6 @@ import android.content.Context
 import android.util.Base64
 import androidx.core.os.LocaleListCompat
 import dagger.hilt.android.qualifiers.ApplicationContext
-import kotlinx.coroutines.Dispatchers
-import kotlinx.coroutines.runBlocking
 import kotlinx.coroutines.withTimeout
 import okhttp3.MediaType.Companion.toMediaType
 import okhttp3.OkHttpClient
@@ -33,7 +31,6 @@ import java.util.Locale
 import java.util.concurrent.TimeUnit
 import javax.inject.Inject
 import javax.inject.Singleton
-import kotlin.coroutines.EmptyCoroutineContext
 
 @Singleton
 class MangaLoaderContextImpl @Inject constructor(
@@ -43,7 +40,6 @@ class MangaLoaderContextImpl @Inject constructor(
 	private val webViewExecutor: WebViewExecutor,
 ) : MangaLoaderContext() {
 
-	private val webViewUserAgent by lazy { obtainWebViewUserAgent() }
 	private val jsTimeout = TimeUnit.SECONDS.toMillis(4)
 
 	@Deprecated("Provide a base url")
@@ -54,7 +50,7 @@ class MangaLoaderContextImpl @Inject constructor(
 		webViewExecutor.evaluateJs(baseUrl, script)
 	}
 
-	override fun getDefaultUserAgent(): String = webViewUserAgent
+	override fun getDefaultUserAgent(): String = webViewExecutor.defaultUserAgent ?: UserAgents.FIREFOX_MOBILE
 
 	override fun getConfig(source: MangaSource): MangaSourceConfig {
 		return SourceSettings(androidContext, source)
@@ -91,15 +87,4 @@ class MangaLoaderContextImpl @Inject constructor(
 	}
 
 	override fun createBitmap(width: Int, height: Int): Bitmap = BitmapWrapper.create(width, height)
-
-	private fun obtainWebViewUserAgent(): String {
-		val mainDispatcher = Dispatchers.Main.immediate
-		return if (!mainDispatcher.isDispatchNeeded(EmptyCoroutineContext)) {
-			webViewExecutor.getDefaultUserAgent()
-		} else {
-			runBlocking(mainDispatcher) {
-				webViewExecutor.getDefaultUserAgent()
-			}
-		} ?: UserAgents.FIREFOX_MOBILE
-	}
 }

app/src/main/kotlin/org/koitharu/kotatsu/core/ui/image/FaviconView.kt

@@ -10,7 +10,7 @@ import coil3.asImage
 import coil3.request.Disposable
 import coil3.request.ImageRequest
 import org.koitharu.kotatsu.R
-import org.koitharu.kotatsu.core.exceptions.resolve.CaptchaHandler.Companion.ignoreCaptchaErrors
+import org.koitharu.kotatsu.core.exceptions.resolve.CaptchaHandler.Companion.suppressCaptchaErrors
 import org.koitharu.kotatsu.core.image.CoilImageView
 import org.koitharu.kotatsu.core.parser.favicon.faviconUri
 import org.koitharu.kotatsu.core.util.ext.isAnimationsEnabled
@@ -57,7 +57,7 @@ class FaviconView @JvmOverloads constructor(
 				.fallback(fallbackFactory)
 				.placeholder(placeholderFactory)
 				.mangaSourceExtra(mangaSource)
-				.ignoreCaptchaErrors()
+				.suppressCaptchaErrors()
 				.build(),
 		)
 	}