Fix passing CloudFlare protection #819 #820

app/build.gradle

@@ -16,8 +16,8 @@ android {
 		applicationId 'org.koitharu.kotatsu'
 		minSdk = 21
 		targetSdk = 34
-		versionCode = 628
+		versionCode = 629
-		versionName = '6.8-a1'
+		versionName = '6.8-b1'
 		generatedDensities = []
 		testInstrumentationRunner 'org.koitharu.kotatsu.HiltTestRunner'
 		ksp {
@@ -82,7 +82,7 @@ afterEvaluate {
 }
 dependencies {
 	//noinspection GradleDependency
-	implementation('com.github.KotatsuApp:kotatsu-parsers:fec60955ed') {
+	implementation('com.github.KotatsuApp:kotatsu-parsers:103ef11f3d') {
 		exclude group: 'org.json', module: 'json'
 	}
 
@@ -94,7 +94,7 @@ dependencies {
 	implementation 'androidx.core:core-ktx:1.12.0'
 	implementation 'androidx.activity:activity-ktx:1.8.2'
 	implementation 'androidx.fragment:fragment-ktx:1.6.2'
-	implementation 'androidx.collection:collection:1.4.0'
+	implementation 'androidx.collection:collection-ktx:1.4.0'
 	implementation 'androidx.lifecycle:lifecycle-viewmodel-ktx:2.7.0'
 	implementation 'androidx.lifecycle:lifecycle-service:2.7.0'
 	implementation 'androidx.lifecycle:lifecycle-process:2.7.0'
@@ -106,6 +106,7 @@ dependencies {
 	implementation 'androidx.biometric:biometric-ktx:1.2.0-alpha05'
 	implementation 'com.google.android.material:material:1.12.0-alpha03'
 	implementation 'androidx.lifecycle:lifecycle-common-java8:2.7.0'
+	implementation 'androidx.webkit:webkit:1.10.0'
 
 	implementation 'androidx.work:work-runtime:2.9.0'
 	//noinspection GradleDependency

app/src/main/kotlin/org/koitharu/kotatsu/browser/BrowserActivity.kt

@@ -14,9 +14,9 @@ import androidx.core.view.isVisible
 import androidx.core.view.updatePadding
 import org.koitharu.kotatsu.R
 import org.koitharu.kotatsu.core.ui.BaseActivity
+import org.koitharu.kotatsu.core.util.ext.configureForParser
 import org.koitharu.kotatsu.core.util.ext.toUriOrNull
 import org.koitharu.kotatsu.databinding.ActivityBrowserBinding
-import org.koitharu.kotatsu.parsers.network.UserAgents
 import com.google.android.material.R as materialR
 
 @SuppressLint("SetJavaScriptEnabled")
@@ -33,10 +33,7 @@ class BrowserActivity : BaseActivity<ActivityBrowserBinding>(), BrowserCallback
 			setDisplayHomeAsUpEnabled(true)
 			setHomeAsUpIndicator(materialR.drawable.abc_ic_clear_material)
 		}
-		with(viewBinding.webView.settings) {
+		viewBinding.webView.configureForParser(null)
-			javaScriptEnabled = true
-			userAgentString = UserAgents.CHROME_MOBILE
-		}
 		CookieManager.getInstance().setAcceptThirdPartyCookies(viewBinding.webView, true)
 		viewBinding.webView.webViewClient = BrowserClient(this)
 		viewBinding.webView.webChromeClient = ProgressChromeClient(viewBinding.progressBar)

app/src/main/kotlin/org/koitharu/kotatsu/browser/cloudflare/CloudFlareActivity.kt

@@ -27,8 +27,8 @@ import org.koitharu.kotatsu.core.network.CommonHeaders
 import org.koitharu.kotatsu.core.network.cookies.MutableCookieJar
 import org.koitharu.kotatsu.core.ui.BaseActivity
 import org.koitharu.kotatsu.core.util.TaggedActivityResult
+import org.koitharu.kotatsu.core.util.ext.configureForParser
 import org.koitharu.kotatsu.databinding.ActivityBrowserBinding
-import org.koitharu.kotatsu.parsers.network.UserAgents
 import javax.inject.Inject
 import com.google.android.material.R as materialR
 
@@ -40,6 +40,7 @@ class CloudFlareActivity : BaseActivity<ActivityBrowserBinding>(), CloudFlareCal
 	@Inject
 	lateinit var cookieJar: MutableCookieJar
 
+	private lateinit var cfClient: CloudFlareClient
 	private var onBackPressedCallback: WebViewBackPressedCallback? = null
 
 	override fun onCreate(savedInstanceState: Bundle?) {
@@ -52,13 +53,9 @@ class CloudFlareActivity : BaseActivity<ActivityBrowserBinding>(), CloudFlareCal
 			setHomeAsUpIndicator(materialR.drawable.abc_ic_clear_material)
 		}
 		val url = intent?.dataString.orEmpty()
-		with(viewBinding.webView.settings) {
+		cfClient = CloudFlareClient(cookieJar, this, url)
-			javaScriptEnabled = true
+		viewBinding.webView.configureForParser(intent?.getStringExtra(ARG_UA))
-			domStorageEnabled = true
+		viewBinding.webView.webViewClient = cfClient
-			databaseEnabled = true
-			userAgentString = intent?.getStringExtra(ARG_UA) ?: UserAgents.CHROME_MOBILE
-		}
-		viewBinding.webView.webViewClient = CloudFlareClient(cookieJar, this, url)
 		onBackPressedCallback = WebViewBackPressedCallback(viewBinding.webView).also {
 			onBackPressedDispatcher.addCallback(it)
 		}
@@ -118,15 +115,7 @@ class CloudFlareActivity : BaseActivity<ActivityBrowserBinding>(), CloudFlareCal
 		}
 
 		R.id.action_retry -> {
-			lifecycleScope.launch {
+			restartCheck()
-				viewBinding.webView.stopLoading()
-				yield()
-				val targetUrl = intent?.dataString?.toHttpUrlOrNull()
-				if (targetUrl != null) {
-					clearCfCookies(targetUrl)
-					viewBinding.webView.loadUrl(targetUrl.toString())
-				}
-			}
 			true
 		}
 
@@ -152,6 +141,10 @@ class CloudFlareActivity : BaseActivity<ActivityBrowserBinding>(), CloudFlareCal
 		viewBinding.progressBar.isInvisible = true
 	}
 
+	override fun onLoopDetected() {
+		restartCheck()
+	}
+
 	override fun onCheckPassed() {
 		pendingResult = RESULT_OK
 		finishAfterTransition()
@@ -171,10 +164,23 @@ class CloudFlareActivity : BaseActivity<ActivityBrowserBinding>(), CloudFlareCal
 			subtitle?.toString()?.toHttpUrlOrNull()?.topPrivateDomain() ?: subtitle
 	}
 
+	private fun restartCheck() {
+		lifecycleScope.launch {
+			viewBinding.webView.stopLoading()
+			yield()
+			cfClient.reset()
+			val targetUrl = intent?.dataString?.toHttpUrlOrNull()
+			if (targetUrl != null) {
+				clearCfCookies(targetUrl)
+				viewBinding.webView.loadUrl(targetUrl.toString())
+			}
+		}
+	}
+
 	private suspend fun clearCfCookies(url: HttpUrl) = runInterruptible(Dispatchers.Default) {
 		cookieJar.removeCookies(url) { cookie ->
 			val name = cookie.name
-			name.startsWith("cf_") || name.startsWith("_cf") || name.startsWith("__cf")
+			name.startsWith("cf_") || name.startsWith("_cf") || name.startsWith("__cf") || name == "csrftoken"
 		}
 	}
 

app/src/main/kotlin/org/koitharu/kotatsu/browser/cloudflare/CloudFlareCallback.kt

@@ -11,4 +11,6 @@ interface CloudFlareCallback : BrowserCallback {
 	fun onPageLoaded()
 
 	fun onCheckPassed()
+
+	fun onLoopDetected()
 }

app/src/main/kotlin/org/koitharu/kotatsu/browser/cloudflare/CloudFlareClient.kt

@@ -7,6 +7,7 @@ import org.koitharu.kotatsu.browser.BrowserClient
 import org.koitharu.kotatsu.core.network.cookies.MutableCookieJar
 
 private const val CF_CLEARANCE = "cf_clearance"
+private const val LOOP_COUNTER = 3
 
 class CloudFlareClient(
 	private val cookieJar: MutableCookieJar,
@@ -15,6 +16,7 @@ class CloudFlareClient(
 ) : BrowserClient(callback) {
 
 	private val oldClearance = getClearance()
+	private var counter = 0
 
 	override fun onPageStarted(view: WebView?, url: String?, favicon: Bitmap?) {
 		super.onPageStarted(view, url, favicon)
@@ -31,10 +33,20 @@ class CloudFlareClient(
 		callback.onPageLoaded()
 	}
 
+	fun reset() {
+		counter = 0
+	}
+
 	private fun checkClearance() {
 		val clearance = getClearance()
 		if (clearance != null && clearance != oldClearance) {
 			callback.onCheckPassed()
+		} else {
+			counter++
+			if (counter >= LOOP_COUNTER) {
+				reset()
+				callback.onLoopDetected()
+			}
 		}
 	}
 

app/src/main/kotlin/org/koitharu/kotatsu/core/network/CommonHeadersInterceptor.kt

@@ -7,11 +7,11 @@ import okhttp3.Interceptor
 import okhttp3.Request
 import okhttp3.Response
 import org.koitharu.kotatsu.BuildConfig
+import org.koitharu.kotatsu.core.parser.MangaLoaderContextImpl
 import org.koitharu.kotatsu.core.parser.MangaRepository
 import org.koitharu.kotatsu.core.parser.RemoteMangaRepository
 import org.koitharu.kotatsu.core.util.ext.printStackTraceDebug
 import org.koitharu.kotatsu.parsers.model.MangaSource
-import org.koitharu.kotatsu.parsers.network.UserAgents
 import org.koitharu.kotatsu.parsers.util.mergeWith
 import java.net.IDN
 import javax.inject.Inject
@@ -20,6 +20,7 @@ import javax.inject.Singleton
 @Singleton
 class CommonHeadersInterceptor @Inject constructor(
 	private val mangaRepositoryFactoryLazy: Lazy<MangaRepository.Factory>,
+	private val mangaLoaderContextLazy: Lazy<MangaLoaderContextImpl>,
 ) : Interceptor {
 
 	override fun intercept(chain: Interceptor.Chain): Response {
@@ -38,7 +39,7 @@ class CommonHeadersInterceptor @Inject constructor(
 			headersBuilder.mergeWith(it, replaceExisting = false)
 		}
 		if (headersBuilder[CommonHeaders.USER_AGENT] == null) {
-			headersBuilder[CommonHeaders.USER_AGENT] = UserAgents.CHROME_MOBILE
+			headersBuilder[CommonHeaders.USER_AGENT] = mangaLoaderContextLazy.get().getDefaultUserAgent()
 		}
 		if (headersBuilder[CommonHeaders.REFERER] == null && repository != null) {
 			val idn = IDN.toASCII(repository.domain)

app/src/main/kotlin/org/koitharu/kotatsu/core/parser/MangaLoaderContextImpl.kt

@@ -4,18 +4,24 @@ import android.annotation.SuppressLint
 import android.content.Context
 import android.util.Base64
 import android.webkit.WebView
+import androidx.annotation.MainThread
 import androidx.core.os.LocaleListCompat
 import dagger.hilt.android.qualifiers.ApplicationContext
 import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.runBlocking
 import kotlinx.coroutines.withContext
 import okhttp3.OkHttpClient
 import org.koitharu.kotatsu.core.network.MangaHttpClient
 import org.koitharu.kotatsu.core.network.cookies.MutableCookieJar
 import org.koitharu.kotatsu.core.prefs.SourceSettings
+import org.koitharu.kotatsu.core.util.ext.configureForParser
+import org.koitharu.kotatsu.core.util.ext.printStackTraceDebug
 import org.koitharu.kotatsu.core.util.ext.toList
 import org.koitharu.kotatsu.parsers.MangaLoaderContext
 import org.koitharu.kotatsu.parsers.config.MangaSourceConfig
 import org.koitharu.kotatsu.parsers.model.MangaSource
+import org.koitharu.kotatsu.parsers.network.UserAgents
+import org.koitharu.kotatsu.parsers.util.SuspendLazy
 import java.lang.ref.WeakReference
 import java.util.Locale
 import javax.inject.Inject
@@ -32,12 +38,15 @@ class MangaLoaderContextImpl @Inject constructor(
 
 	private var webViewCached: WeakReference<WebView>? = null
 
+	private val userAgentLazy = SuspendLazy {
+		withContext(Dispatchers.Main) {
+			obtainWebView().settings.userAgentString
+		}
+	}
+
 	@SuppressLint("SetJavaScriptEnabled")
 	override suspend fun evaluateJs(script: String): String? = withContext(Dispatchers.Main) {
-		val webView = webViewCached?.get() ?: WebView(androidContext).also {
+		val webView = obtainWebView()
-			it.settings.javaScriptEnabled = true
-			webViewCached = WeakReference(it)
-		}
 		suspendCoroutine { cont ->
 			webView.evaluateJavascript(script) { result ->
 				cont.resume(result?.takeUnless { it == "null" })
@@ -45,6 +54,14 @@ class MangaLoaderContextImpl @Inject constructor(
 		}
 	}
 
+	override fun getDefaultUserAgent(): String = runCatching {
+		runBlocking {
+			userAgentLazy.get()
+		}
+	}.onFailure { e ->
+		e.printStackTraceDebug()
+	}.getOrDefault(UserAgents.FIREFOX_MOBILE)
+
 	override fun getConfig(source: MangaSource): MangaSourceConfig {
 		return SourceSettings(androidContext, source)
 	}
@@ -60,4 +77,12 @@ class MangaLoaderContextImpl @Inject constructor(
 	override fun getPreferredLocales(): List<Locale> {
 		return LocaleListCompat.getAdjustedDefault().toList()
 	}
+
+	@MainThread
+	private fun obtainWebView(): WebView {
+		return webViewCached?.get() ?: WebView(androidContext).also {
+			it.configureForParser(null)
+			webViewCached = WeakReference(it)
+		}
+	}
 }

app/src/main/kotlin/org/koitharu/kotatsu/core/util/ext/Android.kt

@@ -27,6 +27,7 @@ import android.provider.Settings
 import android.view.View
 import android.view.ViewPropertyAnimator
 import android.view.Window
+import android.webkit.WebView
 import androidx.activity.result.ActivityResultLauncher
 import androidx.annotation.IntegerRes
 import androidx.annotation.WorkerThread
@@ -235,3 +236,13 @@ fun Context.ensureRamAtLeast(requiredSize: Long) {
 		throw IllegalStateException("Not enough free memory")
 	}
 }
+
+fun WebView.configureForParser(userAgentOverride: String?) = with(settings) {
+	javaScriptEnabled = true
+	domStorageEnabled = true
+	mediaPlaybackRequiresUserGesture = false
+	databaseEnabled = true
+	if (userAgentOverride != null) {
+		userAgentString = userAgentOverride
+	}
+}

app/src/main/kotlin/org/koitharu/kotatsu/settings/sources/SourceSettingsExt.kt

@@ -8,6 +8,7 @@ import androidx.preference.SwitchPreferenceCompat
 import org.koitharu.kotatsu.R
 import org.koitharu.kotatsu.core.parser.RemoteMangaRepository
 import org.koitharu.kotatsu.parsers.config.ConfigKey
+import org.koitharu.kotatsu.parsers.network.UserAgents
 import org.koitharu.kotatsu.settings.utils.AutoCompleteTextViewPreference
 import org.koitharu.kotatsu.settings.utils.EditTextBindListener
 import org.koitharu.kotatsu.settings.utils.EditTextDefaultSummaryProvider
@@ -42,7 +43,13 @@ fun PreferenceFragmentCompat.addPreferencesFromRepository(repository: RemoteMang
 			}
 
 			is ConfigKey.UserAgent -> {
-				EditTextPreference(requireContext()).apply {
+				AutoCompleteTextViewPreference(requireContext()).apply {
+					entries = arrayOf(
+						UserAgents.FIREFOX_MOBILE,
+						UserAgents.CHROME_MOBILE,
+						UserAgents.FIREFOX_DESKTOP,
+						UserAgents.CHROME_DESKTOP,
+					)
 					summaryProvider = EditTextDefaultSummaryProvider(key.defaultValue)
 					setOnBindEditTextListener(
 						EditTextBindListener(
@@ -73,6 +80,7 @@ fun PreferenceFragmentCompat.addPreferencesFromRepository(repository: RemoteMang
 		}
 		preference.isIconSpaceReserved = false
 		preference.key = key.key
+		preference.order = 10
 		screen.addPreference(preference)
 	}
 }

app/src/main/kotlin/org/koitharu/kotatsu/settings/sources/auth/SourceAuthActivity.kt

@@ -18,15 +18,16 @@ import org.koitharu.kotatsu.browser.BrowserCallback
 import org.koitharu.kotatsu.browser.BrowserClient
 import org.koitharu.kotatsu.browser.ProgressChromeClient
 import org.koitharu.kotatsu.browser.WebViewBackPressedCallback
+import org.koitharu.kotatsu.core.network.CommonHeaders
 import org.koitharu.kotatsu.core.parser.MangaRepository
 import org.koitharu.kotatsu.core.parser.RemoteMangaRepository
 import org.koitharu.kotatsu.core.ui.BaseActivity
 import org.koitharu.kotatsu.core.util.TaggedActivityResult
+import org.koitharu.kotatsu.core.util.ext.configureForParser
 import org.koitharu.kotatsu.core.util.ext.getSerializableExtraCompat
 import org.koitharu.kotatsu.databinding.ActivityBrowserBinding
 import org.koitharu.kotatsu.parsers.MangaParserAuthProvider
 import org.koitharu.kotatsu.parsers.model.MangaSource
-import org.koitharu.kotatsu.parsers.network.UserAgents
 import javax.inject.Inject
 import com.google.android.material.R as materialR
 
@@ -64,12 +65,7 @@ class SourceAuthActivity : BaseActivity<ActivityBrowserBinding>(), BrowserCallba
 			setDisplayHomeAsUpEnabled(true)
 			setHomeAsUpIndicator(materialR.drawable.abc_ic_clear_material)
 		}
-		with(viewBinding.webView.settings) {
+		viewBinding.webView.configureForParser(repository.headers[CommonHeaders.USER_AGENT])
-			javaScriptEnabled = true
-			domStorageEnabled = true
-			databaseEnabled = true
-			userAgentString = UserAgents.CHROME_MOBILE
-		}
 		CookieManager.getInstance().setAcceptThirdPartyCookies(viewBinding.webView, true)
 		viewBinding.webView.webViewClient = BrowserClient(this)
 		viewBinding.webView.webChromeClient = ProgressChromeClient(viewBinding.progressBar)