KotatsuApp
/
Kotatsu
2
0
Fork 0
Code Issues 124 Pull Requests Packages Projects Releases 163 Wiki Activity

Fix passing CloudFlare protection #819 #820

Browse Source
master
Koitharu 2 years ago
parent f79d2cb733
commit 7e182cb0ad
Signed by: Koitharu
GPG Key ID: 676DEE768C17A9D7
10 changed files with 100 additions and 41 deletions
Show Stats Download Patch File Download Diff File

9
app/build.gradle
Unescape Escape View File

@ -16,8 +16,8 @@ android {
applicationId 'org.koitharu.kotatsu'
minSdk = 21
targetSdk = 34
versionCode = 628
versionName = '6.8-a1'
versionCode = 629
versionName = '6.8-b1'
generatedDensities = []
testInstrumentationRunner 'org.koitharu.kotatsu.HiltTestRunner'
ksp {
@ -82,7 +82,7 @@ afterEvaluate {
}
dependencies {
//noinspection GradleDependency
implementation('com.github.KotatsuApp:kotatsu-parsers:fec60955ed') {
implementation('com.github.KotatsuApp:kotatsu-parsers:103ef11f3d') {
exclude group: 'org.json', module: 'json'
}
@ -94,7 +94,7 @@ dependencies {
implementation 'androidx.core:core-ktx:1.12.0'
implementation 'androidx.activity:activity-ktx:1.8.2'
implementation 'androidx.fragment:fragment-ktx:1.6.2'
implementation 'androidx.collection:collection:1.4.0'
implementation 'androidx.collection:collection-ktx:1.4.0'
implementation 'androidx.lifecycle:lifecycle-viewmodel-ktx:2.7.0'
implementation 'androidx.lifecycle:lifecycle-service:2.7.0'
implementation 'androidx.lifecycle:lifecycle-process:2.7.0'
@ -106,6 +106,7 @@ dependencies {
implementation 'androidx.biometric:biometric-ktx:1.2.0-alpha05'
implementation 'com.google.android.material:material:1.12.0-alpha03'
implementation 'androidx.lifecycle:lifecycle-common-java8:2.7.0'
implementation 'androidx.webkit:webkit:1.10.0'
implementation 'androidx.work:work-runtime:2.9.0'
//noinspection GradleDependency

7
app/src/main/kotlin/org/koitharu/kotatsu/browser/BrowserActivity.kt
Unescape Escape View File

@ -14,9 +14,9 @@ import androidx.core.view.isVisible
import androidx.core.view.updatePadding
import org.koitharu.kotatsu.R
import org.koitharu.kotatsu.core.ui.BaseActivity
import org.koitharu.kotatsu.core.util.ext.configureForParser
import org.koitharu.kotatsu.core.util.ext.toUriOrNull
import org.koitharu.kotatsu.databinding.ActivityBrowserBinding
import org.koitharu.kotatsu.parsers.network.UserAgents
import com.google.android.material.R as materialR
@SuppressLint("SetJavaScriptEnabled")
@ -33,10 +33,7 @@ class BrowserActivity : BaseActivity<ActivityBrowserBinding>(), BrowserCallback
setDisplayHomeAsUpEnabled(true)
setHomeAsUpIndicator(materialR.drawable.abc_ic_clear_material)
}
with(viewBinding.webView.settings) {
javaScriptEnabled = true
userAgentString = UserAgents.CHROME_MOBILE
}
viewBinding.webView.configureForParser(null)
CookieManager.getInstance().setAcceptThirdPartyCookies(viewBinding.webView, true)
viewBinding.webView.webViewClient = BrowserClient(this)
viewBinding.webView.webChromeClient = ProgressChromeClient(viewBinding.progressBar)

42
app/src/main/kotlin/org/koitharu/kotatsu/browser/cloudflare/CloudFlareActivity.kt
Unescape Escape View File

@ -27,8 +27,8 @@ import org.koitharu.kotatsu.core.network.CommonHeaders
import org.koitharu.kotatsu.core.network.cookies.MutableCookieJar
import org.koitharu.kotatsu.core.ui.BaseActivity
import org.koitharu.kotatsu.core.util.TaggedActivityResult
import org.koitharu.kotatsu.core.util.ext.configureForParser
import org.koitharu.kotatsu.databinding.ActivityBrowserBinding
import org.koitharu.kotatsu.parsers.network.UserAgents
import javax.inject.Inject
import com.google.android.material.R as materialR
@ -40,6 +40,7 @@ class CloudFlareActivity : BaseActivity<ActivityBrowserBinding>(), CloudFlareCal
@Inject
lateinit var cookieJar: MutableCookieJar
private lateinit var cfClient: CloudFlareClient
private var onBackPressedCallback: WebViewBackPressedCallback? = null
override fun onCreate(savedInstanceState: Bundle?) {
@ -52,13 +53,9 @@ class CloudFlareActivity : BaseActivity<ActivityBrowserBinding>(), CloudFlareCal
setHomeAsUpIndicator(materialR.drawable.abc_ic_clear_material)
}
val url = intent?.dataString.orEmpty()
with(viewBinding.webView.settings) {
javaScriptEnabled = true
domStorageEnabled = true
databaseEnabled = true
userAgentString = intent?.getStringExtra(ARG_UA) ?: UserAgents.CHROME_MOBILE
}
viewBinding.webView.webViewClient = CloudFlareClient(cookieJar, this, url)
cfClient = CloudFlareClient(cookieJar, this, url)
viewBinding.webView.configureForParser(intent?.getStringExtra(ARG_UA))
viewBinding.webView.webViewClient = cfClient
onBackPressedCallback = WebViewBackPressedCallback(viewBinding.webView).also {
onBackPressedDispatcher.addCallback(it)
}
@ -118,15 +115,7 @@ class CloudFlareActivity : BaseActivity<ActivityBrowserBinding>(), CloudFlareCal
}
R.id.action_retry -> {
lifecycleScope.launch {
viewBinding.webView.stopLoading()
yield()
val targetUrl = intent?.dataString?.toHttpUrlOrNull()
if (targetUrl != null) {
clearCfCookies(targetUrl)
viewBinding.webView.loadUrl(targetUrl.toString())
}
}
restartCheck()
true
}
@ -152,6 +141,10 @@ class CloudFlareActivity : BaseActivity<ActivityBrowserBinding>(), CloudFlareCal
viewBinding.progressBar.isInvisible = true
}
override fun onLoopDetected() {
restartCheck()
}
override fun onCheckPassed() {
pendingResult = RESULT_OK
finishAfterTransition()
@ -171,10 +164,23 @@ class CloudFlareActivity : BaseActivity<ActivityBrowserBinding>(), CloudFlareCal
subtitle?.toString()?.toHttpUrlOrNull()?.topPrivateDomain() ?: subtitle
}
private fun restartCheck() {
lifecycleScope.launch {
viewBinding.webView.stopLoading()
yield()
cfClient.reset()
val targetUrl = intent?.dataString?.toHttpUrlOrNull()
if (targetUrl != null) {
clearCfCookies(targetUrl)
viewBinding.webView.loadUrl(targetUrl.toString())
}
}
}
private suspend fun clearCfCookies(url: HttpUrl) = runInterruptible(Dispatchers.Default) {
cookieJar.removeCookies(url) { cookie ->
val name = cookie.name
name.startsWith("cf_") || name.startsWith("_cf") || name.startsWith("__cf")
name.startsWith("cf_") || name.startsWith("_cf") || name.startsWith("__cf") || name == "csrftoken"
}
}

2
app/src/main/kotlin/org/koitharu/kotatsu/browser/cloudflare/CloudFlareCallback.kt
Unescape Escape View File

@ -11,4 +11,6 @@ interface CloudFlareCallback : BrowserCallback {
fun onPageLoaded()
fun onCheckPassed()
fun onLoopDetected()
}

12
app/src/main/kotlin/org/koitharu/kotatsu/browser/cloudflare/CloudFlareClient.kt
Unescape Escape View File

@ -7,6 +7,7 @@ import org.koitharu.kotatsu.browser.BrowserClient
import org.koitharu.kotatsu.core.network.cookies.MutableCookieJar
private const val CF_CLEARANCE = "cf_clearance"
private const val LOOP_COUNTER = 3
class CloudFlareClient(
private val cookieJar: MutableCookieJar,
@ -15,6 +16,7 @@ class CloudFlareClient(
) : BrowserClient(callback) {
private val oldClearance = getClearance()
private var counter = 0
override fun onPageStarted(view: WebView?, url: String?, favicon: Bitmap?) {
super.onPageStarted(view, url, favicon)
@ -31,10 +33,20 @@ class CloudFlareClient(
callback.onPageLoaded()
}
fun reset() {
counter = 0
}
private fun checkClearance() {
val clearance = getClearance()
if (clearance != null && clearance != oldClearance) {
callback.onCheckPassed()
} else {
counter++
if (counter >= LOOP_COUNTER) {
reset()
callback.onLoopDetected()
}
}
}

5
app/src/main/kotlin/org/koitharu/kotatsu/core/network/CommonHeadersInterceptor.kt
Unescape Escape View File

@ -7,11 +7,11 @@ import okhttp3.Interceptor
import okhttp3.Request
import okhttp3.Response
import org.koitharu.kotatsu.BuildConfig
import org.koitharu.kotatsu.core.parser.MangaLoaderContextImpl
import org.koitharu.kotatsu.core.parser.MangaRepository
import org.koitharu.kotatsu.core.parser.RemoteMangaRepository
import org.koitharu.kotatsu.core.util.ext.printStackTraceDebug
import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.network.UserAgents
import org.koitharu.kotatsu.parsers.util.mergeWith
import java.net.IDN
import javax.inject.Inject
@ -20,6 +20,7 @@ import javax.inject.Singleton
@Singleton
class CommonHeadersInterceptor @Inject constructor(
private val mangaRepositoryFactoryLazy: Lazy<MangaRepository.Factory>,
private val mangaLoaderContextLazy: Lazy<MangaLoaderContextImpl>,
) : Interceptor {
override fun intercept(chain: Interceptor.Chain): Response {
@ -38,7 +39,7 @@ class CommonHeadersInterceptor @Inject constructor(
headersBuilder.mergeWith(it, replaceExisting = false)
}
if (headersBuilder[CommonHeaders.USER_AGENT] == null) {
headersBuilder[CommonHeaders.USER_AGENT] = UserAgents.CHROME_MOBILE
headersBuilder[CommonHeaders.USER_AGENT] = mangaLoaderContextLazy.get().getDefaultUserAgent()
}
if (headersBuilder[CommonHeaders.REFERER] == null && repository != null) {
val idn = IDN.toASCII(repository.domain)

33
app/src/main/kotlin/org/koitharu/kotatsu/core/parser/MangaLoaderContextImpl.kt
Unescape Escape View File

@ -4,18 +4,24 @@ import android.annotation.SuppressLint
import android.content.Context
import android.util.Base64
import android.webkit.WebView
import androidx.annotation.MainThread
import androidx.core.os.LocaleListCompat
import dagger.hilt.android.qualifiers.ApplicationContext
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.runBlocking
import kotlinx.coroutines.withContext
import okhttp3.OkHttpClient
import org.koitharu.kotatsu.core.network.MangaHttpClient
import org.koitharu.kotatsu.core.network.cookies.MutableCookieJar
import org.koitharu.kotatsu.core.prefs.SourceSettings
import org.koitharu.kotatsu.core.util.ext.configureForParser
import org.koitharu.kotatsu.core.util.ext.printStackTraceDebug
import org.koitharu.kotatsu.core.util.ext.toList
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.config.MangaSourceConfig
import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.network.UserAgents
import org.koitharu.kotatsu.parsers.util.SuspendLazy
import java.lang.ref.WeakReference
import java.util.Locale
import javax.inject.Inject
@ -32,12 +38,15 @@ class MangaLoaderContextImpl @Inject constructor(
private var webViewCached: WeakReference<WebView>? = null
private val userAgentLazy = SuspendLazy {
withContext(Dispatchers.Main) {
obtainWebView().settings.userAgentString
}
}
@SuppressLint("SetJavaScriptEnabled")
override suspend fun evaluateJs(script: String): String? = withContext(Dispatchers.Main) {
val webView = webViewCached?.get() ?: WebView(androidContext).also {
it.settings.javaScriptEnabled = true
webViewCached = WeakReference(it)
}
val webView = obtainWebView()
suspendCoroutine { cont ->
webView.evaluateJavascript(script) { result ->
cont.resume(result?.takeUnless { it == "null" })
@ -45,6 +54,14 @@ class MangaLoaderContextImpl @Inject constructor(
}
}
override fun getDefaultUserAgent(): String = runCatching {
runBlocking {
userAgentLazy.get()
}
}.onFailure { e ->
e.printStackTraceDebug()
}.getOrDefault(UserAgents.FIREFOX_MOBILE)
override fun getConfig(source: MangaSource): MangaSourceConfig {
return SourceSettings(androidContext, source)
}
@ -60,4 +77,12 @@ class MangaLoaderContextImpl @Inject constructor(
override fun getPreferredLocales(): List<Locale> {
return LocaleListCompat.getAdjustedDefault().toList()
}
@MainThread
private fun obtainWebView(): WebView {
return webViewCached?.get() ?: WebView(androidContext).also {
it.configureForParser(null)
webViewCached = WeakReference(it)
}
}
}

11
app/src/main/kotlin/org/koitharu/kotatsu/core/util/ext/Android.kt
Unescape Escape View File

@ -27,6 +27,7 @@ import android.provider.Settings
import android.view.View
import android.view.ViewPropertyAnimator
import android.view.Window
import android.webkit.WebView
import androidx.activity.result.ActivityResultLauncher
import androidx.annotation.IntegerRes
import androidx.annotation.WorkerThread
@ -235,3 +236,13 @@ fun Context.ensureRamAtLeast(requiredSize: Long) {
throw IllegalStateException("Not enough free memory")
}
}
fun WebView.configureForParser(userAgentOverride: String?) = with(settings) {
javaScriptEnabled = true
domStorageEnabled = true
mediaPlaybackRequiresUserGesture = false
databaseEnabled = true
if (userAgentOverride != null) {
userAgentString = userAgentOverride
}
}

10
app/src/main/kotlin/org/koitharu/kotatsu/settings/sources/SourceSettingsExt.kt
Unescape Escape View File

@ -8,6 +8,7 @@ import androidx.preference.SwitchPreferenceCompat
import org.koitharu.kotatsu.R
import org.koitharu.kotatsu.core.parser.RemoteMangaRepository
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.network.UserAgents
import org.koitharu.kotatsu.settings.utils.AutoCompleteTextViewPreference
import org.koitharu.kotatsu.settings.utils.EditTextBindListener
import org.koitharu.kotatsu.settings.utils.EditTextDefaultSummaryProvider
@ -42,7 +43,13 @@ fun PreferenceFragmentCompat.addPreferencesFromRepository(repository: RemoteMang
}
is ConfigKey.UserAgent -> {
EditTextPreference(requireContext()).apply {
AutoCompleteTextViewPreference(requireContext()).apply {
entries = arrayOf(
UserAgents.FIREFOX_MOBILE,
UserAgents.CHROME_MOBILE,
UserAgents.FIREFOX_DESKTOP,
UserAgents.CHROME_DESKTOP,
)
summaryProvider = EditTextDefaultSummaryProvider(key.defaultValue)
setOnBindEditTextListener(
EditTextBindListener(
@ -73,6 +80,7 @@ fun PreferenceFragmentCompat.addPreferencesFromRepository(repository: RemoteMang
}
preference.isIconSpaceReserved = false
preference.key = key.key
preference.order = 10
screen.addPreference(preference)
}
}

10
app/src/main/kotlin/org/koitharu/kotatsu/settings/sources/auth/SourceAuthActivity.kt
Unescape Escape View File

@ -18,15 +18,16 @@ import org.koitharu.kotatsu.browser.BrowserCallback
import org.koitharu.kotatsu.browser.BrowserClient
import org.koitharu.kotatsu.browser.ProgressChromeClient
import org.koitharu.kotatsu.browser.WebViewBackPressedCallback
import org.koitharu.kotatsu.core.network.CommonHeaders
import org.koitharu.kotatsu.core.parser.MangaRepository
import org.koitharu.kotatsu.core.parser.RemoteMangaRepository
import org.koitharu.kotatsu.core.ui.BaseActivity
import org.koitharu.kotatsu.core.util.TaggedActivityResult
import org.koitharu.kotatsu.core.util.ext.configureForParser
import org.koitharu.kotatsu.core.util.ext.getSerializableExtraCompat
import org.koitharu.kotatsu.databinding.ActivityBrowserBinding
import org.koitharu.kotatsu.parsers.MangaParserAuthProvider
import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.network.UserAgents
import javax.inject.Inject
import com.google.android.material.R as materialR
@ -64,12 +65,7 @@ class SourceAuthActivity : BaseActivity<ActivityBrowserBinding>(), BrowserCallba
setDisplayHomeAsUpEnabled(true)
setHomeAsUpIndicator(materialR.drawable.abc_ic_clear_material)
}
with(viewBinding.webView.settings) {
javaScriptEnabled = true
domStorageEnabled = true
databaseEnabled = true
userAgentString = UserAgents.CHROME_MOBILE
}
viewBinding.webView.configureForParser(repository.headers[CommonHeaders.USER_AGENT])
CookieManager.getInstance().setAcceptThirdPartyCookies(viewBinding.webView, true)
viewBinding.webView.webViewClient = BrowserClient(this)
viewBinding.webView.webChromeClient = ProgressChromeClient(viewBinding.progressBar)

Write Preview
Loading…
Cancel
Save