From 03b4fc9f00534152e8a1946a670d9ec3091fc60c Mon Sep 17 00:00:00 2001 From: Koitharu Date: Mon, 31 Jul 2023 09:51:18 +0300 Subject: [PATCH 1/3] Add Result utils --- .../org/koitharu/kotatsu/parsers/util/Result.kt | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Result.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Result.kt index 31357404..e5be1293 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Result.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Result.kt @@ -13,3 +13,17 @@ inline fun T.runCatchingCancellable(block: T.() -> R): Result { Result.failure(e) } } + +inline fun Result.recoverCatchingCancellable(transform: (exception: Throwable) -> R): Result { + return when (val exception = exceptionOrNull()) { + null -> this + else -> runCatchingCancellable { transform(exception) } + } +} + +inline fun Result.recoverNotNull(transform: (exception: Throwable) -> R?): Result { + return when (val exception = exceptionOrNull()) { + null -> this + else -> transform(exception)?.let(Result.Companion::success) ?: this + } +} From 3c4359096da96dc6a7fcf786f531da79589c5796 Mon Sep 17 00:00:00 2001 From: Koitharu Date: Mon, 31 Jul 2023 13:01:59 +0300 Subject: [PATCH 2/3] Add Jsoup extensions --- .../parsers/site/fr/BentomangaParser.kt | 12 +------- .../parsers/site/madara/MadaraParser.kt | 7 ----- .../parsers/site/manga18/Manga18Parser.kt | 9 ------ .../parsers/site/manga18/en/Hentai3zCc.kt | 18 ++--------- .../parsers/site/mangareader/fr/LelManga.kt | 7 ----- .../parsers/site/mmrcms/MmrcmsParser.kt | 10 ------- .../koitharu/kotatsu/parsers/util/Jsoup.kt | 30 +++++++++++++++++-- 7 files changed, 31 insertions(+), 62 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/fr/BentomangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/fr/BentomangaParser.kt index 2eece60e..f8892de1 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/fr/BentomangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/fr/BentomangaParser.kt @@ -75,7 +75,7 @@ internal class BentomangaParser(context: MangaLoaderContext) : PagedMangaParser( ?.div(10f) ?: RATING_UNKNOWN, isNsfw = div.selectFirst(".badge-adult_content") != null, - coverUrl = div.selectFirstOrThrow("img").src(), + coverUrl = div.selectFirstOrThrow("img").src().assertNotNull("src").orEmpty(), tags = div.selectFirst(".component-manga-categories") .assertNotNull("tags") ?.select("a") @@ -223,14 +223,4 @@ internal class BentomangaParser(context: MangaLoaderContext) : PagedMangaParser( calendar.add(calendarUnit, -count) return calendar.timeInMillis } - - private fun Element.src(): String { - return attrAsAbsoluteUrlOrNull("data-cfsrc") - ?: attrAsAbsoluteUrlOrNull("src") - ?: attrAsAbsoluteUrlOrNull("data-src") - ?: run { - assert(false) { "Image src not found" } - "" - } - } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt index 3e82266a..d7bd6971 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt @@ -540,13 +540,6 @@ internal abstract class MadaraParser( } } - protected fun Element.src(): String? { - var result = absUrl("data-src") - if (result.isEmpty()) result = absUrl("data-cfsrc") - if (result.isEmpty()) result = absUrl("src") - return result.ifEmpty { null } - } - private companion object { private fun createRequestTemplate() = diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/manga18/Manga18Parser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/manga18/Manga18Parser.kt index cc17baa0..29b37941 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/manga18/Manga18Parser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/manga18/Manga18Parser.kt @@ -204,13 +204,4 @@ internal abstract class Manga18Parser( ) } } - - - protected fun Element.src(): String? { - var result = absUrl("data-src") - if (result.isEmpty()) result = absUrl("data-cfsrc") - if (result.isEmpty()) result = absUrl("src") - return result.ifEmpty { null } - } - } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/manga18/en/Hentai3zCc.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/manga18/en/Hentai3zCc.kt index fe77fddc..1186eb14 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/manga18/en/Hentai3zCc.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/manga18/en/Hentai3zCc.kt @@ -1,23 +1,11 @@ -package org.koitharu.kotatsu.parsers.site.madara.en +package org.koitharu.kotatsu.parsers.site.manga18.en import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.ContentType -import org.koitharu.kotatsu.parsers.model.Manga -import org.koitharu.kotatsu.parsers.model.MangaSource -import org.koitharu.kotatsu.parsers.model.MangaTag -import org.koitharu.kotatsu.parsers.model.RATING_UNKNOWN -import org.koitharu.kotatsu.parsers.model.SortOrder +import org.koitharu.kotatsu.parsers.model.* import org.koitharu.kotatsu.parsers.site.manga18.Manga18Parser -import org.koitharu.kotatsu.parsers.util.attrAsRelativeUrl -import org.koitharu.kotatsu.parsers.util.domain -import org.koitharu.kotatsu.parsers.util.generateUid -import org.koitharu.kotatsu.parsers.util.host -import org.koitharu.kotatsu.parsers.util.parseHtml -import org.koitharu.kotatsu.parsers.util.selectFirstOrThrow -import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl -import org.koitharu.kotatsu.parsers.util.urlEncoded +import org.koitharu.kotatsu.parsers.util.* @MangaSourceParser("HENTAI3ZCC", "Hentai3z Cc", "en", ContentType.HENTAI) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/fr/LelManga.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/fr/LelManga.kt index 10b5e6f8..97a37650 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/fr/LelManga.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/fr/LelManga.kt @@ -27,11 +27,4 @@ internal class LelManga(context: MangaLoaderContext) : ) } } - - private fun Element.src(): String? { - var result = absUrl("data-src") - if (result.isEmpty()) result = absUrl("data-cfsrc") - if (result.isEmpty()) result = absUrl("src") - return result.ifEmpty { null } - } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mmrcms/MmrcmsParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mmrcms/MmrcmsParser.kt index 37948641..d5334049 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mmrcms/MmrcmsParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mmrcms/MmrcmsParser.kt @@ -3,7 +3,6 @@ package org.koitharu.kotatsu.parsers.site.mmrcms import kotlinx.coroutines.async import kotlinx.coroutines.coroutineScope import org.jsoup.nodes.Document -import org.jsoup.nodes.Element import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.PagedMangaParser import org.koitharu.kotatsu.parsers.config.ConfigKey @@ -248,13 +247,4 @@ internal abstract class MmrcmsParser( ) } } - - - protected fun Element.src(): String? { - var result = absUrl("data-src") - if (result.isEmpty()) result = absUrl("data-cfsrc") - if (result.isEmpty()) result = absUrl("src") - return result.ifEmpty { null } - } - } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Jsoup.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Jsoup.kt index a155fb62..acd3f771 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Jsoup.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Jsoup.kt @@ -23,7 +23,7 @@ val Element.host: String? * Return an attribute value or null if it is missing or empty * @see [Element.attr] which returns empty string instead of null */ -fun Element.attrOrNull(attributeKey: String) = attr(attributeKey).takeUnless { it.isEmpty() } +fun Element.attrOrNull(attributeKey: String) = attr(attributeKey).takeUnless { it.isBlank() }?.trim() /** @@ -43,7 +43,7 @@ fun Element.attrOrThrow(attributeKey: String): String = if (hasAttr(attributeKey * @see attrAsAbsoluteUrl */ fun Element.attrAsRelativeUrlOrNull(attributeKey: String): String? { - val attr = attr(attributeKey).trim() + val attr = attrOrNull(attributeKey) ?: return null if (attr.isEmpty() || attr.startsWith("data:")) { return null } @@ -74,7 +74,7 @@ fun Element.attrAsRelativeUrl(attributeKey: String): String { * @see attrAsRelativeUrlOrNull */ fun Element.attrAsAbsoluteUrlOrNull(attributeKey: String): String? { - val attr = attr(attributeKey).trim() + val attr = attrOrNull(attributeKey) ?: return null if (attr.isEmpty() || attr.startsWith("data:")) { return null } @@ -140,3 +140,27 @@ fun Element.selectFirstParent(query: String): Element? { selector.matches(root, it) } } + +/** + * Return a first non-empty attribute value of [names] or null if it is missing or empty + */ +fun Element.attrOrNull(vararg names: String): String? { + for (name in names) { + val value = attr(name) + if (value.isNotEmpty()) { + return value.trim() + } + } + return null +} + +@JvmOverloads +fun Element.src(names: Array = arrayOf("data-src", "data-cfsrc", "src")): String? { + for (name in names) { + val value = attrAsAbsoluteUrlOrNull(name) + if (value != null) { + return value + } + } + return null +} From 341aa0e13d7df4227af09f6974efbd9f7116dde4 Mon Sep 17 00:00:00 2001 From: Koitharu Date: Mon, 31 Jul 2023 15:08:46 +0300 Subject: [PATCH 3/3] [JapScan] Migrate to desktop version --- .../kotatsu/parsers/network/UserAgents.kt | 2 ++ .../kotatsu/parsers/site/fr/JapScanParser.kt | 18 ++++++++---------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/network/UserAgents.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/network/UserAgents.kt index a814c9ba..fdd9eb8f 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/network/UserAgents.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/network/UserAgents.kt @@ -8,5 +8,7 @@ object UserAgents { const val CHROME_DESKTOP = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36" + const val FIREFOX_DESKTOP = "Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/116.0" + const val KOTATSU = "Kotatsu/5.3 (Android 12;;; en)" } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/fr/JapScanParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/fr/JapScanParser.kt index 85cad9fc..2591e772 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/fr/JapScanParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/fr/JapScanParser.kt @@ -24,7 +24,7 @@ internal class JapScanParser(context: MangaLoaderContext) : PagedMangaParser(con override val configKeyDomain = ConfigKey.Domain("www.japscan.lol", "japscan.ws") override val headers: Headers = Headers.Builder() - .add("User-Agent", UserAgents.CHROME_MOBILE) + .add("User-Agent", UserAgents.FIREFOX_DESKTOP) .build() override suspend fun getListPage( @@ -41,15 +41,15 @@ internal class JapScanParser(context: MangaLoaderContext) : PagedMangaParser(con .addPathSegment("mangas") .addPathSegment(page.toString()) .build() - val root = webClient.httpGet(url).parseHtml() - .requireElementById("main") - .selectFirstOrThrow(".p-2.row.d-flex") - return root.select("div.col-4") + val doc = webClient.httpGet(url).parseHtml() + val root = doc.requireElementById("main") + .selectFirstOrThrow("div.d-flex.flex-wrap") + return root.select("div.p-2") .map { div -> val href = div.selectFirstOrThrow("a").attrAsRelativeUrl("href") Manga( id = generateUid(href), - title = div.selectFirstOrThrow("p.p-1 a").text(), + title = div.selectFirstOrThrow(".mainTitle").text(), altTitle = null, url = href, publicUrl = href.toAbsoluteUrl(domain), @@ -184,10 +184,8 @@ internal class JapScanParser(context: MangaLoaderContext) : PagedMangaParser(con var key1 = calc1tab.joinToString("") var key2 = calc2tab.joinToString("") - key1 = key1.replace("'", "") - key2 = key2.replace("'", "") - key1 = key1.replace(" ", "") - key2 = key2.replace(" ", "") + key1 = key1.filterNot { c -> c == '\'' || c == ' ' } + key2 = key2.filterNot { c -> c == '\'' || c == ' ' } val keyTables = listOf( key1.reversed(),