Update dependencies

master
Koitharu 8 months ago
parent 4a854c7a23
commit 19567f9642
Signed by: Koitharu
GPG Key ID: 676DEE768C17A9D7

@ -1,5 +1,5 @@
plugins { plugins {
kotlin("jvm") version "2.0.20" kotlin("jvm") version "2.2.10"
} }
repositories { repositories {
@ -14,5 +14,5 @@ dependencies {
implementation(gradleApi()) implementation(gradleApi())
implementation("org.simpleframework:simple-xml:2.7.1") implementation("org.simpleframework:simple-xml:2.7.1")
implementation("com.soywiz.korlibs.korte:korte-jvm:4.0.10") implementation("com.soywiz.korlibs.korte:korte-jvm:4.0.10")
implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.10.1") implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.10.2")
} }

@ -1,5 +1,6 @@
distributionBase=GRADLE_USER_HOME distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists distributionPath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-7.4-bin.zip distributionSha256Sum=bd71102213493060956ec229d946beee57158dbd89d0e62b91bca0fa2c5f3531
distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.3-bin.zip
zipStoreBase=GRADLE_USER_HOME zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists zipStorePath=wrapper/dists

@ -1,13 +1,13 @@
[versions] [versions]
kotlin = "2.0.20" kotlin = "2.2.10"
ksp = "2.0.20-1.0.25" ksp = "2.2.10-2.0.2"
coroutines = "1.10.2" coroutines = "1.10.2"
junit = "5.10.1" junit = "5.10.1"
okhttp = "4.12.0" okhttp = "5.1.0"
okio = "3.11.0" okio = "3.16.0"
json = "20240303" json = "20240303"
androidx-collection = "1.5.0" androidx-collection = "1.5.0"
jsoup = "1.19.1" jsoup = "1.21.2"
quickjs = "1.1.0" quickjs = "1.1.0"
[plugins] [plugins]

@ -1,7 +1,7 @@
#Wed Aug 27 01:56:37 ICT 2025 #Wed Aug 27 01:56:37 ICT 2025
distributionBase=GRADLE_USER_HOME distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists distributionPath=wrapper/dists
distributionSha256Sum=20f1b1176237254a6fc204d8434196fa11a4cfb387567519c61556e8710aed78 distributionSha256Sum=bd71102213493060956ec229d946beee57158dbd89d0e62b91bca0fa2c5f3531
distributionUrl=https\://services.gradle.org/distributions/gradle-8.13-bin.zip distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.3-bin.zip
zipStoreBase=GRADLE_USER_HOME zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists zipStorePath=wrapper/dists

@ -12,61 +12,61 @@ import java.io.Writer
import java.util.* import java.util.*
class ParserProcessor( class ParserProcessor(
private val codeGenerator: CodeGenerator, private val codeGenerator: CodeGenerator,
private val logger: KSPLogger, private val logger: KSPLogger,
private val options: Map<String, String>, private val options: Map<String, String>,
) : SymbolProcessor { ) : SymbolProcessor {
private val availableLocales = Locale.getAvailableLocales().toSet() private val availableLocales = Locale.getAvailableLocales().toSet()
private val sourceNamePattern = Regex("[A-Z_][A-Z0-9_]{3,}") private val sourceNamePattern = Regex("[A-Z_][A-Z0-9_]{3,}")
override fun process(resolver: Resolver): List<KSAnnotated> { override fun process(resolver: Resolver): List<KSAnnotated> {
val symbols = resolver.getSymbolsWithAnnotation("org.koitharu.kotatsu.parsers.MangaSourceParser") val symbols = resolver.getSymbolsWithAnnotation("org.koitharu.kotatsu.parsers.MangaSourceParser")
val ret = symbols.filterNot { it.validate() }.toList() val ret = symbols.filterNot { it.validate() }.toList()
if (!symbols.iterator().hasNext()) { if (!symbols.iterator().hasNext()) {
return ret return ret
} }
val dependencies = Dependencies.ALL_FILES val dependencies = Dependencies.ALL_FILES
val factoryFile = val factoryFile =
try { try {
codeGenerator.createNewFile( codeGenerator.createNewFile(
dependencies = dependencies, dependencies = dependencies,
packageName = "org.koitharu.kotatsu.parsers", packageName = "org.koitharu.kotatsu.parsers",
fileName = "MangaParserFactory", fileName = "MangaParserFactory",
) )
} catch (e: FileAlreadyExistsException) { } catch (e: FileAlreadyExistsException) {
logger.warn(e.toString(), null) logger.warn(e.toString(), null)
null null
} }
val sourcesFile = val sourcesFile =
try { try {
codeGenerator.createNewFile( codeGenerator.createNewFile(
dependencies = dependencies, dependencies = dependencies,
packageName = "org.koitharu.kotatsu.parsers.model", packageName = "org.koitharu.kotatsu.parsers.model",
fileName = "MangaSource", fileName = "MangaSource",
) )
} catch (e: FileAlreadyExistsException) { } catch (e: FileAlreadyExistsException) {
logger.warn(e.toString(), null) logger.warn(e.toString(), null)
null null
} }
val totalCount = sourcesFile?.writer().use { sourcesWriter -> val totalCount = sourcesFile?.writer().use { sourcesWriter ->
factoryFile?.writer().use { factoryWriter -> factoryFile?.writer().use { factoryWriter ->
writeContent(sourcesWriter, factoryWriter, symbols) writeContent(sourcesWriter, factoryWriter, symbols)
} }
} }
writeSummary(totalCount) writeSummary(totalCount)
return ret return ret
} }
private fun writeContent( private fun writeContent(
sourcesWriter: Writer?, sourcesWriter: Writer?,
factoryWriter: Writer?, factoryWriter: Writer?,
symbols: Sequence<KSAnnotated>, symbols: Sequence<KSAnnotated>,
): Int { ): Int {
if (sourcesWriter == null && factoryWriter == null) { if (sourcesWriter == null && factoryWriter == null) {
return 0 return 0
} }
factoryWriter?.write( factoryWriter?.write(
""" """
package org.koitharu.kotatsu.parsers package org.koitharu.kotatsu.parsers
import org.koitharu.kotatsu.parsers.model.MangaParserSource import org.koitharu.kotatsu.parsers.model.MangaParserSource
@ -75,9 +75,9 @@ class ParserProcessor(
internal fun MangaParserSource.newParser(context: MangaLoaderContext): MangaParser = when (this) { internal fun MangaParserSource.newParser(context: MangaLoaderContext): MangaParser = when (this) {
""".trimIndent(), """.trimIndent(),
) )
sourcesWriter?.write( sourcesWriter?.write(
""" """
package org.koitharu.kotatsu.parsers.model package org.koitharu.kotatsu.parsers.model
public enum class MangaParserSource( public enum class MangaParserSource(
@ -88,16 +88,16 @@ class ParserProcessor(
): MangaSource { ): MangaSource {
""".trimIndent(), """.trimIndent(),
) )
val visitor = ParserVisitor(sourcesWriter, factoryWriter) val visitor = ParserVisitor(sourcesWriter, factoryWriter)
val totalCount = symbols val totalCount = symbols
.filter { it is KSClassDeclaration && it.validate() } .filter { it is KSClassDeclaration && it.validate() }
.onEach { it.accept(visitor, Unit) } .onEach { it.accept(visitor, Unit) }
.count() .count()
factoryWriter?.write( factoryWriter?.write(
""" """
MangaParserSource.DUMMY -> throw NotImplementedError("Manga parser ${'$'}name cannot be instantiated") MangaParserSource.DUMMY -> throw NotImplementedError("Manga parser ${'$'}name cannot be instantiated")
}.let { }.let {
require(it.source == this) { require(it.source == this) {
@ -106,83 +106,83 @@ class ParserProcessor(
MangaParserWrapper(it) MangaParserWrapper(it)
} }
""".trimIndent(), """.trimIndent(),
) )
sourcesWriter?.write( sourcesWriter?.write(
""" """
DUMMY("Dummy", "", ContentType.OTHER, false), DUMMY("Dummy", "", ContentType.OTHER, false),
; ;
} }
""".trimIndent(), """.trimIndent(),
) )
return totalCount return totalCount
} }
private fun writeSummary(totalCount: Int) { private fun writeSummary(totalCount: Int) {
val file = File(options["summaryOutputDir"] ?: return, "summary.yaml") val file = File(options["summaryOutputDir"] ?: return, "summary.yaml")
file.writeText("total: $totalCount") file.writeText("total: $totalCount")
} }
private inner class ParserVisitor( private inner class ParserVisitor(
private val sourcesWriter: Writer?, private val sourcesWriter: Writer?,
private val factoryWriter: Writer?, private val factoryWriter: Writer?,
) : KSVisitorVoid() { ) : KSVisitorVoid() {
private val titles = HashMap<String, String>() private val titles = HashMap<String, String>()
override fun visitClassDeclaration( override fun visitClassDeclaration(
classDeclaration: KSClassDeclaration, classDeclaration: KSClassDeclaration,
data: Unit, data: Unit,
) { ) {
if (classDeclaration.classKind != ClassKind.CLASS || classDeclaration.isAbstract()) { if (classDeclaration.classKind != ClassKind.CLASS || classDeclaration.isAbstract()) {
logger.error("Only non-abstract can be annotated with @MangaSourceParser", classDeclaration) logger.error("Only non-abstract can be annotated with @MangaSourceParser", classDeclaration)
} }
val annotation = classDeclaration.annotations.single { it.shortName.asString() == "MangaSourceParser" } val annotation = classDeclaration.annotations.single { it.shortName.asString() == "MangaSourceParser" }
val deprecation = classDeclaration.annotations.singleOrNull { it.shortName.asString() == "Deprecated" } val deprecation = classDeclaration.annotations.singleOrNull { it.shortName.asString() == "Deprecated" }
val isBroken = classDeclaration.annotations.any { it.shortName.asString() == "Broken" } val isBroken = classDeclaration.annotations.any { it.shortName.asString() == "Broken" }
val name = annotation.arguments.single { it.name?.asString() == "name" }.value as String val name = annotation.arguments.single { it.name?.asString() == "name" }.value as String
val title = annotation.arguments.single { it.name?.asString() == "title" }.value as String val title = annotation.arguments.single { it.name?.asString() == "title" }.value as String
val locale = annotation.arguments.single { it.name?.asString() == "locale" }.value as String val locale = annotation.arguments.single { it.name?.asString() == "locale" }.value as String
val type = annotation.arguments.single { it.name?.asString() == "type" }.value val type = annotation.arguments.single { it.name?.asString() == "type" }.value
val localeString = "\"$locale\"" val localeString = "\"$locale\""
val localeObj = if (locale.isEmpty()) null else Locale(locale) val localeObj = if (locale.isEmpty()) null else Locale(locale)
val localeTitle = localeObj?.getDisplayLanguage(localeObj) val localeTitle = localeObj?.getDisplayLanguage(localeObj)
if (localeObj != null && localeObj !in availableLocales) { if (localeObj != null && localeObj !in availableLocales) {
logger.error("Manga source $name has wrong locale: $localeTitle") logger.error("Manga source $name has wrong locale: $localeTitle")
} }
if (!sourceNamePattern.matches(name)) { if (!sourceNamePattern.matches(name)) {
logger.error("Manga source name must be uppercase: $name") logger.error("Manga source name must be uppercase: $name")
} }
val constructor = classDeclaration.primaryConstructor val constructor = classDeclaration.primaryConstructor
if (constructor == null || constructor.parameters.count { !it.hasDefault } != 1) { if (constructor == null || constructor.parameters.count { !it.hasDefault } != 1) {
logger.error( logger.error(
"Class with @MangaSourceParser must have a primary constructor with one parameter", "Class with @MangaSourceParser must have a primary constructor with one parameter",
classDeclaration, classDeclaration,
) )
} }
val className = checkNotNull(classDeclaration.qualifiedName?.asString()) { "Class name is null" } val className = checkNotNull(classDeclaration.qualifiedName?.asString()) { "Class name is null" }
val prevTitleClass = titles.put(title, className) val prevTitleClass = titles.put(title, className)
if (prevTitleClass != null) { if (prevTitleClass != null) {
logger.warn("Source title duplication: \"$title\" is assigned to both $prevTitleClass and $className") logger.warn("Source title duplication: \"$title\" is assigned to both $prevTitleClass and $className")
} }
factoryWriter?.write("\tMangaParserSource.$name -> $className(context)\n") factoryWriter?.write("\tMangaParserSource.$name -> $className(context)\n")
val deprecationString = val deprecationString =
if (deprecation != null) { if (deprecation != null) {
val reason = val reason =
deprecation.arguments deprecation.arguments
.find { it.name?.asString() == "message" } .find { it.name?.asString() == "message" }
?.value ?.value
?.toString() ?: "Unknown reason" ?.toString() ?: "Unknown reason"
"@Deprecated(\"$reason\") " "@Deprecated(\"$reason\") "
} else { } else {
"" ""
} }
val localeComment = localeTitle?.toTitleCase(localeObj)?.let { " /* $it */" }.orEmpty() val localeComment = localeTitle?.toTitleCase(localeObj)?.let { " /* $it */" }.orEmpty()
sourcesWriter?.write( sourcesWriter?.write(
"\t$deprecationString$name(\"$title\", $localeString$localeComment, ContentType.$type, $isBroken),\n", "\t$deprecationString$name(\"$title\", $localeString$localeComment, $type, $isBroken),\n",
) )
} }
} }
} }

@ -2,18 +2,17 @@ package org.koitharu.kotatsu.parsers
public object ErrorMessages { public object ErrorMessages {
public const val FILTER_MULTIPLE_STATES_NOT_SUPPORTED: String = "Multiple states are not supported by this source" public const val FILTER_MULTIPLE_STATES_NOT_SUPPORTED: String = "Multiple states are not supported by this source"
public const val FILTER_MULTIPLE_GENRES_NOT_SUPPORTED: String = "Multiple genres are not supported by this source" public const val FILTER_MULTIPLE_GENRES_NOT_SUPPORTED: String = "Multiple genres are not supported by this source"
public const val FILTER_MULTIPLE_CONTENT_RATING_NOT_SUPPORTED: String = public const val FILTER_MULTIPLE_CONTENT_RATING_NOT_SUPPORTED: String =
"Multiple Content ratings are not supported by this source" "Multiple Content ratings are not supported by this source"
public const val FILTER_MULTIPLE_CONTENT_TYPES_NOT_SUPPORTED: String = public const val FILTER_MULTIPLE_CONTENT_TYPES_NOT_SUPPORTED: String =
"Multiple Content types are not supported by this source" "Multiple Content types are not supported by this source"
public const val FILTER_MULTIPLE_DEMOGRAPHICS_NOT_SUPPORTED: String = public const val FILTER_MULTIPLE_DEMOGRAPHICS_NOT_SUPPORTED: String =
"Multiple Demographics are not supported by this source" "Multiple Demographics are not supported by this source"
public const val FILTER_BOTH_LOCALE_GENRES_NOT_SUPPORTED: String = public const val FILTER_BOTH_LOCALE_GENRES_NOT_SUPPORTED: String =
"Filtering by both genres and locale is not supported by this source" "Filtering by both genres and locale is not supported by this source"
public const val FILTER_BOTH_STATES_GENRES_NOT_SUPPORTED: String = public const val FILTER_BOTH_STATES_GENRES_NOT_SUPPORTED: String =
"Filtering by both genres and states is not supported by this source" "Filtering by both genres and states is not supported by this source"
public const val SEARCH_NOT_SUPPORTED: String = "Search is not supported by this source" public const val SEARCH_NOT_SUPPORTED: String = "Search is not supported by this source"
public const val RESPONSE_NULL_BODY: String = "Response has no body"
} }

@ -9,39 +9,39 @@ import java.net.HttpURLConnection.HTTP_UNAVAILABLE
public object CloudFlareHelper { public object CloudFlareHelper {
public const val PROTECTION_NOT_DETECTED: Int = 0 public const val PROTECTION_NOT_DETECTED: Int = 0
public const val PROTECTION_CAPTCHA: Int = 1 public const val PROTECTION_CAPTCHA: Int = 1
public const val PROTECTION_BLOCKED: Int = 2 public const val PROTECTION_BLOCKED: Int = 2
private const val CF_CLEARANCE = "cf_clearance" private const val CF_CLEARANCE = "cf_clearance"
public fun checkResponseForProtection(response: Response): Int { public fun checkResponseForProtection(response: Response): Int {
if (response.code != HTTP_FORBIDDEN && response.code != HTTP_UNAVAILABLE) { if (response.code != HTTP_FORBIDDEN && response.code != HTTP_UNAVAILABLE) {
return PROTECTION_NOT_DETECTED return PROTECTION_NOT_DETECTED
} }
val content = if (response.body != null) { val content = try {
response.peekBody(Long.MAX_VALUE).use { response.peekBody(Long.MAX_VALUE).use {
Jsoup.parse(it.byteStream(), Charsets.UTF_8.name(), response.request.url.toString()) Jsoup.parse(it.byteStream(), Charsets.UTF_8.name(), response.request.url.toString())
} }
} else { } catch (_: IllegalStateException) {
return PROTECTION_NOT_DETECTED return PROTECTION_NOT_DETECTED
} }
return when { return when {
content.selectFirst("h2[data-translate=\"blocked_why_headline\"]") != null -> PROTECTION_BLOCKED content.selectFirst("h2[data-translate=\"blocked_why_headline\"]") != null -> PROTECTION_BLOCKED
content.getElementById("challenge-error-title") != null || content.getElementById("challenge-error-text") != null -> PROTECTION_CAPTCHA content.getElementById("challenge-error-title") != null || content.getElementById("challenge-error-text") != null -> PROTECTION_CAPTCHA
else -> PROTECTION_NOT_DETECTED else -> PROTECTION_NOT_DETECTED
} }
} }
public fun getClearanceCookie(cookieJar: CookieJar, url: String): String? { public fun getClearanceCookie(cookieJar: CookieJar, url: String): String? {
return cookieJar.loadForRequest(url.toHttpUrl()).find { it.name == CF_CLEARANCE }?.value return cookieJar.loadForRequest(url.toHttpUrl()).find { it.name == CF_CLEARANCE }?.value
} }
public fun isCloudFlareCookie(name: String): Boolean { public fun isCloudFlareCookie(name: String): Boolean {
return name.startsWith("cf_") return name.startsWith("cf_")
|| name.startsWith("_cf") || name.startsWith("_cf")
|| name.startsWith("__cf") || name.startsWith("__cf")
|| name == "csrftoken" || name == "csrftoken"
} }
} }

@ -7,7 +7,6 @@ import okhttp3.HttpUrl.Companion.toHttpUrlOrNull
import okhttp3.Interceptor import okhttp3.Interceptor
import okhttp3.Response import okhttp3.Response
import okhttp3.internal.closeQuietly import okhttp3.internal.closeQuietly
import okhttp3.internal.headersContentLength
import org.jsoup.internal.StringUtil import org.jsoup.internal.StringUtil
import org.jsoup.nodes.Element import org.jsoup.nodes.Element
import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaLoaderContext
@ -28,470 +27,471 @@ import java.util.concurrent.TimeUnit
private const val DOMAIN_UNAUTHORIZED = "e-hentai.org" private const val DOMAIN_UNAUTHORIZED = "e-hentai.org"
private const val DOMAIN_AUTHORIZED = "exhentai.org" private const val DOMAIN_AUTHORIZED = "exhentai.org"
private val TAG_PREFIXES = arrayOf("male:", "female:", "other:") private val TAG_PREFIXES = arrayOf("male:", "female:", "other:")
private const val BANNED_RESPONSE_LENGTH = 256L
@MangaSourceParser("EXHENTAI", "ExHentai", type = ContentType.HENTAI) @MangaSourceParser("EXHENTAI", "ExHentai", type = ContentType.HENTAI)
internal class ExHentaiParser( internal class ExHentaiParser(
context: MangaLoaderContext, context: MangaLoaderContext,
) : PagedMangaParser(context, MangaParserSource.EXHENTAI, pageSize = 25), MangaParserAuthProvider, Interceptor { ) : PagedMangaParser(context, MangaParserSource.EXHENTAI, pageSize = 25), MangaParserAuthProvider, Interceptor {
override val availableSortOrders: Set<SortOrder> = setOf(SortOrder.NEWEST) override val availableSortOrders: Set<SortOrder> = setOf(SortOrder.NEWEST)
override val configKeyDomain: ConfigKey.Domain override val configKeyDomain: ConfigKey.Domain
get() { get() {
val isAuthorized = checkAuth() val isAuthorized = checkAuth()
return ConfigKey.Domain( return ConfigKey.Domain(
if (isAuthorized) DOMAIN_AUTHORIZED else DOMAIN_UNAUTHORIZED, if (isAuthorized) DOMAIN_AUTHORIZED else DOMAIN_UNAUTHORIZED,
if (isAuthorized) DOMAIN_UNAUTHORIZED else DOMAIN_AUTHORIZED, if (isAuthorized) DOMAIN_UNAUTHORIZED else DOMAIN_AUTHORIZED,
) )
} }
override val authUrl: String override val authUrl: String
get() = "https://${domain}/bounce_login.php" get() = "https://${domain}/bounce_login.php"
private val ratingPattern = Regex("-?[0-9]+px") private val ratingPattern = Regex("-?[0-9]+px")
private val titleCleanupPattern = Regex("(\\[.*?]|\\([C0-9]*\\))") private val titleCleanupPattern = Regex("(\\[.*?]|\\([C0-9]*\\))")
private val spacesCleanupPattern = Regex("(^\\s+|\\s+\$|\\s+(?=\\s))") private val spacesCleanupPattern = Regex("(^\\s+|\\s+\$|\\s+(?=\\s))")
private val authCookies = arrayOf("ipb_member_id", "ipb_pass_hash") private val authCookies = arrayOf("ipb_member_id", "ipb_pass_hash")
private val suspiciousContentKey = ConfigKey.ShowSuspiciousContent(false) private val suspiciousContentKey = ConfigKey.ShowSuspiciousContent(false)
private val nextPages = MutableIntObjectMap<MutableIntLongMap>() private val nextPages = MutableIntObjectMap<MutableIntLongMap>()
override val filterCapabilities: MangaListFilterCapabilities override val filterCapabilities: MangaListFilterCapabilities
get() = MangaListFilterCapabilities( get() = MangaListFilterCapabilities(
isMultipleTagsSupported = true, isMultipleTagsSupported = true,
isTagsExclusionSupported = true, isTagsExclusionSupported = true,
isSearchSupported = true, isSearchSupported = true,
isSearchWithFiltersSupported = true, isSearchWithFiltersSupported = true,
isAuthorSearchSupported = true, isAuthorSearchSupported = true,
) )
override suspend fun isAuthorized(): Boolean = checkAuth() override suspend fun isAuthorized(): Boolean = checkAuth()
init { init {
context.cookieJar.insertCookies(DOMAIN_AUTHORIZED, "nw=1", "sl=dm_2") context.cookieJar.insertCookies(DOMAIN_AUTHORIZED, "nw=1", "sl=dm_2")
context.cookieJar.insertCookies(DOMAIN_UNAUTHORIZED, "nw=1", "sl=dm_2") context.cookieJar.insertCookies(DOMAIN_UNAUTHORIZED, "nw=1", "sl=dm_2")
paginator.firstPage = 0 paginator.firstPage = 0
searchPaginator.firstPage = 0 searchPaginator.firstPage = 0
} }
override suspend fun getFilterOptions() = MangaListFilterOptions( override suspend fun getFilterOptions() = MangaListFilterOptions(
availableTags = mapTags(), availableTags = mapTags(),
availableContentTypes = EnumSet.of( availableContentTypes = EnumSet.of(
ContentType.DOUJINSHI, ContentType.DOUJINSHI,
ContentType.MANGA, ContentType.MANGA,
ContentType.ARTIST_CG, ContentType.ARTIST_CG,
ContentType.GAME_CG, ContentType.GAME_CG,
ContentType.COMICS, ContentType.COMICS,
ContentType.IMAGE_SET, ContentType.IMAGE_SET,
ContentType.OTHER, ContentType.OTHER,
), ),
availableLocales = setOf( availableLocales = setOf(
Locale.JAPANESE, Locale.JAPANESE,
Locale.ENGLISH, Locale.ENGLISH,
Locale.CHINESE, Locale.CHINESE,
Locale("nl"), Locale("nl"),
Locale.FRENCH, Locale.FRENCH,
Locale.GERMAN, Locale.GERMAN,
Locale("hu"), Locale("hu"),
Locale.ITALIAN, Locale.ITALIAN,
Locale("kr"), Locale("kr"),
Locale("pl"), Locale("pl"),
Locale("pt"), Locale("pt"),
Locale("ru"), Locale("ru"),
Locale("es"), Locale("es"),
Locale("th"), Locale("th"),
Locale("vi"), Locale("vi"),
), ),
) )
override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> {
return getListPage(page, order, filter, updateDm = false) return getListPage(page, order, filter, updateDm = false)
} }
private suspend fun getListPage( private suspend fun getListPage(
page: Int, page: Int,
order: SortOrder, order: SortOrder,
filter: MangaListFilter, filter: MangaListFilter,
updateDm: Boolean, updateDm: Boolean,
): List<Manga> { ): List<Manga> {
val next = synchronized(nextPages) { val next = synchronized(nextPages) {
nextPages[filter.hashCode()]?.getOrDefault(page, 0L) ?: 0L nextPages[filter.hashCode()]?.getOrDefault(page, 0L) ?: 0L
} }
if (page > 0 && next == 0L) { if (page > 0 && next == 0L) {
assert(false) { "Page timestamp not found" } assert(false) { "Page timestamp not found" }
return emptyList() return emptyList()
} }
val url = urlBuilder() val url = urlBuilder()
url.addEncodedQueryParameter("next", next.toString()) url.addEncodedQueryParameter("next", next.toString())
url.addQueryParameter("f_search", filter.toSearchQuery()) url.addQueryParameter("f_search", filter.toSearchQuery())
val fCats = filter.types.toFCats() val fCats = filter.types.toFCats()
if (fCats != 0) { if (fCats != 0) {
url.addEncodedQueryParameter("f_cats", (1023 - fCats).toString()) url.addEncodedQueryParameter("f_cats", (1023 - fCats).toString())
} }
if (updateDm) { if (updateDm) {
// by unknown reason cookie "sl=dm_2" is ignored, so, we should request it again // by unknown reason cookie "sl=dm_2" is ignored, so, we should request it again
url.addQueryParameter("inline_set", "dm_e") url.addQueryParameter("inline_set", "dm_e")
} }
url.addQueryParameter("advsearch", "1") url.addQueryParameter("advsearch", "1")
if (config[suspiciousContentKey]) { if (config[suspiciousContentKey]) {
url.addQueryParameter("f_sh", "on") url.addQueryParameter("f_sh", "on")
} }
val body = webClient.httpGet(url.build()).parseHtml().body() val body = webClient.httpGet(url.build()).parseHtml().body()
val root = body.selectFirst("table.itg")?.selectFirst("tbody") val root = body.selectFirst("table.itg")?.selectFirst("tbody")
if (root == null) { if (root == null) {
if (updateDm) { if (updateDm) {
if (body.getElementsContainingText("No hits found").isNotEmpty()) { if (body.getElementsContainingText("No hits found").isNotEmpty()) {
return emptyList() return emptyList()
} else { } else {
body.parseFailed("Cannot find root") body.parseFailed("Cannot find root")
} }
} else { } else {
return getListPage(page, order, filter, updateDm = true) return getListPage(page, order, filter, updateDm = true)
} }
} }
val nextTimestamp = getNextTimestamp(body) val nextTimestamp = getNextTimestamp(body)
synchronized(nextPages) { synchronized(nextPages) {
nextPages.getOrPut(filter.hashCode()) { nextPages.getOrPut(filter.hashCode()) {
MutableIntLongMap() MutableIntLongMap()
}.put(page + 1, nextTimestamp) }.put(page + 1, nextTimestamp)
} }
return root.children().mapNotNull { tr -> return root.children().mapNotNull { tr ->
if (tr.childrenSize() != 2) return@mapNotNull null if (tr.childrenSize() != 2) return@mapNotNull null
val (td1, td2) = tr.children() val (td1, td2) = tr.children()
val gLink = td2.selectFirstOrThrow("div.glink") val gLink = td2.selectFirstOrThrow("div.glink")
val a = gLink.parents().select("a").first() ?: gLink.parseFailed("link not found") val a = gLink.parents().select("a").first() ?: gLink.parseFailed("link not found")
val href = a.attrAsRelativeUrl("href") val href = a.attrAsRelativeUrl("href")
val tagsDiv = gLink.nextElementSibling() ?: gLink.parseFailed("tags div not found") val tagsDiv = gLink.nextElementSibling() ?: gLink.parseFailed("tags div not found")
val rawTitle = gLink.text() val rawTitle = gLink.text()
val author = tagsDiv.getElementsContainingOwnText("artist:").first() val author = tagsDiv.getElementsContainingOwnText("artist:").first()
?.nextElementSibling()?.textOrNull() ?.nextElementSibling()?.textOrNull()
Manga( Manga(
id = generateUid(href), id = generateUid(href),
title = rawTitle.cleanupTitle(), title = rawTitle.cleanupTitle(),
altTitles = emptySet(), altTitles = emptySet(),
url = href, url = href,
publicUrl = a.absUrl("href"), publicUrl = a.absUrl("href"),
rating = td2.selectFirst("div.ir")?.parseRating() ?: RATING_UNKNOWN, rating = td2.selectFirst("div.ir")?.parseRating() ?: RATING_UNKNOWN,
contentRating = ContentRating.ADULT, contentRating = ContentRating.ADULT,
coverUrl = td1.selectFirst("img")?.attrAsAbsoluteUrlOrNull("src"), coverUrl = td1.selectFirst("img")?.attrAsAbsoluteUrlOrNull("src"),
tags = tagsDiv.parseTags(), tags = tagsDiv.parseTags(),
state = when { state = when {
rawTitle.contains("(ongoing)", ignoreCase = true) -> MangaState.ONGOING rawTitle.contains("(ongoing)", ignoreCase = true) -> MangaState.ONGOING
else -> null else -> null
}, },
authors = setOfNotNull(author), authors = setOfNotNull(author),
source = source, source = source,
) )
} }
} }
override suspend fun getDetails(manga: Manga): Manga { override suspend fun getDetails(manga: Manga): Manga {
val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml()
val root = doc.body().selectFirstOrThrow("div.gm") val root = doc.body().selectFirstOrThrow("div.gm")
val cover = root.getElementById("gd1")?.children()?.first() val cover = root.getElementById("gd1")?.children()?.first()
val title = root.getElementById("gd2") val title = root.getElementById("gd2")
val tagList = root.getElementById("taglist") val tagList = root.getElementById("taglist")
val tabs = doc.body().selectFirst("table.ptt")?.selectFirst("tr") val tabs = doc.body().selectFirst("table.ptt")?.selectFirst("tr")
val gd3 = root.getElementById("gd3") val gd3 = root.getElementById("gd3")
val lang = gd3 val lang = gd3
?.selectFirst("tr:contains(Language)") ?.selectFirst("tr:contains(Language)")
?.selectFirst(".gdt2")?.ownTextOrNull() ?.selectFirst(".gdt2")?.ownTextOrNull()
val uploadDate = gd3 val uploadDate = gd3
?.selectFirst("tr:contains(Posted)") ?.selectFirst("tr:contains(Posted)")
?.selectFirst(".gdt2")?.ownTextOrNull() ?.selectFirst(".gdt2")?.ownTextOrNull()
.let { SimpleDateFormat("yyyy-MM-dd HH:mm", sourceLocale).parseSafe(it) } .let { SimpleDateFormat("yyyy-MM-dd HH:mm", sourceLocale).parseSafe(it) }
val uploader = gd3 val uploader = gd3
?.getElementsByAttributeValueContaining("href", "/uploader/") ?.getElementsByAttributeValueContaining("href", "/uploader/")
?.firstOrNull() ?.firstOrNull()
?.ownTextOrNull() ?.ownTextOrNull()
val tags = tagList?.parseTags().orEmpty() val tags = tagList?.parseTags().orEmpty()
return manga.copy( return manga.copy(
title = title?.getElementById("gn")?.text()?.cleanupTitle() ?: manga.title, title = title?.getElementById("gn")?.text()?.cleanupTitle() ?: manga.title,
altTitles = setOfNotNull(title?.getElementById("gj")?.text()?.cleanupTitle()?.nullIfEmpty()), altTitles = setOfNotNull(title?.getElementById("gj")?.text()?.cleanupTitle()?.nullIfEmpty()),
publicUrl = doc.baseUri().ifEmpty { manga.publicUrl }, publicUrl = doc.baseUri().ifEmpty { manga.publicUrl },
rating = root.getElementById("rating_label")?.text() rating = root.getElementById("rating_label")?.text()
?.substringAfterLast(' ') ?.substringAfterLast(' ')
?.toFloatOrNull() ?.toFloatOrNull()
?.div(5f) ?: manga.rating, ?.div(5f) ?: manga.rating,
largeCoverUrl = cover?.styleValueOrNull("background")?.cssUrl(), largeCoverUrl = cover?.styleValueOrNull("background")?.cssUrl(),
tags = manga.tags + tags, tags = manga.tags + tags,
description = tagList?.select("tr")?.joinToString("<br>") { tr -> description = tagList?.select("tr")?.joinToString("<br>") { tr ->
val (tc, td) = tr.children() val (tc, td) = tr.children()
val subTags = td.select("a").joinToString { it.html() } val subTags = td.select("a").joinToString { it.html() }
"<b>${tc.html()}</b> $subTags" "<b>${tc.html()}</b> $subTags"
}, },
chapters = tabs?.select("a")?.findLast { a -> chapters = tabs?.select("a")?.findLast { a ->
a.text().toIntOrNull() != null a.text().toIntOrNull() != null
}?.let { a -> }?.let { a ->
val count = a.text().toInt() val count = a.text().toInt()
val chapters = ChaptersListBuilder(count) val chapters = ChaptersListBuilder(count)
for (i in 1..count) { for (i in 1..count) {
val url = "${manga.url}?p=${i - 1}" val url = "${manga.url}?p=${i - 1}"
chapters += MangaChapter( chapters += MangaChapter(
id = generateUid(url), id = generateUid(url),
title = null, title = null,
number = i.toFloat(), number = i.toFloat(),
volume = 0, volume = 0,
url = url, url = url,
uploadDate = uploadDate, uploadDate = uploadDate,
source = source, source = source,
scanlator = uploader, scanlator = uploader,
branch = lang, branch = lang,
) )
} }
chapters.toList() chapters.toList()
}, },
) )
} }
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml()
val root = doc.body().requireElementById("gdt") val root = doc.body().requireElementById("gdt")
return root.select("a").map { a -> return root.select("a").map { a ->
val url = a.attrAsRelativeUrl("href") val url = a.attrAsRelativeUrl("href")
MangaPage( MangaPage(
id = generateUid(url), id = generateUid(url),
url = url, url = url,
preview = a.children().firstOrNull()?.extractPreview(), preview = a.children().firstOrNull()?.extractPreview(),
source = source, source = source,
) )
} }
} }
override suspend fun getPageUrl(page: MangaPage): String { override suspend fun getPageUrl(page: MangaPage): String {
val doc = webClient.httpGet(page.url.toAbsoluteUrl(domain)).parseHtml() val doc = webClient.httpGet(page.url.toAbsoluteUrl(domain)).parseHtml()
return doc.body().requireElementById("img").attrAsAbsoluteUrl("src") return doc.body().requireElementById("img").attrAsAbsoluteUrl("src")
} }
@Suppress("SpellCheckingInspection") @Suppress("SpellCheckingInspection")
private val tags: String private val tags: String
get() = "ahegao,anal,angel,apron,bandages,bbw,bdsm,beauty mark,big areolae,big ass,big breasts,big clit,big lips," + get() = "ahegao,anal,angel,apron,bandages,bbw,bdsm,beauty mark,big areolae,big ass,big breasts,big clit,big lips," +
"big nipples,bikini,blackmail,bloomers,blowjob,bodysuit,bondage,breast expansion,bukkake,bunny girl,business suit," + "big nipples,bikini,blackmail,bloomers,blowjob,bodysuit,bondage,breast expansion,bukkake,bunny girl,business suit," +
"catgirl,centaur,cheating,chinese dress,christmas,collar,corset,cosplaying,cowgirl,crossdressing,cunnilingus," + "catgirl,centaur,cheating,chinese dress,christmas,collar,corset,cosplaying,cowgirl,crossdressing,cunnilingus," +
"dark skin,daughter,deepthroat,defloration,demon girl,double penetration,dougi,dragon,drunk,elf,exhibitionism,farting," + "dark skin,daughter,deepthroat,defloration,demon girl,double penetration,dougi,dragon,drunk,elf,exhibitionism,farting," +
"females only,femdom,filming,fingering,fishnets,footjob,fox girl,furry,futanari,garter belt,ghost,giantess," + "females only,femdom,filming,fingering,fishnets,footjob,fox girl,furry,futanari,garter belt,ghost,giantess," +
"glasses,gloves,goblin,gothic lolita,growth,guro,gyaru,hair buns,hairy,hairy armpits,handjob,harem,hidden sex," + "glasses,gloves,goblin,gothic lolita,growth,guro,gyaru,hair buns,hairy,hairy armpits,handjob,harem,hidden sex," +
"horns,huge breasts,humiliation,impregnation,incest,inverted nipples,kemonomimi,kimono,kissing,lactation," + "horns,huge breasts,humiliation,impregnation,incest,inverted nipples,kemonomimi,kimono,kissing,lactation," +
"latex,leg lock,leotard,lingerie,lizard girl,maid,masked face,masturbation,midget,miko,milf,mind break," + "latex,leg lock,leotard,lingerie,lizard girl,maid,masked face,masturbation,midget,miko,milf,mind break," +
"mind control,monster girl,mother,muscle,nakadashi,netorare,nose hook,nun,nurse,oil,paizuri,panda girl," + "mind control,monster girl,mother,muscle,nakadashi,netorare,nose hook,nun,nurse,oil,paizuri,panda girl," +
"pantyhose,piercing,pixie cut,policewoman,ponytail,pregnant,rape,rimjob,robot,scat,lolicon,schoolgirl uniform," + "pantyhose,piercing,pixie cut,policewoman,ponytail,pregnant,rape,rimjob,robot,scat,lolicon,schoolgirl uniform," +
"sex toys,shemale,sister,small breasts,smell,sole dickgirl,sole female,squirting,stockings,sundress,sweating," + "sex toys,shemale,sister,small breasts,smell,sole dickgirl,sole female,squirting,stockings,sundress,sweating," +
"swimsuit,swinging,tail,tall girl,teacher,tentacles,thigh high boots,tomboy,transformation,twins,twintails," + "swimsuit,swinging,tail,tall girl,teacher,tentacles,thigh high boots,tomboy,transformation,twins,twintails," +
"unusual pupils,urination,vore,vtuber,widow,wings,witch,wolf girl,x-ray,yuri,zombie,sole male,males only,yaoi," + "unusual pupils,urination,vore,vtuber,widow,wings,witch,wolf girl,x-ray,yuri,zombie,sole male,males only,yaoi," +
"tomgirl,tall man,oni,shotacon,prostate massage,policeman,males only,huge penis,fox boy,feminization,dog boy,dickgirl on male,big penis" "tomgirl,tall man,oni,shotacon,prostate massage,policeman,males only,huge penis,fox boy,feminization,dog boy,dickgirl on male,big penis"
private fun mapTags(): Set<MangaTag> { private fun mapTags(): Set<MangaTag> {
val tagElements = tags.split(",") val tagElements = tags.split(",")
val result = ArraySet<MangaTag>(tagElements.size) val result = ArraySet<MangaTag>(tagElements.size)
for (tag in tagElements) { for (tag in tagElements) {
val el = tag.trim() val el = tag.trim()
if (el.isEmpty()) continue if (el.isEmpty()) continue
result += MangaTag( result += MangaTag(
title = el.toTitleCase(Locale.ENGLISH), title = el.toTitleCase(Locale.ENGLISH),
key = el, key = el,
source = source, source = source,
) )
} }
return result return result
} }
override fun intercept(chain: Interceptor.Chain): Response { override fun intercept(chain: Interceptor.Chain): Response {
val response = chain.proceed(chain.request()) val response = chain.proceed(chain.request())
if (response.headersContentLength() <= 256) { if (response.headersContentLength(BANNED_RESPONSE_LENGTH) <= BANNED_RESPONSE_LENGTH) {
val text = response.peekBody(256).use { it.string() } val text = response.peekBody(BANNED_RESPONSE_LENGTH).use { it.string() }
if (text.contains("IP address has been temporarily banned", ignoreCase = true)) { if (text.contains("IP address has been temporarily banned", ignoreCase = true)) {
val hours = Regex("([0-9]+) hours?").find(text)?.groupValues?.getOrNull(1)?.toLongOrNull() ?: 0 val hours = Regex("([0-9]+) hours?").find(text)?.groupValues?.getOrNull(1)?.toLongOrNull() ?: 0
val minutes = Regex("([0-9]+) minutes?").find(text)?.groupValues?.getOrNull(1)?.toLongOrNull() ?: 0 val minutes = Regex("([0-9]+) minutes?").find(text)?.groupValues?.getOrNull(1)?.toLongOrNull() ?: 0
val seconds = Regex("([0-9]+) seconds?").find(text)?.groupValues?.getOrNull(1)?.toLongOrNull() ?: 0 val seconds = Regex("([0-9]+) seconds?").find(text)?.groupValues?.getOrNull(1)?.toLongOrNull() ?: 0
response.closeQuietly() response.closeQuietly()
throw TooManyRequestExceptions( throw TooManyRequestExceptions(
url = response.request.url.toString(), url = response.request.url.toString(),
retryAfter = TimeUnit.HOURS.toMillis(hours) retryAfter = TimeUnit.HOURS.toMillis(hours)
+ TimeUnit.MINUTES.toMillis(minutes) + TimeUnit.MINUTES.toMillis(minutes)
+ TimeUnit.SECONDS.toMillis(seconds), + TimeUnit.SECONDS.toMillis(seconds),
) )
} }
} }
val imageRect = response.request.url.fragment?.split(',') val imageRect = response.request.url.fragment?.split(',')
if (imageRect != null && imageRect.size == 4) { if (imageRect != null && imageRect.size == 4) {
// rect: top,left,right,bottom // rect: top,left,right,bottom
return context.redrawImageResponse(response) { bitmap -> return context.redrawImageResponse(response) { bitmap ->
val srcRect = Rect( val srcRect = Rect(
left = imageRect[0].toInt(), left = imageRect[0].toInt(),
top = imageRect[1].toInt(), top = imageRect[1].toInt(),
right = imageRect[2].toInt(), right = imageRect[2].toInt(),
bottom = imageRect[3].toInt(), bottom = imageRect[3].toInt(),
) )
val dstRect = Rect(0, 0, srcRect.width, srcRect.height) val dstRect = Rect(0, 0, srcRect.width, srcRect.height)
val result = context.createBitmap(dstRect.width, dstRect.height) val result = context.createBitmap(dstRect.width, dstRect.height)
result.drawBitmap(bitmap, srcRect, dstRect) result.drawBitmap(bitmap, srcRect, dstRect)
result result
} }
} }
return response return response
} }
private fun Locale.toLanguagePath() = when (language) { private fun Locale.toLanguagePath() = when (language) {
else -> getDisplayLanguage(Locale.ENGLISH).lowercase() else -> getDisplayLanguage(Locale.ENGLISH).lowercase()
} }
override suspend fun getUsername(): String { override suspend fun getUsername(): String {
val doc = webClient.httpGet("https://forums.$DOMAIN_UNAUTHORIZED/").parseHtml().body() val doc = webClient.httpGet("https://forums.$DOMAIN_UNAUTHORIZED/").parseHtml().body()
val username = doc.getElementById("userlinks") val username = doc.getElementById("userlinks")
?.getElementsByAttributeValueContaining("href", "showuser=") ?.getElementsByAttributeValueContaining("href", "showuser=")
?.firstOrNull() ?.firstOrNull()
?.ownText() ?.ownText()
?: if (doc.getElementById("userlinksguest") != null) { ?: if (doc.getElementById("userlinksguest") != null) {
throw AuthRequiredException(source) throw AuthRequiredException(source)
} else { } else {
doc.parseFailed() doc.parseFailed()
} }
return username return username
} }
override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) {
super.onCreateConfig(keys) super.onCreateConfig(keys)
keys.add(userAgentKey) keys.add(userAgentKey)
keys.add(suspiciousContentKey) keys.add(suspiciousContentKey)
} }
override suspend fun getRelatedManga(seed: Manga): List<Manga> { override suspend fun getRelatedManga(seed: Manga): List<Manga> {
val query = seed.title val query = seed.title
return getListPage( return getListPage(
page = 0, page = 0,
order = defaultSortOrder, order = defaultSortOrder,
filter = MangaListFilter(query = query), filter = MangaListFilter(query = query),
) )
} }
private fun isAuthorized(domain: String): Boolean { private fun isAuthorized(domain: String): Boolean {
val cookies = context.cookieJar.getCookies(domain).mapToSet { x -> x.name } val cookies = context.cookieJar.getCookies(domain).mapToSet { x -> x.name }
return authCookies.all { it in cookies } return authCookies.all { it in cookies }
} }
private fun Element.parseRating(): Float { private fun Element.parseRating(): Float {
return runCatching { return runCatching {
val style = requireNotNull(attr("style")) val style = requireNotNull(attr("style"))
val (v1, v2) = ratingPattern.findAll(style).toList() val (v1, v2) = ratingPattern.findAll(style).toList()
var p1 = v1.groupValues.first().dropLast(2).toInt() var p1 = v1.groupValues.first().dropLast(2).toInt()
val p2 = v2.groupValues.first().dropLast(2).toInt() val p2 = v2.groupValues.first().dropLast(2).toInt()
if (p2 != -1) { if (p2 != -1) {
p1 += 8 p1 += 8
} }
(80 - p1) / 80f (80 - p1) / 80f
}.getOrDefault(RATING_UNKNOWN) }.getOrDefault(RATING_UNKNOWN)
} }
private fun String.cleanupTitle(): String { private fun String.cleanupTitle(): String {
return replace(titleCleanupPattern, "") return replace(titleCleanupPattern, "")
.replace(spacesCleanupPattern, "") .replace(spacesCleanupPattern, "")
} }
private fun Element.parseTags(): Set<MangaTag> { private fun Element.parseTags(): Set<MangaTag> {
fun Element.parseTag() = textOrNull()?.let { fun Element.parseTag() = textOrNull()?.let {
MangaTag(title = it.toTitleCase(Locale.ENGLISH), key = it, source = source) MangaTag(title = it.toTitleCase(Locale.ENGLISH), key = it, source = source)
} }
val result = ArraySet<MangaTag>() val result = ArraySet<MangaTag>()
for (prefix in TAG_PREFIXES) { for (prefix in TAG_PREFIXES) {
getElementsByAttributeValueStarting("id", "ta_$prefix").mapNotNullTo(result, Element::parseTag) getElementsByAttributeValueStarting("id", "ta_$prefix").mapNotNullTo(result, Element::parseTag)
getElementsByAttributeValueStarting("title", prefix).mapNotNullTo(result, Element::parseTag) getElementsByAttributeValueStarting("title", prefix).mapNotNullTo(result, Element::parseTag)
} }
return result return result
} }
private fun Element.extractPreview(): String? { private fun Element.extractPreview(): String? {
val bg = backgroundOrNull() ?: return null val bg = backgroundOrNull() ?: return null
return buildString { return buildString {
append(bg.url) append(bg.url)
append('#') append('#')
// rect: left,top,right,bottom // rect: left,top,right,bottom
append(bg.left) append(bg.left)
append(',') append(',')
append(bg.top) append(bg.top)
append(',') append(',')
append(bg.right) append(bg.right)
append(',') append(',')
append(bg.bottom) append(bg.bottom)
} }
} }
private fun getNextTimestamp(root: Element): Long { private fun getNextTimestamp(root: Element): Long {
return root.getElementById("unext") return root.getElementById("unext")
?.attrAsAbsoluteUrlOrNull("href") ?.attrAsAbsoluteUrlOrNull("href")
?.toHttpUrlOrNull() ?.toHttpUrlOrNull()
?.queryParameter("next") ?.queryParameter("next")
?.toLongOrNull() ?: 1 ?.toLongOrNull() ?: 1
} }
private fun MangaListFilter.toSearchQuery(): String? { private fun MangaListFilter.toSearchQuery(): String? {
if (isEmpty()) { if (isEmpty()) {
return null return null
} }
val joiner = StringUtil.StringJoiner(" ") val joiner = StringUtil.StringJoiner(" ")
if (!query.isNullOrEmpty()) { if (!query.isNullOrEmpty()) {
joiner.add(query) joiner.add(query)
} }
for (tag in tags) { for (tag in tags) {
if (tag.key.isNumeric()) { if (tag.key.isNumeric()) {
continue continue
} }
joiner.add("tag:\"") joiner.add("tag:\"")
joiner.append(tag.key) joiner.append(tag.key)
joiner.append("\"$") joiner.append("\"$")
} }
for (tag in tagsExclude) { for (tag in tagsExclude) {
if (tag.key.isNumeric()) { if (tag.key.isNumeric()) {
continue continue
} }
joiner.add("-tag:\"") joiner.add("-tag:\"")
joiner.append(tag.key) joiner.append(tag.key)
joiner.append("\"$") joiner.append("\"$")
} }
locale?.let { lc -> locale?.let { lc ->
joiner.add("language:\"") joiner.add("language:\"")
joiner.append(lc.toLanguagePath()) joiner.append(lc.toLanguagePath())
joiner.append("\"$") joiner.append("\"$")
} }
if (!author.isNullOrEmpty()) { if (!author.isNullOrEmpty()) {
joiner.add("artist:\"") joiner.add("artist:\"")
joiner.append(author) joiner.append(author)
joiner.append("\"$") joiner.append("\"$")
} }
return joiner.complete().nullIfEmpty() return joiner.complete().nullIfEmpty()
} }
private fun Collection<ContentType>.toFCats(): Int = fold(0) { acc, ct -> private fun Collection<ContentType>.toFCats(): Int = fold(0) { acc, ct ->
val cat: Int = when (ct) { val cat: Int = when (ct) {
ContentType.DOUJINSHI -> 2 ContentType.DOUJINSHI -> 2
ContentType.MANGA -> 4 ContentType.MANGA -> 4
ContentType.ARTIST_CG -> 8 ContentType.ARTIST_CG -> 8
ContentType.GAME_CG -> 16 ContentType.GAME_CG -> 16
ContentType.COMICS -> 512 ContentType.COMICS -> 512
ContentType.IMAGE_SET -> 32 ContentType.IMAGE_SET -> 32
else -> 449 // 1 or 64 or 128 or 256 else -> 449 // 1 or 64 or 128 or 256
} }
acc or cat acc or cat
} }
private fun checkAuth(): Boolean { private fun checkAuth(): Boolean {
val authorized = isAuthorized(DOMAIN_UNAUTHORIZED) val authorized = isAuthorized(DOMAIN_UNAUTHORIZED)
if (authorized) { if (authorized) {
if (!isAuthorized(DOMAIN_AUTHORIZED)) { if (!isAuthorized(DOMAIN_AUTHORIZED)) {
context.cookieJar.copyCookies( context.cookieJar.copyCookies(
DOMAIN_UNAUTHORIZED, DOMAIN_UNAUTHORIZED,
DOMAIN_AUTHORIZED, DOMAIN_AUTHORIZED,
authCookies, authCookies,
) )
context.cookieJar.insertCookies(DOMAIN_AUTHORIZED, "yay=louder") context.cookieJar.insertCookies(DOMAIN_AUTHORIZED, "yay=louder")
} }
return true return true
} }
return false return false
} }
} }

@ -19,6 +19,7 @@ import org.koitharu.kotatsu.parsers.model.MangaTag
import org.koitharu.kotatsu.parsers.model.RATING_UNKNOWN import org.koitharu.kotatsu.parsers.model.RATING_UNKNOWN
import org.koitharu.kotatsu.parsers.model.SortOrder import org.koitharu.kotatsu.parsers.model.SortOrder
import org.koitharu.kotatsu.parsers.util.attrAsRelativeUrl import org.koitharu.kotatsu.parsers.util.attrAsRelativeUrl
import org.koitharu.kotatsu.parsers.util.attrOrNull
import org.koitharu.kotatsu.parsers.util.generateUid import org.koitharu.kotatsu.parsers.util.generateUid
import org.koitharu.kotatsu.parsers.util.mapChapters import org.koitharu.kotatsu.parsers.util.mapChapters
import org.koitharu.kotatsu.parsers.util.mapNotNullToSet import org.koitharu.kotatsu.parsers.util.mapNotNullToSet
@ -28,6 +29,7 @@ import org.koitharu.kotatsu.parsers.util.parseFailed
import org.koitharu.kotatsu.parsers.util.parseHtml import org.koitharu.kotatsu.parsers.util.parseHtml
import org.koitharu.kotatsu.parsers.util.selectFirstOrThrow import org.koitharu.kotatsu.parsers.util.selectFirstOrThrow
import org.koitharu.kotatsu.parsers.util.src import org.koitharu.kotatsu.parsers.util.src
import org.koitharu.kotatsu.parsers.util.textOrNull
import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl
import org.koitharu.kotatsu.parsers.util.toTitleCase import org.koitharu.kotatsu.parsers.util.toTitleCase
import org.koitharu.kotatsu.parsers.util.urlEncoded import org.koitharu.kotatsu.parsers.util.urlEncoded
@ -36,232 +38,232 @@ import java.util.EnumSet
@MangaSourceParser("KDTSCANS", "KdtScans", "en") @MangaSourceParser("KDTSCANS", "KdtScans", "en")
internal class KdtScans(context: MangaLoaderContext) : internal class KdtScans(context: MangaLoaderContext) :
PagedMangaParser(context, MangaParserSource.KDTSCANS, 20) { PagedMangaParser(context, MangaParserSource.KDTSCANS, 20) {
override val configKeyDomain = ConfigKey.Domain("www.silentquill.net") override val configKeyDomain = ConfigKey.Domain("www.silentquill.net")
override val availableSortOrders: Set<SortOrder> = EnumSet.of( override val availableSortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.RELEVANCE, SortOrder.RELEVANCE,
SortOrder.UPDATED, SortOrder.UPDATED,
SortOrder.POPULARITY, SortOrder.POPULARITY,
SortOrder.NEWEST, SortOrder.NEWEST,
SortOrder.ALPHABETICAL, SortOrder.ALPHABETICAL,
SortOrder.ALPHABETICAL_DESC, SortOrder.ALPHABETICAL_DESC,
) )
override val filterCapabilities = MangaListFilterCapabilities( override val filterCapabilities = MangaListFilterCapabilities(
isSearchSupported = true, isSearchSupported = true,
isMultipleTagsSupported = true, isMultipleTagsSupported = true,
isTagsExclusionSupported = true, isTagsExclusionSupported = true,
) )
override suspend fun getFilterOptions(): MangaListFilterOptions { override suspend fun getFilterOptions(): MangaListFilterOptions {
return MangaListFilterOptions( return MangaListFilterOptions(
availableTags = fetchAvailableTags(), availableTags = fetchAvailableTags(),
availableStates = EnumSet.of( availableStates = EnumSet.of(
MangaState.ONGOING, MangaState.ONGOING,
MangaState.FINISHED, MangaState.FINISHED,
MangaState.PAUSED, MangaState.PAUSED,
), ),
availableContentTypes = EnumSet.of( availableContentTypes = EnumSet.of(
ContentType.MANGA, ContentType.MANGA,
ContentType.MANHWA, ContentType.MANHWA,
ContentType.MANHUA, ContentType.MANHUA,
ContentType.COMICS, ContentType.COMICS,
ContentType.NOVEL, ContentType.NOVEL,
), ),
) )
} }
override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> {
val url = buildString { val url = buildString {
append("https://$domain/manga/?page=${page}") append("https://$domain/manga/?page=${page}")
filter.query?.let { filter.query?.let {
append("&s=${it.urlEncoded()}") append("&s=${it.urlEncoded()}")
} }
val sortValue = when (order) { val sortValue = when (order) {
SortOrder.UPDATED -> "update" SortOrder.UPDATED -> "update"
SortOrder.POPULARITY -> "popular" SortOrder.POPULARITY -> "popular"
SortOrder.NEWEST -> "latest" SortOrder.NEWEST -> "latest"
SortOrder.ALPHABETICAL -> "title" SortOrder.ALPHABETICAL -> "title"
SortOrder.ALPHABETICAL_DESC -> "titlereverse" SortOrder.ALPHABETICAL_DESC -> "titlereverse"
else -> "" // Default/Relevance else -> "" // Default/Relevance
} }
if (sortValue.isNotEmpty()) { if (sortValue.isNotEmpty()) {
append("&order=$sortValue") append("&order=$sortValue")
} }
filter.tags.forEach { tag -> filter.tags.forEach { tag ->
append("&genre[]=${tag.key}") append("&genre[]=${tag.key}")
} }
filter.tagsExclude.forEach { tag -> filter.tagsExclude.forEach { tag ->
append("&genre[]=-${tag.key}") append("&genre[]=-${tag.key}")
} }
filter.states.oneOrThrowIfMany().let { state -> filter.states.oneOrThrowIfMany().let { state ->
val stateValue = when (state) { val stateValue = when (state) {
MangaState.ONGOING -> "ongoing" MangaState.ONGOING -> "ongoing"
MangaState.FINISHED -> "completed" MangaState.FINISHED -> "completed"
MangaState.PAUSED -> "hiatus" MangaState.PAUSED -> "hiatus"
else -> "" else -> ""
} }
if (stateValue.isNotEmpty()) { if (stateValue.isNotEmpty()) {
append("&status=$stateValue") append("&status=$stateValue")
} }
} }
filter.types.oneOrThrowIfMany()?.let { type -> filter.types.oneOrThrowIfMany()?.let { type ->
val typeValue = when (type) { val typeValue = when (type) {
ContentType.MANGA -> "manga" ContentType.MANGA -> "manga"
ContentType.MANHWA -> "manhwa" ContentType.MANHWA -> "manhwa"
ContentType.MANHUA -> "manhua" ContentType.MANHUA -> "manhua"
ContentType.COMICS -> "comic" ContentType.COMICS -> "comic"
ContentType.NOVEL -> "novel" ContentType.NOVEL -> "novel"
else -> "" else -> ""
} }
if (typeValue.isNotEmpty()) { if (typeValue.isNotEmpty()) {
append("&type=$typeValue") append("&type=$typeValue")
} }
} }
} }
val doc = webClient.httpGet(url).parseHtml() val doc = webClient.httpGet(url).parseHtml()
return parseMangaList(doc) return parseMangaList(doc)
} }
private fun parseMangaList(doc: Document): List<Manga> { private fun parseMangaList(doc: Document): List<Manga> {
val elements = doc.select("div.listupd div.bs") val elements = doc.select("div.listupd div.bs")
if (elements.isEmpty()) { if (elements.isEmpty()) {
return emptyList() return emptyList()
} }
return elements.map { div -> return elements.map { div ->
val a = div.selectFirstOrThrow("a") val a = div.selectFirstOrThrow("a")
val href = a.attrAsRelativeUrl("href") val href = a.attrAsRelativeUrl("href")
val img = div.selectFirst("img") val img = div.selectFirst("img")
val title = a.attr("title").ifEmpty { val title = a.attr("title").ifEmpty {
div.selectFirst(".tt")?.text().orEmpty() div.selectFirst(".tt")?.text().orEmpty()
} }
val rating = div.selectFirst(".numscore")?.text()?.toFloatOrNull()?.div(10f) ?: RATING_UNKNOWN val rating = div.selectFirst(".numscore")?.text()?.toFloatOrNull()?.div(10f) ?: RATING_UNKNOWN
Manga( Manga(
id = generateUid(href), id = generateUid(href),
url = href, url = href,
publicUrl = href.toAbsoluteUrl(domain), publicUrl = href.toAbsoluteUrl(domain),
coverUrl = img?.src(), coverUrl = img?.src(),
title = title, title = title,
altTitles = emptySet(), altTitles = emptySet(),
rating = rating, rating = rating,
tags = emptySet(), tags = emptySet(),
authors = emptySet(), authors = emptySet(),
state = parseStatus(div.selectFirst(".status")?.text().orEmpty()), state = parseStatus(div.selectFirst(".status")?.text().orEmpty()),
source = source, source = source,
contentRating = if (isNsfwSource) ContentRating.ADULT else null, contentRating = if (isNsfwSource) ContentRating.ADULT else null,
) )
} }
} }
override suspend fun getDetails(manga: Manga): Manga { override suspend fun getDetails(manga: Manga): Manga {
val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml()
val infoElement = val infoElement =
doc.selectFirst(".main-info, .postbody") ?: doc.parseFailed("Cannot find manga details element") doc.selectFirst(".main-info, .postbody") ?: doc.parseFailed("Cannot find manga details element")
val statusText = val statusText =
infoElement.selectFirst(".tsinfo .imptdt:contains(Status) i, .infotable tr:contains(Status) td:last-child") infoElement.selectFirst(".tsinfo .imptdt:contains(Status) i, .infotable tr:contains(Status) td:last-child")
?.text() ?.text()
val chapters = doc.select("#chapterlist li").mapChapters(reversed = true) { i, li -> val chapters = doc.select("#chapterlist li").mapChapters(reversed = true) { i, li ->
val a = li.selectFirstOrThrow("a") val a = li.selectFirstOrThrow("a")
val href = a.attrAsRelativeUrl("href") val href = a.attrAsRelativeUrl("href")
MangaChapter( MangaChapter(
id = generateUid(href), id = generateUid(href),
url = href, url = href,
title = a.selectFirst(".chapternum")?.text() ?: a.text(), title = a.selectFirst(".chapternum")?.text() ?: a.text(),
number = i + 1f, number = i + 1f,
uploadDate = parseChapterDate(li.selectFirst(".chapterdate")?.text()), uploadDate = parseChapterDate(li.selectFirst(".chapterdate")?.text()),
source = source, source = source,
volume = 0, volume = 0,
scanlator = null, scanlator = null,
branch = null, branch = null,
) )
} }
val genres = infoElement.select(".mgen a, .seriestugenre a").mapToSet { a -> val genres = infoElement.select(".mgen a, .seriestugenre a").mapToSet { a ->
MangaTag( MangaTag(
key = a.attr("href").substringAfterLast("/").removeSuffix("/"), key = a.attr("href").substringAfterLast("/").removeSuffix("/"),
title = a.text(), title = a.text(),
source = source, source = source,
) )
} }
val typeTag = infoElement.selectFirst(".tsinfo .imptdt:contains(Type) a")?.text()?.let { typeText -> val typeTag = infoElement.selectFirst(".tsinfo .imptdt:contains(Type) a")?.text()?.let { typeText ->
MangaTag( MangaTag(
key = typeText.lowercase(), key = typeText.lowercase(),
title = typeText.trim(), title = typeText.trim(),
source = source, source = source,
) )
} }
val allTags = genres.toMutableSet() val allTags = genres.toMutableSet()
typeTag?.let { allTags.add(it) } typeTag?.let { allTags.add(it) }
return manga.copy( return manga.copy(
title = infoElement.selectFirst("h1.entry-title")?.text() ?: manga.title, title = infoElement.selectFirst("h1.entry-title")?.text() ?: manga.title,
authors = infoElement.select(".tsinfo .imptdt:contains(Author) i, .infotable tr:contains(Author) td:last-child") authors = infoElement.select(".tsinfo .imptdt:contains(Author) i, .infotable tr:contains(Author) td:last-child")
.mapToSet { it.text() }, .mapToSet { it.text() },
description = infoElement.select(".desc, .entry-content[itemprop=description]") description = infoElement.select(".desc, .entry-content[itemprop=description]")
.joinToString("\n") { it.text() }, .joinToString("\n") { it.text() },
state = parseStatus(statusText.orEmpty()), state = parseStatus(statusText.orEmpty()),
tags = allTags, tags = allTags,
chapters = chapters, chapters = chapters,
) )
} }
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml()
return doc.select("#readerarea img").map { img -> return doc.select("#readerarea img").map { img ->
val url = img.attr("data-src").ifEmpty { img.src().orEmpty() } val url = img.attr("data-src").ifEmpty { img.src().orEmpty() }
MangaPage( MangaPage(
id = generateUid(url), id = generateUid(url),
url = url, url = url,
preview = null, preview = null,
source = source, source = source,
) )
} }
} }
private fun parseStatus(status: String): MangaState? { private fun parseStatus(status: String): MangaState? {
return when { return when {
status.contains("ongoing", ignoreCase = true) -> MangaState.ONGOING status.contains("ongoing", ignoreCase = true) -> MangaState.ONGOING
status.contains("completed", ignoreCase = true) -> MangaState.FINISHED status.contains("completed", ignoreCase = true) -> MangaState.FINISHED
status.contains("hiatus", ignoreCase = true) -> MangaState.PAUSED status.contains("hiatus", ignoreCase = true) -> MangaState.PAUSED
status.contains("dropped", ignoreCase = true) -> MangaState.ABANDONED status.contains("dropped", ignoreCase = true) -> MangaState.ABANDONED
status.contains("canceled", ignoreCase = true) -> MangaState.ABANDONED status.contains("canceled", ignoreCase = true) -> MangaState.ABANDONED
else -> null else -> null
} }
} }
private fun parseChapterDate(date: String?): Long { private fun parseChapterDate(date: String?): Long {
return try { return try {
SimpleDateFormat("MMMM dd, yyyy", sourceLocale).parse(date?.trim()).time SimpleDateFormat("MMMM dd, yyyy", sourceLocale).parse(date?.trim()).time
} catch (_: Exception) { } catch (_: Exception) {
0L 0L
} }
} }
private suspend fun fetchAvailableTags(): Set<MangaTag> { private suspend fun fetchAvailableTags(): Set<MangaTag> {
val doc = webClient.httpGet("https://$domain/manga/").parseHtml() val doc = webClient.httpGet("https://$domain/manga/").parseHtml()
return doc.select("ul.genrez li").mapNotNullToSet { li -> return doc.select("ul.genrez li").mapNotNullToSet { li ->
val key = li.selectFirst("input").attr("value") ?: return@mapNotNullToSet null val key = li.selectFirst("input")?.attrOrNull("value") ?: return@mapNotNullToSet null
val title = li.selectFirst("label").text().toTitleCase() val title = li.selectFirst("label")?.textOrNull()?.toTitleCase() ?: return@mapNotNullToSet null
MangaTag( MangaTag(
key = key, key = key,
title = title, title = title,
source = source, source = source,
) )
} }
} }
} }

@ -1,174 +1,184 @@
package org.koitharu.kotatsu.parsers.site.es package org.koitharu.kotatsu.parsers.site.es
import org.koitharu.kotatsu.parsers.Broken
import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.config.ConfigKey import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.core.PagedMangaParser import org.koitharu.kotatsu.parsers.core.PagedMangaParser
import org.koitharu.kotatsu.parsers.model.* import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.* import org.koitharu.kotatsu.parsers.util.*
import org.koitharu.kotatsu.parsers.Broken
import java.util.* import java.util.*
@Broken // Website closed @Broken // Website closed
@MangaSourceParser("DRAGONTRANSLATION", "Dragon Translation", "es") @MangaSourceParser("DRAGONTRANSLATION", "Dragon Translation", "es")
internal class DragonTranslationParser(context: MangaLoaderContext) : PagedMangaParser(context, MangaParserSource.DRAGONTRANSLATION, 30) { internal class DragonTranslationParser(context: MangaLoaderContext) :
PagedMangaParser(context, MangaParserSource.DRAGONTRANSLATION, 30) {
override val configKeyDomain = ConfigKey.Domain("dragontranslation.net")
override val configKeyDomain = ConfigKey.Domain("dragontranslation.net")
override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) {
super.onCreateConfig(keys) override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) {
keys.add(userAgentKey) super.onCreateConfig(keys)
} keys.add(userAgentKey)
}
override fun getRequestHeaders() = super.getRequestHeaders().newBuilder()
.add("referer", "no-referrer") override fun getRequestHeaders() = super.getRequestHeaders().newBuilder()
.build() .add("referer", "no-referrer")
.build()
override val availableSortOrders: Set<SortOrder> = EnumSet.of(SortOrder.UPDATED)
override val availableSortOrders: Set<SortOrder> = EnumSet.of(SortOrder.UPDATED)
override val filterCapabilities: MangaListFilterCapabilities
get() = MangaListFilterCapabilities( override val filterCapabilities: MangaListFilterCapabilities
isSearchSupported = true, get() = MangaListFilterCapabilities(
) isSearchSupported = true,
)
override suspend fun getFilterOptions() = MangaListFilterOptions(
availableTags = emptySet(), // cant find any URLs for fetch tags override suspend fun getFilterOptions() = MangaListFilterOptions(
availableContentTypes = EnumSet.of(ContentType.MANGA, ContentType.MANHWA, ContentType.MANHUA), availableTags = emptySet(), // cant find any URLs for fetch tags
) availableContentTypes = EnumSet.of(ContentType.MANGA, ContentType.MANHWA, ContentType.MANHUA),
)
override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> {
val url = buildString { override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> {
append("https://") val url = buildString {
append(domain) append("https://")
when { append(domain)
!filter.query.isNullOrEmpty() -> { when {
append("/mangas?buscar=") !filter.query.isNullOrEmpty() -> {
append(filter.query.urlEncoded()) append("/mangas?buscar=")
append("&page=") append(filter.query.urlEncoded())
append(page.toString()) append("&page=")
} append(page.toString())
}
else -> {
append("/mangas?page=") else -> {
append(page.toString()) append("/mangas?page=")
append(page.toString())
val tag = filter.tags.oneOrThrowIfMany()
if (filter.tags.isNotEmpty()) { val tag = filter.tags.oneOrThrowIfMany()
append("&tag=") if (filter.tags.isNotEmpty()) {
append(tag?.key.orEmpty()) append("&tag=")
} append(tag?.key.orEmpty())
}
if (filter.types.isNotEmpty()) {
append("&type=") if (filter.types.isNotEmpty()) {
when (filter.types.oneOrThrowIfMany()) { append("&type=")
ContentType.MANGA -> "manga" append(
ContentType.MANHWA -> "manhwa" when (filter.types.oneOrThrowIfMany()) {
ContentType.MANHUA -> "manhua" ContentType.MANGA -> "manga"
else -> "" ContentType.MANHWA -> "manhwa"
} ContentType.MANHUA -> "manhua"
} else -> ""
} },
} )
} }
}
val doc = webClient.httpGet(url).parseHtml() }
val row = doc.select("div.row.gy-3").firstOrNull() ?: return emptyList() }
return row.select("article.position-relative.card").mapNotNull { div ->
val href = div.selectFirst("a.lanzador")?.attrAsRelativeUrlOrNull("href") ?: return@mapNotNull null val doc = webClient.httpGet(url).parseHtml()
val coverUrl = div.selectFirst("img.card-img-top.wp-post-image.lazy.loaded")?.src().orEmpty() val row = doc.select("div.row.gy-3").firstOrNull() ?: return emptyList()
Manga( return row.select("article.position-relative.card").mapNotNull { div ->
id = generateUid(href), val href = div.selectFirst("a.lanzador")?.attrAsRelativeUrlOrNull("href") ?: return@mapNotNull null
url = href, val coverUrl = div.selectFirst("img.card-img-top.wp-post-image.lazy.loaded")?.src().orEmpty()
publicUrl = href, Manga(
coverUrl = coverUrl, id = generateUid(href),
title = div.selectFirst("h2.card-title.fs-6.entry-title").text(), url = href,
altTitles = emptySet(), publicUrl = href,
rating = RATING_UNKNOWN, coverUrl = coverUrl,
tags = emptySet(), title = div.selectFirst("h2.card-title.fs-6.entry-title")?.text().orEmpty(),
authors = emptySet(), altTitles = emptySet(),
state = null, rating = RATING_UNKNOWN,
source = source, tags = emptySet(),
contentRating = if (isNsfwSource) ContentRating.ADULT else null, authors = emptySet(),
) state = null,
} source = source,
} contentRating = if (isNsfwSource) ContentRating.ADULT else null,
)
override suspend fun getDetails(manga: Manga): Manga { }
val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() }
val statusText = doc.selectFirst("p:contains(Status:)")?.text()
val status = when { override suspend fun getDetails(manga: Manga): Manga {
statusText?.contains("publishing", ignoreCase = true) == true -> MangaState.ONGOING val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml()
else -> null val statusText = doc.selectFirst("p:contains(Status:)")?.text()
} val status = when {
statusText?.contains("publishing", ignoreCase = true) == true -> MangaState.ONGOING
val chapterElements = doc.select("ul.list-group a") else -> null
val totalChapters = chapterElements.size }
val chapters = chapterElements.mapIndexed { index, a -> val chapterElements = doc.select("ul.list-group a")
val href = a.attrAsRelativeUrl("href") val totalChapters = chapterElements.size
val title = a.text()
MangaChapter( val chapters = chapterElements.mapIndexed { index, a ->
id = generateUid(href), val href = a.attrAsRelativeUrl("href")
title = title, val title = a.text()
number = totalChapters - index.toFloat(), MangaChapter(
volume = 0, id = generateUid(href),
url = href, title = title,
scanlator = null, number = totalChapters - index.toFloat(),
uploadDate = parseDate(a.selectFirst("span")?.text()), volume = 0,
branch = null, url = href,
source = source, scanlator = null,
) uploadDate = parseDate(a.selectFirst("span")?.text()),
} branch = null,
source = source,
return manga.copy( )
state = status, }
chapters = chapters.reversed(),
) return manga.copy(
} state = status,
chapters = chapters.reversed(),
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { )
val fullUrl = chapter.url.toAbsoluteUrl(domain) }
val doc = webClient.httpGet(fullUrl).parseHtml()
return doc.select("div#chapter_imgs img").map { img -> override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val url = img.attr("src") val fullUrl = chapter.url.toAbsoluteUrl(domain)
MangaPage( val doc = webClient.httpGet(fullUrl).parseHtml()
id = generateUid(url), return doc.select("div#chapter_imgs img").map { img ->
url = url, val url = img.attr("src")
preview = null, MangaPage(
source = source, id = generateUid(url),
) url = url,
} preview = null,
} source = source,
)
private fun parseDate(dateText: String?): Long { }
if (dateText == null) return 0 }
val number = dateText.filter { it.isDigit() }.toIntOrNull() ?: return 0 private fun parseDate(dateText: String?): Long {
val now = System.currentTimeMillis() if (dateText == null) return 0
return when { val number = dateText.filter { it.isDigit() }.toIntOrNull() ?: return 0
dateText.contains("minutos") -> { val now = System.currentTimeMillis()
now - (number * 60 * 1000L)
} return when {
dateText.contains("horas") -> { dateText.contains("minutos") -> {
now - (number * 60 * 60 * 1000L) now - (number * 60 * 1000L)
} }
dateText.contains("días") -> {
now - (number * 24 * 60 * 60 * 1000L) dateText.contains("horas") -> {
} now - (number * 60 * 60 * 1000L)
dateText.contains("día") -> { }
now - (number * 24 * 60 * 60 * 1000L)
} dateText.contains("días") -> {
dateText.contains("semanas") -> { now - (number * 24 * 60 * 60 * 1000L)
now - (number * 7 * 24 * 60 * 60 * 1000L) }
}
dateText.contains("meses") -> { dateText.contains("día") -> {
now - (number * 30 * 24 * 60 * 60 * 1000L) now - (number * 24 * 60 * 60 * 1000L)
} }
dateText.contains("años") -> {
now - (number * 365 * 24 * 60 * 60 * 1000L) dateText.contains("semanas") -> {
} now - (number * 7 * 24 * 60 * 60 * 1000L)
else -> 0L }
}
} dateText.contains("meses") -> {
now - (number * 30 * 24 * 60 * 60 * 1000L)
}
dateText.contains("años") -> {
now - (number * 365 * 24 * 60 * 60 * 1000L)
}
else -> 0L
}
}
} }

@ -20,143 +20,143 @@ import java.util.zip.ZipInputStream
@Broken // Not dead but changed template @Broken // Not dead but changed template
@MangaSourceParser("RANDOMSCANS", "LuratoonScan", "pt") @MangaSourceParser("RANDOMSCANS", "LuratoonScan", "pt")
internal class LuratoonScansParser(context: MangaLoaderContext) : internal class LuratoonScansParser(context: MangaLoaderContext) :
SinglePageMangaParser(context, MangaParserSource.RANDOMSCANS), SinglePageMangaParser(context, MangaParserSource.RANDOMSCANS),
Interceptor { Interceptor {
override val configKeyDomain = ConfigKey.Domain("luratoons.net") override val configKeyDomain = ConfigKey.Domain("luratoons.net")
override fun getRequestHeaders(): Headers = Headers.Builder().add("User-Agent", config[userAgentKey]).build() override fun getRequestHeaders(): Headers = Headers.Builder().add("User-Agent", config[userAgentKey]).build()
override val availableSortOrders = setOf(SortOrder.ALPHABETICAL) override val availableSortOrders = setOf(SortOrder.ALPHABETICAL)
override val filterCapabilities: MangaListFilterCapabilities override val filterCapabilities: MangaListFilterCapabilities
get() = MangaListFilterCapabilities() get() = MangaListFilterCapabilities()
override suspend fun getFilterOptions() = MangaListFilterOptions() override suspend fun getFilterOptions() = MangaListFilterOptions()
override suspend fun getList(order: SortOrder, filter: MangaListFilter): List<Manga> { override suspend fun getList(order: SortOrder, filter: MangaListFilter): List<Manga> {
require(filter.query.isNullOrEmpty()) { ErrorMessages.SEARCH_NOT_SUPPORTED } require(filter.query.isNullOrEmpty()) { ErrorMessages.SEARCH_NOT_SUPPORTED }
val url = urlBuilder() val url = urlBuilder()
val tag = filter.tags.oneOrThrowIfMany() val tag = filter.tags.oneOrThrowIfMany()
if (tag == null) { if (tag == null) {
url.addPathSegment("todas-as-obras") url.addPathSegment("todas-as-obras")
} else { } else {
url.addPathSegment("pesquisar").addQueryParameter("category", tag.key) url.addPathSegment("pesquisar").addQueryParameter("category", tag.key)
} }
val doc = webClient.httpGet(url.build()).parseHtml() val doc = webClient.httpGet(url.build()).parseHtml()
return doc.selectFirstOrThrow(".todas__as__obras").select(".comics__all__box").map { div -> return doc.selectFirstOrThrow(".todas__as__obras").select(".comics__all__box").map { div ->
val a = div.selectFirstOrThrow("a") val a = div.selectFirstOrThrow("a")
val href = a.attrAsRelativeUrl("href") val href = a.attrAsRelativeUrl("href")
Manga( Manga(
id = generateUid(href), id = generateUid(href),
url = href, url = href,
publicUrl = href.toAbsoluteUrl(div.host ?: domain), publicUrl = href.toAbsoluteUrl(div.host ?: domain),
coverUrl = div.selectFirst("img")?.src().orEmpty(), coverUrl = div.selectFirst("img")?.src().orEmpty(),
title = div.text(), title = div.text(),
altTitles = emptySet(), altTitles = emptySet(),
rating = RATING_UNKNOWN, rating = RATING_UNKNOWN,
tags = emptySet(), tags = emptySet(),
authors = emptySet(), authors = emptySet(),
state = null, state = null,
source = source, source = source,
contentRating = null, contentRating = null,
) )
} }
} }
override suspend fun getDetails(manga: Manga): Manga { override suspend fun getDetails(manga: Manga): Manga {
val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml().body() val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml().body()
val summaryContainer = doc.selectFirstOrThrow(".sumario__container") val summaryContainer = doc.selectFirstOrThrow(".sumario__container")
// 1 de Maio de 2024 às 20:15 // 1 de Maio de 2024 às 20:15
val dateFormat = SimpleDateFormat("dd 'de' MMM 'de' YYYY 'às' HH:mm", sourceLocale) val dateFormat = SimpleDateFormat("dd 'de' MMM 'de' YYYY 'às' HH:mm", sourceLocale)
val author = summaryContainer.getElementsContainingOwnText("Autor(es)").firstOrNull() val author = summaryContainer.getElementsContainingOwnText("Autor(es)").firstOrNull()
?.nextElementSibling()?.textOrNull() ?.nextElementSibling()?.textOrNull()
return manga.copy( return manga.copy(
title = doc.selectFirst("h1.desc__titulo__comic")?.textOrNull() ?: manga.title, title = doc.selectFirst("h1.desc__titulo__comic")?.textOrNull() ?: manga.title,
altTitles = setOfNotNull( altTitles = setOfNotNull(
summaryContainer.getElementsContainingOwnText("Alternativo").firstOrNull() summaryContainer.getElementsContainingOwnText("Alternativo").firstOrNull()
?.nextElementSibling()?.textOrNull(), ?.nextElementSibling()?.textOrNull(),
), ),
tags = summaryContainer.getElementsByAttributeValueContaining("href", "?category=").mapToSet { tags = summaryContainer.getElementsByAttributeValueContaining("href", "?category=").mapToSet {
MangaTag( MangaTag(
title = it.text().toTitleCase(sourceLocale), title = it.text().toTitleCase(sourceLocale),
key = it.attr("href").substringAfterLast('='), key = it.attr("href").substringAfterLast('='),
source = source, source = source,
) )
}, },
state = when (summaryContainer.getElementsContainingOwnText("Status").firstOrNull() state = when (summaryContainer.getElementsContainingOwnText("Status").firstOrNull()
?.nextElementSibling()?.text()?.lowercase()) { ?.nextElementSibling()?.text()?.lowercase()) {
"em lançamento" -> MangaState.ONGOING "em lançamento" -> MangaState.ONGOING
"hiato" -> MangaState.PAUSED "hiato" -> MangaState.PAUSED
"finalizado" -> MangaState.FINISHED "finalizado" -> MangaState.FINISHED
else -> null else -> null
}, },
authors = setOfNotNull(author), authors = setOfNotNull(author),
largeCoverUrl = doc.selectFirst("img.sumario__img")?.attrAsAbsoluteUrlOrNull("src"), largeCoverUrl = doc.selectFirst("img.sumario__img")?.attrAsAbsoluteUrlOrNull("src"),
description = summaryContainer.selectFirst(".sumario__sinopse__texto")?.html(), description = summaryContainer.selectFirst(".sumario__sinopse__texto")?.html(),
chapters = doc.selectFirstOrThrow("ul.capitulos__lista") chapters = doc.selectFirstOrThrow("ul.capitulos__lista")
.select("li") .select("li")
.mapChapters(reversed = true) { _, li -> .mapChapters(reversed = true) { _, li ->
val href = li.parent()?.attrAsRelativeUrlOrNull("href") ?: return@mapChapters null val href = li.parent()?.attrAsRelativeUrlOrNull("href") ?: return@mapChapters null
val span = li.selectFirstOrThrow(".numero__capitulo") val span = li.selectFirstOrThrow(".numero__capitulo")
MangaChapter( MangaChapter(
id = generateUid(href), id = generateUid(href),
title = span.text(), title = span.text(),
number = 0.0f, number = 0.0f,
volume = 0, volume = 0,
url = href, url = href,
scanlator = null, scanlator = null,
uploadDate = dateFormat.parseSafe(span.nextElementSibling()?.text()), uploadDate = dateFormat.parseSafe(span.nextElementSibling()?.text()),
branch = null, branch = null,
source = source, source = source,
) )
}, },
) )
} }
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml()
val regex = Regex("const\\s+urls\\s*=\\s*(\\[.*])") val regex = Regex("const\\s+urls\\s*=\\s*(\\[.*])")
val urls = doc.select("script").firstNotNullOf { val urls = doc.select("script").firstNotNullOf {
regex.find(it.data())?.groupValues?.getOrNull(1) regex.find(it.data())?.groupValues?.getOrNull(1)
} }
val ja = JSONArray(urls) val ja = JSONArray(urls)
return (0 until ja.length()).map { i -> return (0 until ja.length()).map { i ->
val url = ja.getString(i) val url = ja.getString(i)
MangaPage( MangaPage(
id = generateUid(url), id = generateUid(url),
url = url, url = url,
preview = null, preview = null,
source = source, source = source,
) )
} }
} }
override fun intercept(chain: Interceptor.Chain): Response { override fun intercept(chain: Interceptor.Chain): Response {
val response = chain.proceed(chain.request()) val response = chain.proceed(chain.request())
if (response.mimeType == "application/octet-stream") { if (response.mimeType == "application/octet-stream") {
val (bytes, name) = response.use { resp -> val (bytes, name) = response.use { resp ->
ZipInputStream(resp.requireBody().byteStream()).use { ZipInputStream(resp.body.byteStream()).use {
val entry = it.nextEntry val entry = it.nextEntry
it.readBytes() to entry?.name it.readBytes() to entry?.name
} }
} }
val type = if (name?.endsWith(".avif", ignoreCase = true) == true) { val type = if (name?.endsWith(".avif", ignoreCase = true) == true) {
"image/avif" "image/avif"
} else { } else {
"image/*" "image/*"
}.toMediaTypeOrNull() }.toMediaTypeOrNull()
return response.newBuilder() return response.newBuilder()
.setHeader("Content-Type", type?.toString()) .setHeader("Content-Type", type?.toString())
.body(bytes.toResponseBody(type)) .body(bytes.toResponseBody(type))
.build() .build()
} else { } else {
return response return response
} }
} }
override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) {
super.onCreateConfig(keys) super.onCreateConfig(keys)
keys.add(userAgentKey) keys.add(userAgentKey)
} }
} }

@ -4,67 +4,52 @@ package org.koitharu.kotatsu.parsers.util
import kotlinx.coroutines.suspendCancellableCoroutine import kotlinx.coroutines.suspendCancellableCoroutine
import okhttp3.* import okhttp3.*
import okhttp3.internal.toLongOrDefault
import kotlin.contracts.InvocationKind import kotlin.contracts.InvocationKind
import kotlin.contracts.contract import kotlin.contracts.contract
public suspend fun Call.await(): Response = suspendCancellableCoroutine { continuation -> public suspend fun Call.await(): Response = suspendCancellableCoroutine { continuation ->
val callback = ContinuationCallCallback(this, continuation) val callback = ContinuationCallCallback(this, continuation)
enqueue(callback) enqueue(callback)
continuation.invokeOnCancellation(callback) continuation.invokeOnCancellation(callback)
} }
public val Response.mimeType: String? public val Response.mimeType: String?
get() = header("content-type")?.substringBefore(';')?.trim()?.nullIfEmpty()?.lowercase() get() = header("content-type")?.substringBefore(';')?.trim()?.nullIfEmpty()?.lowercase()
public val HttpUrl.isHttpOrHttps: Boolean public val HttpUrl.isHttpOrHttps: Boolean
get() = scheme.equals("https", ignoreCase = true) || scheme.equals("http", ignoreCase = true) get() = scheme.equals("https", ignoreCase = true) || scheme.equals("http", ignoreCase = true)
public fun Headers.Builder.mergeWith(other: Headers, replaceExisting: Boolean): Headers.Builder { public fun Headers.Builder.mergeWith(other: Headers, replaceExisting: Boolean): Headers.Builder {
for ((name, value) in other) { for ((name, value) in other) {
if (replaceExisting || this[name] == null) { if (replaceExisting || this[name] == null) {
this[name] = value this[name] = value
} }
} }
return this return this
} }
public fun Response.copy(): Response = newBuilder() public fun Response.copy(): Response = newBuilder()
.body(peekBody(Long.MAX_VALUE)) .body(peekBody(Long.MAX_VALUE))
.build() .build()
public fun Response.Builder.setHeader(name: String, value: String?): Response.Builder = if (value == null) { public fun Response.Builder.setHeader(name: String, value: String?): Response.Builder = if (value == null) {
removeHeader(name) removeHeader(name)
} else { } else {
header(name, value) header(name, value)
} }
public inline fun Response.map(mapper: (ResponseBody) -> ResponseBody): Response { public inline fun Response.map(mapper: (ResponseBody) -> ResponseBody): Response {
contract { contract {
callsInPlace(mapper, InvocationKind.AT_MOST_ONCE) callsInPlace(mapper, InvocationKind.AT_MOST_ONCE)
} }
return body?.use { responseBody -> return body.use { responseBody ->
newBuilder() newBuilder()
.body(mapper(responseBody)) .body(mapper(responseBody))
.build() .build()
} ?: this }
} }
public fun Cookie.newBuilder(): Cookie.Builder = Cookie.Builder().also { c -> public fun Response.headersContentLength(
c.name(name) defaultValue: Long = -1,
c.value(value) ): Long = headers["Content-Length"]?.toLongOrDefault(defaultValue) ?: defaultValue
if (persistent) {
c.expiresAt(expiresAt)
}
if (hostOnly) {
c.hostOnlyDomain(domain)
} else {
c.domain(domain)
}
c.path(path)
if (secure) {
c.secure()
}
if (httpOnly) {
c.httpOnly()
}
}

@ -8,7 +8,6 @@ import org.json.JSONArray
import org.json.JSONObject import org.json.JSONObject
import org.jsoup.Jsoup import org.jsoup.Jsoup
import org.jsoup.nodes.Document import org.jsoup.nodes.Document
import org.koitharu.kotatsu.parsers.ErrorMessages
import org.koitharu.kotatsu.parsers.InternalParsersApi import org.koitharu.kotatsu.parsers.InternalParsersApi
import java.text.DateFormat import java.text.DateFormat
@ -22,9 +21,9 @@ internal const val SCHEME_HTTPS = "https"
*/ */
// TODO suspend // TODO suspend
public fun Response.parseHtml(): Document = use { response -> public fun Response.parseHtml(): Document = use { response ->
val body = response.requireBody() val body = response.body
val charset = body.contentType()?.charset()?.name() val charset = body.contentType()?.charset()?.name()
Jsoup.parse(body.byteStream(), charset, response.request.url.toString()) Jsoup.parse(body.byteStream(), charset, response.request.url.toString())
} }
/** /**
@ -33,7 +32,7 @@ public fun Response.parseHtml(): Document = use { response ->
* @see [parseHtml] * @see [parseHtml]
*/ */
public fun Response.parseJson(): JSONObject = use { response -> public fun Response.parseJson(): JSONObject = use { response ->
JSONObject(response.requireBody().string()) JSONObject(response.body.string())
} }
/** /**
@ -42,15 +41,15 @@ public fun Response.parseJson(): JSONObject = use { response ->
* @see [parseHtml] * @see [parseHtml]
*/ */
public fun Response.parseJsonArray(): JSONArray = use { response -> public fun Response.parseJsonArray(): JSONArray = use { response ->
JSONArray(response.requireBody().string()) JSONArray(response.body.string())
} }
public fun Response.parseRaw(): String = use { response -> public fun Response.parseRaw(): String = use { response ->
response.requireBody().string() response.body.string()
} }
public fun Response.parseBytes(): ByteArray = use { response -> public fun Response.parseBytes(): ByteArray = use { response ->
response.requireBody().bytes() response.body.bytes()
} }
/** /**
@ -58,10 +57,10 @@ public fun Response.parseBytes(): ByteArray = use { response ->
* @return an url relative to the [domain] or absolute, if domain is mismatching * @return an url relative to the [domain] or absolute, if domain is mismatching
*/ */
public fun String.toRelativeUrl(domain: String): String { public fun String.toRelativeUrl(domain: String): String {
if (isEmpty() || startsWith("/")) { if (isEmpty() || startsWith("/")) {
return this return this
} }
return replace(Regex("^[^/]{2,6}://${Regex.escape(domain)}+/", RegexOption.IGNORE_CASE), "/") return replace(Regex("^[^/]{2,6}://${Regex.escape(domain)}+/", RegexOption.IGNORE_CASE), "/")
} }
/** /**
@ -69,36 +68,35 @@ public fun String.toRelativeUrl(domain: String): String {
* @return an absolute url with [domain] if this is relative * @return an absolute url with [domain] if this is relative
*/ */
public fun String.toAbsoluteUrl(domain: String): String = when { public fun String.toAbsoluteUrl(domain: String): String = when {
startsWith("//") -> "$SCHEME_HTTPS:$this" startsWith("//") -> "$SCHEME_HTTPS:$this"
startsWith('/') -> "$SCHEME_HTTPS://$domain$this" startsWith('/') -> "$SCHEME_HTTPS://$domain$this"
REGEX_SCHEME_PREFIX.containsMatchIn(this) -> this REGEX_SCHEME_PREFIX.containsMatchIn(this) -> this
else -> "$SCHEME_HTTPS://$domain/$this" else -> "$SCHEME_HTTPS://$domain/$this"
} }
public fun concatUrl(host: String, path: String): String { public fun concatUrl(host: String, path: String): String {
val hostWithSlash = host.endsWith('/') val hostWithSlash = host.endsWith('/')
val pathWithSlash = path.startsWith('/') val pathWithSlash = path.startsWith('/')
val hostWithScheme = if (host.startsWith("//")) "https:$host" else host val hostWithScheme = if (host.startsWith("//")) "https:$host" else host
return when { return when {
hostWithSlash && pathWithSlash -> hostWithScheme + path.drop(1) hostWithSlash && pathWithSlash -> hostWithScheme + path.drop(1)
!hostWithSlash && !pathWithSlash -> "$hostWithScheme/$path" !hostWithSlash && !pathWithSlash -> "$hostWithScheme/$path"
else -> hostWithScheme + path else -> hostWithScheme + path
} }
} }
@InternalParsersApi @InternalParsersApi
public fun DateFormat.parseSafe(str: String?): Long = if (str.isNullOrEmpty()) { public fun DateFormat.parseSafe(str: String?): Long = if (str.isNullOrEmpty()) {
0L 0L
} else { } else {
runCatching { runCatching {
parse(str)?.time ?: 0L parse(str)?.time ?: 0L
}.onFailure { }.onFailure {
if (javaClass.desiredAssertionStatus()) { if (javaClass.desiredAssertionStatus()) {
throw AssertionError("Cannot parse date $str", it) throw AssertionError("Cannot parse date $str", it)
} }
}.getOrDefault(0L) }.getOrDefault(0L)
} }
public fun Response.requireBody(): ResponseBody = requireNotNull(body) { @Deprecated("Useless since OkHttp 5.0", replaceWith = ReplaceWith("body"))
ErrorMessages.RESPONSE_NULL_BODY public fun Response.requireBody(): ResponseBody = body
}

@ -3,51 +3,55 @@ package org.koitharu.kotatsu.parsers.util
import kotlinx.coroutines.* import kotlinx.coroutines.*
import org.koitharu.kotatsu.parsers.MangaParser import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.model.Manga import org.koitharu.kotatsu.parsers.model.Manga
import org.koitharu.kotatsu.parsers.model.MangaListFilter
import org.koitharu.kotatsu.parsers.model.SortOrder import org.koitharu.kotatsu.parsers.model.SortOrder
import org.koitharu.kotatsu.parsers.model.search.MangaSearchQuery
import org.koitharu.kotatsu.parsers.model.search.QueryCriteria
import org.koitharu.kotatsu.parsers.model.search.SearchableField
public class RelatedMangaFinder( public class RelatedMangaFinder(
private val parsers: Collection<MangaParser>, private val parsers: Collection<MangaParser>,
) { ) {
public suspend operator fun invoke(seed: Manga): List<Manga> = withContext(Dispatchers.Default) { public suspend operator fun invoke(seed: Manga): List<Manga> = withContext(Dispatchers.Default) {
coroutineScope { coroutineScope {
parsers.singleOrNull()?.let { parser -> parsers.singleOrNull()?.let { parser ->
findRelatedImpl(this, parser, seed) findRelatedImpl(this, parser, seed)
} ?: parsers.map { parser -> } ?: parsers.map { parser ->
async { async {
findRelatedImpl(this, parser, seed) findRelatedImpl(this, parser, seed)
} }
}.awaitAll().flatten() }.awaitAll().flatten()
} }
} }
private suspend fun findRelatedImpl(scope: CoroutineScope, parser: MangaParser, seed: Manga): List<Manga> { private suspend fun findRelatedImpl(scope: CoroutineScope, parser: MangaParser, seed: Manga): List<Manga> {
val words = HashSet<String>() val words = HashSet<String>()
words += seed.title.splitByWhitespace() words += seed.title.splitByWhitespace()
seed.altTitles.forEach { seed.altTitles.forEach {
words += it.splitByWhitespace() words += it.splitByWhitespace()
} }
if (words.isEmpty()) { if (words.isEmpty()) {
return emptyList() return emptyList()
} }
val results = words.map { keyword -> val results = words.map { keyword ->
scope.async { scope.async {
val result = parser.getList( val result = parser.getList(
MangaSearchQuery.Builder() 0,
.order(SortOrder.RELEVANCE) if (SortOrder.RELEVANCE in parser.availableSortOrders) {
.criterion(QueryCriteria.Match(SearchableField.TITLE_NAME, keyword)) SortOrder.RELEVANCE
.build(), } else {
) parser.availableSortOrders.first()
result.filter { it.id != seed.id && it.containKeyword(keyword) } },
} MangaListFilter(
}.awaitAll() query = keyword,
return results.minBy { if (it.isEmpty()) Int.MAX_VALUE else it.size } ),
} )
result.filter { it.id != seed.id && it.containKeyword(keyword) }
}
}.awaitAll()
return results.minBy { if (it.isEmpty()) Int.MAX_VALUE else it.size }
}
private fun Manga.containKeyword(keyword: String): Boolean { private fun Manga.containKeyword(keyword: String): Boolean {
return title.contains(keyword, ignoreCase = true) || altTitle?.contains(keyword, ignoreCase = true) == true return title.contains(keyword, ignoreCase = true)
} || altTitles.any { it.contains(keyword, ignoreCase = true) }
}
} }

Loading…
Cancel
Save