diff --git a/buildSrc/build.gradle.kts b/buildSrc/build.gradle.kts index fc4b7904..01d255b4 100644 --- a/buildSrc/build.gradle.kts +++ b/buildSrc/build.gradle.kts @@ -1,5 +1,5 @@ plugins { - kotlin("jvm") version "2.0.20" + kotlin("jvm") version "2.2.10" } repositories { @@ -14,5 +14,5 @@ dependencies { implementation(gradleApi()) implementation("org.simpleframework:simple-xml:2.7.1") implementation("com.soywiz.korlibs.korte:korte-jvm:4.0.10") - implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.10.1") + implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.10.2") } diff --git a/buildSrc/gradle/wrapper/gradle-wrapper.properties b/buildSrc/gradle/wrapper/gradle-wrapper.properties index 41dfb879..95590242 100644 --- a/buildSrc/gradle/wrapper/gradle-wrapper.properties +++ b/buildSrc/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,6 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.4-bin.zip +distributionSha256Sum=bd71102213493060956ec229d946beee57158dbd89d0e62b91bca0fa2c5f3531 +distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.3-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 02db3bf8..ba21b482 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -1,13 +1,13 @@ [versions] -kotlin = "2.0.20" -ksp = "2.0.20-1.0.25" +kotlin = "2.2.10" +ksp = "2.2.10-2.0.2" coroutines = "1.10.2" junit = "5.10.1" -okhttp = "4.12.0" -okio = "3.11.0" +okhttp = "5.1.0" +okio = "3.16.0" json = "20240303" androidx-collection = "1.5.0" -jsoup = "1.19.1" +jsoup = "1.21.2" quickjs = "1.1.0" [plugins] diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index b032a705..68fc9372 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,7 +1,7 @@ #Wed Aug 27 01:56:37 ICT 2025 distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionSha256Sum=20f1b1176237254a6fc204d8434196fa11a4cfb387567519c61556e8710aed78 -distributionUrl=https\://services.gradle.org/distributions/gradle-8.13-bin.zip +distributionSha256Sum=bd71102213493060956ec229d946beee57158dbd89d0e62b91bca0fa2c5f3531 +distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.3-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/kotatsu-parsers-ksp/src/main/kotlin/org/koitharu/kotatsu/parsers/ksp/ParserProcessor.kt b/kotatsu-parsers-ksp/src/main/kotlin/org/koitharu/kotatsu/parsers/ksp/ParserProcessor.kt index 46f1293c..362a7ce5 100644 --- a/kotatsu-parsers-ksp/src/main/kotlin/org/koitharu/kotatsu/parsers/ksp/ParserProcessor.kt +++ b/kotatsu-parsers-ksp/src/main/kotlin/org/koitharu/kotatsu/parsers/ksp/ParserProcessor.kt @@ -12,92 +12,92 @@ import java.io.Writer import java.util.* class ParserProcessor( - private val codeGenerator: CodeGenerator, - private val logger: KSPLogger, - private val options: Map, + private val codeGenerator: CodeGenerator, + private val logger: KSPLogger, + private val options: Map, ) : SymbolProcessor { - private val availableLocales = Locale.getAvailableLocales().toSet() - private val sourceNamePattern = Regex("[A-Z_][A-Z0-9_]{3,}") - - override fun process(resolver: Resolver): List { - val symbols = resolver.getSymbolsWithAnnotation("org.koitharu.kotatsu.parsers.MangaSourceParser") - val ret = symbols.filterNot { it.validate() }.toList() - if (!symbols.iterator().hasNext()) { - return ret - } - val dependencies = Dependencies.ALL_FILES - val factoryFile = - try { - codeGenerator.createNewFile( - dependencies = dependencies, - packageName = "org.koitharu.kotatsu.parsers", - fileName = "MangaParserFactory", - ) - } catch (e: FileAlreadyExistsException) { - logger.warn(e.toString(), null) - null - } - val sourcesFile = - try { - codeGenerator.createNewFile( - dependencies = dependencies, - packageName = "org.koitharu.kotatsu.parsers.model", - fileName = "MangaSource", - ) - } catch (e: FileAlreadyExistsException) { - logger.warn(e.toString(), null) - null - } - val totalCount = sourcesFile?.writer().use { sourcesWriter -> - factoryFile?.writer().use { factoryWriter -> - writeContent(sourcesWriter, factoryWriter, symbols) - } - } - writeSummary(totalCount) - return ret - } - - private fun writeContent( - sourcesWriter: Writer?, - factoryWriter: Writer?, - symbols: Sequence, - ): Int { - if (sourcesWriter == null && factoryWriter == null) { - return 0 - } - factoryWriter?.write( - """ + private val availableLocales = Locale.getAvailableLocales().toSet() + private val sourceNamePattern = Regex("[A-Z_][A-Z0-9_]{3,}") + + override fun process(resolver: Resolver): List { + val symbols = resolver.getSymbolsWithAnnotation("org.koitharu.kotatsu.parsers.MangaSourceParser") + val ret = symbols.filterNot { it.validate() }.toList() + if (!symbols.iterator().hasNext()) { + return ret + } + val dependencies = Dependencies.ALL_FILES + val factoryFile = + try { + codeGenerator.createNewFile( + dependencies = dependencies, + packageName = "org.koitharu.kotatsu.parsers", + fileName = "MangaParserFactory", + ) + } catch (e: FileAlreadyExistsException) { + logger.warn(e.toString(), null) + null + } + val sourcesFile = + try { + codeGenerator.createNewFile( + dependencies = dependencies, + packageName = "org.koitharu.kotatsu.parsers.model", + fileName = "MangaSource", + ) + } catch (e: FileAlreadyExistsException) { + logger.warn(e.toString(), null) + null + } + val totalCount = sourcesFile?.writer().use { sourcesWriter -> + factoryFile?.writer().use { factoryWriter -> + writeContent(sourcesWriter, factoryWriter, symbols) + } + } + writeSummary(totalCount) + return ret + } + + private fun writeContent( + sourcesWriter: Writer?, + factoryWriter: Writer?, + symbols: Sequence, + ): Int { + if (sourcesWriter == null && factoryWriter == null) { + return 0 + } + factoryWriter?.write( + """ package org.koitharu.kotatsu.parsers import org.koitharu.kotatsu.parsers.model.MangaParserSource import org.koitharu.kotatsu.parsers.core.MangaParserWrapper internal fun MangaParserSource.newParser(context: MangaLoaderContext): MangaParser = when (this) { - + """.trimIndent(), - ) - sourcesWriter?.write( - """ + ) + sourcesWriter?.write( + """ package org.koitharu.kotatsu.parsers.model - + public enum class MangaParserSource( public val title: String, public val locale: String, public val contentType: ContentType, public val isBroken: Boolean, ): MangaSource { - + """.trimIndent(), - ) + ) - val visitor = ParserVisitor(sourcesWriter, factoryWriter) - val totalCount = symbols - .filter { it is KSClassDeclaration && it.validate() } - .onEach { it.accept(visitor, Unit) } - .count() + val visitor = ParserVisitor(sourcesWriter, factoryWriter) + val totalCount = symbols + .filter { it is KSClassDeclaration && it.validate() } + .onEach { it.accept(visitor, Unit) } + .count() - factoryWriter?.write( - """ + factoryWriter?.write( + """ MangaParserSource.DUMMY -> throw NotImplementedError("Manga parser ${'$'}name cannot be instantiated") }.let { require(it.source == this) { @@ -106,83 +106,83 @@ class ParserProcessor( MangaParserWrapper(it) } """.trimIndent(), - ) - sourcesWriter?.write( - """ + ) + sourcesWriter?.write( + """ DUMMY("Dummy", "", ContentType.OTHER, false), ; } """.trimIndent(), - ) - return totalCount - } - - private fun writeSummary(totalCount: Int) { - val file = File(options["summaryOutputDir"] ?: return, "summary.yaml") - file.writeText("total: $totalCount") - } - - private inner class ParserVisitor( - private val sourcesWriter: Writer?, - private val factoryWriter: Writer?, - ) : KSVisitorVoid() { - private val titles = HashMap() - - override fun visitClassDeclaration( - classDeclaration: KSClassDeclaration, - data: Unit, - ) { - if (classDeclaration.classKind != ClassKind.CLASS || classDeclaration.isAbstract()) { - logger.error("Only non-abstract can be annotated with @MangaSourceParser", classDeclaration) - } - val annotation = classDeclaration.annotations.single { it.shortName.asString() == "MangaSourceParser" } - val deprecation = classDeclaration.annotations.singleOrNull { it.shortName.asString() == "Deprecated" } - val isBroken = classDeclaration.annotations.any { it.shortName.asString() == "Broken" } - val name = annotation.arguments.single { it.name?.asString() == "name" }.value as String - val title = annotation.arguments.single { it.name?.asString() == "title" }.value as String - val locale = annotation.arguments.single { it.name?.asString() == "locale" }.value as String - val type = annotation.arguments.single { it.name?.asString() == "type" }.value - val localeString = "\"$locale\"" - val localeObj = if (locale.isEmpty()) null else Locale(locale) - val localeTitle = localeObj?.getDisplayLanguage(localeObj) - if (localeObj != null && localeObj !in availableLocales) { - logger.error("Manga source $name has wrong locale: $localeTitle") - } + ) + return totalCount + } - if (!sourceNamePattern.matches(name)) { - logger.error("Manga source name must be uppercase: $name") - } + private fun writeSummary(totalCount: Int) { + val file = File(options["summaryOutputDir"] ?: return, "summary.yaml") + file.writeText("total: $totalCount") + } - val constructor = classDeclaration.primaryConstructor - if (constructor == null || constructor.parameters.count { !it.hasDefault } != 1) { - logger.error( - "Class with @MangaSourceParser must have a primary constructor with one parameter", - classDeclaration, - ) - } - val className = checkNotNull(classDeclaration.qualifiedName?.asString()) { "Class name is null" } + private inner class ParserVisitor( + private val sourcesWriter: Writer?, + private val factoryWriter: Writer?, + ) : KSVisitorVoid() { + private val titles = HashMap() - val prevTitleClass = titles.put(title, className) - if (prevTitleClass != null) { - logger.warn("Source title duplication: \"$title\" is assigned to both $prevTitleClass and $className") - } + override fun visitClassDeclaration( + classDeclaration: KSClassDeclaration, + data: Unit, + ) { + if (classDeclaration.classKind != ClassKind.CLASS || classDeclaration.isAbstract()) { + logger.error("Only non-abstract can be annotated with @MangaSourceParser", classDeclaration) + } + val annotation = classDeclaration.annotations.single { it.shortName.asString() == "MangaSourceParser" } + val deprecation = classDeclaration.annotations.singleOrNull { it.shortName.asString() == "Deprecated" } + val isBroken = classDeclaration.annotations.any { it.shortName.asString() == "Broken" } + val name = annotation.arguments.single { it.name?.asString() == "name" }.value as String + val title = annotation.arguments.single { it.name?.asString() == "title" }.value as String + val locale = annotation.arguments.single { it.name?.asString() == "locale" }.value as String + val type = annotation.arguments.single { it.name?.asString() == "type" }.value + val localeString = "\"$locale\"" + val localeObj = if (locale.isEmpty()) null else Locale(locale) + val localeTitle = localeObj?.getDisplayLanguage(localeObj) + if (localeObj != null && localeObj !in availableLocales) { + logger.error("Manga source $name has wrong locale: $localeTitle") + } - factoryWriter?.write("\tMangaParserSource.$name -> $className(context)\n") - val deprecationString = - if (deprecation != null) { - val reason = - deprecation.arguments - .find { it.name?.asString() == "message" } - ?.value - ?.toString() ?: "Unknown reason" - "@Deprecated(\"$reason\") " - } else { - "" - } - val localeComment = localeTitle?.toTitleCase(localeObj)?.let { " /* $it */" }.orEmpty() - sourcesWriter?.write( - "\t$deprecationString$name(\"$title\", $localeString$localeComment, ContentType.$type, $isBroken),\n", - ) - } - } + if (!sourceNamePattern.matches(name)) { + logger.error("Manga source name must be uppercase: $name") + } + + val constructor = classDeclaration.primaryConstructor + if (constructor == null || constructor.parameters.count { !it.hasDefault } != 1) { + logger.error( + "Class with @MangaSourceParser must have a primary constructor with one parameter", + classDeclaration, + ) + } + val className = checkNotNull(classDeclaration.qualifiedName?.asString()) { "Class name is null" } + + val prevTitleClass = titles.put(title, className) + if (prevTitleClass != null) { + logger.warn("Source title duplication: \"$title\" is assigned to both $prevTitleClass and $className") + } + + factoryWriter?.write("\tMangaParserSource.$name -> $className(context)\n") + val deprecationString = + if (deprecation != null) { + val reason = + deprecation.arguments + .find { it.name?.asString() == "message" } + ?.value + ?.toString() ?: "Unknown reason" + "@Deprecated(\"$reason\") " + } else { + "" + } + val localeComment = localeTitle?.toTitleCase(localeObj)?.let { " /* $it */" }.orEmpty() + sourcesWriter?.write( + "\t$deprecationString$name(\"$title\", $localeString$localeComment, $type, $isBroken),\n", + ) + } + } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/ErrorMessages.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/ErrorMessages.kt index fceffaf7..d20006eb 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/ErrorMessages.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/ErrorMessages.kt @@ -2,18 +2,17 @@ package org.koitharu.kotatsu.parsers public object ErrorMessages { - public const val FILTER_MULTIPLE_STATES_NOT_SUPPORTED: String = "Multiple states are not supported by this source" - public const val FILTER_MULTIPLE_GENRES_NOT_SUPPORTED: String = "Multiple genres are not supported by this source" - public const val FILTER_MULTIPLE_CONTENT_RATING_NOT_SUPPORTED: String = - "Multiple Content ratings are not supported by this source" - public const val FILTER_MULTIPLE_CONTENT_TYPES_NOT_SUPPORTED: String = - "Multiple Content types are not supported by this source" - public const val FILTER_MULTIPLE_DEMOGRAPHICS_NOT_SUPPORTED: String = - "Multiple Demographics are not supported by this source" - public const val FILTER_BOTH_LOCALE_GENRES_NOT_SUPPORTED: String = - "Filtering by both genres and locale is not supported by this source" - public const val FILTER_BOTH_STATES_GENRES_NOT_SUPPORTED: String = - "Filtering by both genres and states is not supported by this source" - public const val SEARCH_NOT_SUPPORTED: String = "Search is not supported by this source" - public const val RESPONSE_NULL_BODY: String = "Response has no body" + public const val FILTER_MULTIPLE_STATES_NOT_SUPPORTED: String = "Multiple states are not supported by this source" + public const val FILTER_MULTIPLE_GENRES_NOT_SUPPORTED: String = "Multiple genres are not supported by this source" + public const val FILTER_MULTIPLE_CONTENT_RATING_NOT_SUPPORTED: String = + "Multiple Content ratings are not supported by this source" + public const val FILTER_MULTIPLE_CONTENT_TYPES_NOT_SUPPORTED: String = + "Multiple Content types are not supported by this source" + public const val FILTER_MULTIPLE_DEMOGRAPHICS_NOT_SUPPORTED: String = + "Multiple Demographics are not supported by this source" + public const val FILTER_BOTH_LOCALE_GENRES_NOT_SUPPORTED: String = + "Filtering by both genres and locale is not supported by this source" + public const val FILTER_BOTH_STATES_GENRES_NOT_SUPPORTED: String = + "Filtering by both genres and states is not supported by this source" + public const val SEARCH_NOT_SUPPORTED: String = "Search is not supported by this source" } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/network/CloudFlareHelper.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/network/CloudFlareHelper.kt index f5f514db..353db066 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/network/CloudFlareHelper.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/network/CloudFlareHelper.kt @@ -9,39 +9,39 @@ import java.net.HttpURLConnection.HTTP_UNAVAILABLE public object CloudFlareHelper { - public const val PROTECTION_NOT_DETECTED: Int = 0 - public const val PROTECTION_CAPTCHA: Int = 1 - public const val PROTECTION_BLOCKED: Int = 2 - - private const val CF_CLEARANCE = "cf_clearance" - - public fun checkResponseForProtection(response: Response): Int { - if (response.code != HTTP_FORBIDDEN && response.code != HTTP_UNAVAILABLE) { - return PROTECTION_NOT_DETECTED - } - val content = if (response.body != null) { - response.peekBody(Long.MAX_VALUE).use { - Jsoup.parse(it.byteStream(), Charsets.UTF_8.name(), response.request.url.toString()) - } - } else { - return PROTECTION_NOT_DETECTED - } - return when { - content.selectFirst("h2[data-translate=\"blocked_why_headline\"]") != null -> PROTECTION_BLOCKED - content.getElementById("challenge-error-title") != null || content.getElementById("challenge-error-text") != null -> PROTECTION_CAPTCHA - - else -> PROTECTION_NOT_DETECTED - } - } - - public fun getClearanceCookie(cookieJar: CookieJar, url: String): String? { - return cookieJar.loadForRequest(url.toHttpUrl()).find { it.name == CF_CLEARANCE }?.value - } - - public fun isCloudFlareCookie(name: String): Boolean { - return name.startsWith("cf_") - || name.startsWith("_cf") - || name.startsWith("__cf") - || name == "csrftoken" - } + public const val PROTECTION_NOT_DETECTED: Int = 0 + public const val PROTECTION_CAPTCHA: Int = 1 + public const val PROTECTION_BLOCKED: Int = 2 + + private const val CF_CLEARANCE = "cf_clearance" + + public fun checkResponseForProtection(response: Response): Int { + if (response.code != HTTP_FORBIDDEN && response.code != HTTP_UNAVAILABLE) { + return PROTECTION_NOT_DETECTED + } + val content = try { + response.peekBody(Long.MAX_VALUE).use { + Jsoup.parse(it.byteStream(), Charsets.UTF_8.name(), response.request.url.toString()) + } + } catch (_: IllegalStateException) { + return PROTECTION_NOT_DETECTED + } + return when { + content.selectFirst("h2[data-translate=\"blocked_why_headline\"]") != null -> PROTECTION_BLOCKED + content.getElementById("challenge-error-title") != null || content.getElementById("challenge-error-text") != null -> PROTECTION_CAPTCHA + + else -> PROTECTION_NOT_DETECTED + } + } + + public fun getClearanceCookie(cookieJar: CookieJar, url: String): String? { + return cookieJar.loadForRequest(url.toHttpUrl()).find { it.name == CF_CLEARANCE }?.value + } + + public fun isCloudFlareCookie(name: String): Boolean { + return name.startsWith("cf_") + || name.startsWith("_cf") + || name.startsWith("__cf") + || name == "csrftoken" + } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/ExHentaiParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/ExHentaiParser.kt index 6dc26891..cf536228 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/ExHentaiParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/ExHentaiParser.kt @@ -7,7 +7,6 @@ import okhttp3.HttpUrl.Companion.toHttpUrlOrNull import okhttp3.Interceptor import okhttp3.Response import okhttp3.internal.closeQuietly -import okhttp3.internal.headersContentLength import org.jsoup.internal.StringUtil import org.jsoup.nodes.Element import org.koitharu.kotatsu.parsers.MangaLoaderContext @@ -28,470 +27,471 @@ import java.util.concurrent.TimeUnit private const val DOMAIN_UNAUTHORIZED = "e-hentai.org" private const val DOMAIN_AUTHORIZED = "exhentai.org" private val TAG_PREFIXES = arrayOf("male:", "female:", "other:") +private const val BANNED_RESPONSE_LENGTH = 256L @MangaSourceParser("EXHENTAI", "ExHentai", type = ContentType.HENTAI) internal class ExHentaiParser( - context: MangaLoaderContext, + context: MangaLoaderContext, ) : PagedMangaParser(context, MangaParserSource.EXHENTAI, pageSize = 25), MangaParserAuthProvider, Interceptor { - override val availableSortOrders: Set = setOf(SortOrder.NEWEST) - - override val configKeyDomain: ConfigKey.Domain - get() { - val isAuthorized = checkAuth() - return ConfigKey.Domain( - if (isAuthorized) DOMAIN_AUTHORIZED else DOMAIN_UNAUTHORIZED, - if (isAuthorized) DOMAIN_UNAUTHORIZED else DOMAIN_AUTHORIZED, - ) - } - - override val authUrl: String - get() = "https://${domain}/bounce_login.php" - - private val ratingPattern = Regex("-?[0-9]+px") - private val titleCleanupPattern = Regex("(\\[.*?]|\\([C0-9]*\\))") - private val spacesCleanupPattern = Regex("(^\\s+|\\s+\$|\\s+(?=\\s))") - private val authCookies = arrayOf("ipb_member_id", "ipb_pass_hash") - private val suspiciousContentKey = ConfigKey.ShowSuspiciousContent(false) - private val nextPages = MutableIntObjectMap() - - override val filterCapabilities: MangaListFilterCapabilities - get() = MangaListFilterCapabilities( - isMultipleTagsSupported = true, - isTagsExclusionSupported = true, - isSearchSupported = true, - isSearchWithFiltersSupported = true, - isAuthorSearchSupported = true, - ) - - override suspend fun isAuthorized(): Boolean = checkAuth() - - init { - context.cookieJar.insertCookies(DOMAIN_AUTHORIZED, "nw=1", "sl=dm_2") - context.cookieJar.insertCookies(DOMAIN_UNAUTHORIZED, "nw=1", "sl=dm_2") - paginator.firstPage = 0 - searchPaginator.firstPage = 0 - } - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = mapTags(), - availableContentTypes = EnumSet.of( - ContentType.DOUJINSHI, - ContentType.MANGA, - ContentType.ARTIST_CG, - ContentType.GAME_CG, - ContentType.COMICS, - ContentType.IMAGE_SET, - ContentType.OTHER, - ), - availableLocales = setOf( - Locale.JAPANESE, - Locale.ENGLISH, - Locale.CHINESE, - Locale("nl"), - Locale.FRENCH, - Locale.GERMAN, - Locale("hu"), - Locale.ITALIAN, - Locale("kr"), - Locale("pl"), - Locale("pt"), - Locale("ru"), - Locale("es"), - Locale("th"), - Locale("vi"), - ), - ) - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { - return getListPage(page, order, filter, updateDm = false) - } - - private suspend fun getListPage( - page: Int, - order: SortOrder, - filter: MangaListFilter, - updateDm: Boolean, - ): List { - val next = synchronized(nextPages) { - nextPages[filter.hashCode()]?.getOrDefault(page, 0L) ?: 0L - } - - if (page > 0 && next == 0L) { - assert(false) { "Page timestamp not found" } - return emptyList() - } - - val url = urlBuilder() - url.addEncodedQueryParameter("next", next.toString()) - url.addQueryParameter("f_search", filter.toSearchQuery()) - - val fCats = filter.types.toFCats() - if (fCats != 0) { - url.addEncodedQueryParameter("f_cats", (1023 - fCats).toString()) - } - if (updateDm) { - // by unknown reason cookie "sl=dm_2" is ignored, so, we should request it again - url.addQueryParameter("inline_set", "dm_e") - } - url.addQueryParameter("advsearch", "1") - if (config[suspiciousContentKey]) { - url.addQueryParameter("f_sh", "on") - } - val body = webClient.httpGet(url.build()).parseHtml().body() - val root = body.selectFirst("table.itg")?.selectFirst("tbody") - if (root == null) { - if (updateDm) { - if (body.getElementsContainingText("No hits found").isNotEmpty()) { - return emptyList() - } else { - body.parseFailed("Cannot find root") - } - } else { - return getListPage(page, order, filter, updateDm = true) - } - } - val nextTimestamp = getNextTimestamp(body) - synchronized(nextPages) { - nextPages.getOrPut(filter.hashCode()) { - MutableIntLongMap() - }.put(page + 1, nextTimestamp) - } - - return root.children().mapNotNull { tr -> - if (tr.childrenSize() != 2) return@mapNotNull null - val (td1, td2) = tr.children() - val gLink = td2.selectFirstOrThrow("div.glink") - val a = gLink.parents().select("a").first() ?: gLink.parseFailed("link not found") - val href = a.attrAsRelativeUrl("href") - val tagsDiv = gLink.nextElementSibling() ?: gLink.parseFailed("tags div not found") - val rawTitle = gLink.text() - val author = tagsDiv.getElementsContainingOwnText("artist:").first() - ?.nextElementSibling()?.textOrNull() - Manga( - id = generateUid(href), - title = rawTitle.cleanupTitle(), - altTitles = emptySet(), - url = href, - publicUrl = a.absUrl("href"), - rating = td2.selectFirst("div.ir")?.parseRating() ?: RATING_UNKNOWN, - contentRating = ContentRating.ADULT, - coverUrl = td1.selectFirst("img")?.attrAsAbsoluteUrlOrNull("src"), - tags = tagsDiv.parseTags(), - state = when { - rawTitle.contains("(ongoing)", ignoreCase = true) -> MangaState.ONGOING - else -> null - }, - authors = setOfNotNull(author), - source = source, - ) - } - } - - override suspend fun getDetails(manga: Manga): Manga { - val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - val root = doc.body().selectFirstOrThrow("div.gm") - val cover = root.getElementById("gd1")?.children()?.first() - val title = root.getElementById("gd2") - val tagList = root.getElementById("taglist") - val tabs = doc.body().selectFirst("table.ptt")?.selectFirst("tr") - val gd3 = root.getElementById("gd3") - val lang = gd3 - ?.selectFirst("tr:contains(Language)") - ?.selectFirst(".gdt2")?.ownTextOrNull() - val uploadDate = gd3 - ?.selectFirst("tr:contains(Posted)") - ?.selectFirst(".gdt2")?.ownTextOrNull() - .let { SimpleDateFormat("yyyy-MM-dd HH:mm", sourceLocale).parseSafe(it) } - val uploader = gd3 - ?.getElementsByAttributeValueContaining("href", "/uploader/") - ?.firstOrNull() - ?.ownTextOrNull() - val tags = tagList?.parseTags().orEmpty() - - return manga.copy( - title = title?.getElementById("gn")?.text()?.cleanupTitle() ?: manga.title, - altTitles = setOfNotNull(title?.getElementById("gj")?.text()?.cleanupTitle()?.nullIfEmpty()), - publicUrl = doc.baseUri().ifEmpty { manga.publicUrl }, - rating = root.getElementById("rating_label")?.text() - ?.substringAfterLast(' ') - ?.toFloatOrNull() - ?.div(5f) ?: manga.rating, - largeCoverUrl = cover?.styleValueOrNull("background")?.cssUrl(), - tags = manga.tags + tags, - description = tagList?.select("tr")?.joinToString("
") { tr -> - val (tc, td) = tr.children() - val subTags = td.select("a").joinToString { it.html() } - "${tc.html()} $subTags" - }, - chapters = tabs?.select("a")?.findLast { a -> - a.text().toIntOrNull() != null - }?.let { a -> - val count = a.text().toInt() - val chapters = ChaptersListBuilder(count) - for (i in 1..count) { - val url = "${manga.url}?p=${i - 1}" - chapters += MangaChapter( - id = generateUid(url), - title = null, - number = i.toFloat(), - volume = 0, - url = url, - uploadDate = uploadDate, - source = source, - scanlator = uploader, - branch = lang, - ) - } - chapters.toList() - }, - ) - } - - override suspend fun getPages(chapter: MangaChapter): List { - val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() - val root = doc.body().requireElementById("gdt") - return root.select("a").map { a -> - val url = a.attrAsRelativeUrl("href") - MangaPage( - id = generateUid(url), - url = url, - preview = a.children().firstOrNull()?.extractPreview(), - source = source, - ) - } - } - - override suspend fun getPageUrl(page: MangaPage): String { - val doc = webClient.httpGet(page.url.toAbsoluteUrl(domain)).parseHtml() - return doc.body().requireElementById("img").attrAsAbsoluteUrl("src") - } - - @Suppress("SpellCheckingInspection") - private val tags: String - get() = "ahegao,anal,angel,apron,bandages,bbw,bdsm,beauty mark,big areolae,big ass,big breasts,big clit,big lips," + - "big nipples,bikini,blackmail,bloomers,blowjob,bodysuit,bondage,breast expansion,bukkake,bunny girl,business suit," + - "catgirl,centaur,cheating,chinese dress,christmas,collar,corset,cosplaying,cowgirl,crossdressing,cunnilingus," + - "dark skin,daughter,deepthroat,defloration,demon girl,double penetration,dougi,dragon,drunk,elf,exhibitionism,farting," + - "females only,femdom,filming,fingering,fishnets,footjob,fox girl,furry,futanari,garter belt,ghost,giantess," + - "glasses,gloves,goblin,gothic lolita,growth,guro,gyaru,hair buns,hairy,hairy armpits,handjob,harem,hidden sex," + - "horns,huge breasts,humiliation,impregnation,incest,inverted nipples,kemonomimi,kimono,kissing,lactation," + - "latex,leg lock,leotard,lingerie,lizard girl,maid,masked face,masturbation,midget,miko,milf,mind break," + - "mind control,monster girl,mother,muscle,nakadashi,netorare,nose hook,nun,nurse,oil,paizuri,panda girl," + - "pantyhose,piercing,pixie cut,policewoman,ponytail,pregnant,rape,rimjob,robot,scat,lolicon,schoolgirl uniform," + - "sex toys,shemale,sister,small breasts,smell,sole dickgirl,sole female,squirting,stockings,sundress,sweating," + - "swimsuit,swinging,tail,tall girl,teacher,tentacles,thigh high boots,tomboy,transformation,twins,twintails," + - "unusual pupils,urination,vore,vtuber,widow,wings,witch,wolf girl,x-ray,yuri,zombie,sole male,males only,yaoi," + - "tomgirl,tall man,oni,shotacon,prostate massage,policeman,males only,huge penis,fox boy,feminization,dog boy,dickgirl on male,big penis" - - private fun mapTags(): Set { - val tagElements = tags.split(",") - val result = ArraySet(tagElements.size) - for (tag in tagElements) { - val el = tag.trim() - if (el.isEmpty()) continue - result += MangaTag( - title = el.toTitleCase(Locale.ENGLISH), - key = el, - source = source, - ) - } - return result - } - - override fun intercept(chain: Interceptor.Chain): Response { - val response = chain.proceed(chain.request()) - if (response.headersContentLength() <= 256) { - val text = response.peekBody(256).use { it.string() } - if (text.contains("IP address has been temporarily banned", ignoreCase = true)) { - val hours = Regex("([0-9]+) hours?").find(text)?.groupValues?.getOrNull(1)?.toLongOrNull() ?: 0 - val minutes = Regex("([0-9]+) minutes?").find(text)?.groupValues?.getOrNull(1)?.toLongOrNull() ?: 0 - val seconds = Regex("([0-9]+) seconds?").find(text)?.groupValues?.getOrNull(1)?.toLongOrNull() ?: 0 - response.closeQuietly() - throw TooManyRequestExceptions( - url = response.request.url.toString(), - retryAfter = TimeUnit.HOURS.toMillis(hours) - + TimeUnit.MINUTES.toMillis(minutes) - + TimeUnit.SECONDS.toMillis(seconds), - ) - } - } - val imageRect = response.request.url.fragment?.split(',') - if (imageRect != null && imageRect.size == 4) { - // rect: top,left,right,bottom - return context.redrawImageResponse(response) { bitmap -> - val srcRect = Rect( - left = imageRect[0].toInt(), - top = imageRect[1].toInt(), - right = imageRect[2].toInt(), - bottom = imageRect[3].toInt(), - ) - val dstRect = Rect(0, 0, srcRect.width, srcRect.height) - val result = context.createBitmap(dstRect.width, dstRect.height) - result.drawBitmap(bitmap, srcRect, dstRect) - result - } - } - return response - } - - private fun Locale.toLanguagePath() = when (language) { - else -> getDisplayLanguage(Locale.ENGLISH).lowercase() - } - - override suspend fun getUsername(): String { - val doc = webClient.httpGet("https://forums.$DOMAIN_UNAUTHORIZED/").parseHtml().body() - val username = doc.getElementById("userlinks") - ?.getElementsByAttributeValueContaining("href", "showuser=") - ?.firstOrNull() - ?.ownText() - ?: if (doc.getElementById("userlinksguest") != null) { - throw AuthRequiredException(source) - } else { - doc.parseFailed() - } - return username - } - - override fun onCreateConfig(keys: MutableCollection>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - keys.add(suspiciousContentKey) - } - - override suspend fun getRelatedManga(seed: Manga): List { - val query = seed.title - return getListPage( - page = 0, - order = defaultSortOrder, - filter = MangaListFilter(query = query), - ) - } - - private fun isAuthorized(domain: String): Boolean { - val cookies = context.cookieJar.getCookies(domain).mapToSet { x -> x.name } - return authCookies.all { it in cookies } - } - - private fun Element.parseRating(): Float { - return runCatching { - val style = requireNotNull(attr("style")) - val (v1, v2) = ratingPattern.findAll(style).toList() - var p1 = v1.groupValues.first().dropLast(2).toInt() - val p2 = v2.groupValues.first().dropLast(2).toInt() - if (p2 != -1) { - p1 += 8 - } - (80 - p1) / 80f - }.getOrDefault(RATING_UNKNOWN) - } - - private fun String.cleanupTitle(): String { - return replace(titleCleanupPattern, "") - .replace(spacesCleanupPattern, "") - } - - private fun Element.parseTags(): Set { - - fun Element.parseTag() = textOrNull()?.let { - MangaTag(title = it.toTitleCase(Locale.ENGLISH), key = it, source = source) - } - - val result = ArraySet() - for (prefix in TAG_PREFIXES) { - getElementsByAttributeValueStarting("id", "ta_$prefix").mapNotNullTo(result, Element::parseTag) - getElementsByAttributeValueStarting("title", prefix).mapNotNullTo(result, Element::parseTag) - } - return result - } - - private fun Element.extractPreview(): String? { - val bg = backgroundOrNull() ?: return null - return buildString { - append(bg.url) - append('#') - // rect: left,top,right,bottom - append(bg.left) - append(',') - append(bg.top) - append(',') - append(bg.right) - append(',') - append(bg.bottom) - } - } - - private fun getNextTimestamp(root: Element): Long { - return root.getElementById("unext") - ?.attrAsAbsoluteUrlOrNull("href") - ?.toHttpUrlOrNull() - ?.queryParameter("next") - ?.toLongOrNull() ?: 1 - } - - private fun MangaListFilter.toSearchQuery(): String? { - if (isEmpty()) { - return null - } - val joiner = StringUtil.StringJoiner(" ") - if (!query.isNullOrEmpty()) { - joiner.add(query) - } - for (tag in tags) { - if (tag.key.isNumeric()) { - continue - } - joiner.add("tag:\"") - joiner.append(tag.key) - joiner.append("\"$") - } - for (tag in tagsExclude) { - if (tag.key.isNumeric()) { - continue - } - joiner.add("-tag:\"") - joiner.append(tag.key) - joiner.append("\"$") - } - locale?.let { lc -> - joiner.add("language:\"") - joiner.append(lc.toLanguagePath()) - joiner.append("\"$") - } - if (!author.isNullOrEmpty()) { - joiner.add("artist:\"") - joiner.append(author) - joiner.append("\"$") - } - return joiner.complete().nullIfEmpty() - } - - private fun Collection.toFCats(): Int = fold(0) { acc, ct -> - val cat: Int = when (ct) { - ContentType.DOUJINSHI -> 2 - ContentType.MANGA -> 4 - ContentType.ARTIST_CG -> 8 - ContentType.GAME_CG -> 16 - ContentType.COMICS -> 512 - ContentType.IMAGE_SET -> 32 - else -> 449 // 1 or 64 or 128 or 256 - } - acc or cat - } - - private fun checkAuth(): Boolean { - val authorized = isAuthorized(DOMAIN_UNAUTHORIZED) - if (authorized) { - if (!isAuthorized(DOMAIN_AUTHORIZED)) { - context.cookieJar.copyCookies( - DOMAIN_UNAUTHORIZED, - DOMAIN_AUTHORIZED, - authCookies, - ) - context.cookieJar.insertCookies(DOMAIN_AUTHORIZED, "yay=louder") - } - return true - } - return false - } + override val availableSortOrders: Set = setOf(SortOrder.NEWEST) + + override val configKeyDomain: ConfigKey.Domain + get() { + val isAuthorized = checkAuth() + return ConfigKey.Domain( + if (isAuthorized) DOMAIN_AUTHORIZED else DOMAIN_UNAUTHORIZED, + if (isAuthorized) DOMAIN_UNAUTHORIZED else DOMAIN_AUTHORIZED, + ) + } + + override val authUrl: String + get() = "https://${domain}/bounce_login.php" + + private val ratingPattern = Regex("-?[0-9]+px") + private val titleCleanupPattern = Regex("(\\[.*?]|\\([C0-9]*\\))") + private val spacesCleanupPattern = Regex("(^\\s+|\\s+\$|\\s+(?=\\s))") + private val authCookies = arrayOf("ipb_member_id", "ipb_pass_hash") + private val suspiciousContentKey = ConfigKey.ShowSuspiciousContent(false) + private val nextPages = MutableIntObjectMap() + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isMultipleTagsSupported = true, + isTagsExclusionSupported = true, + isSearchSupported = true, + isSearchWithFiltersSupported = true, + isAuthorSearchSupported = true, + ) + + override suspend fun isAuthorized(): Boolean = checkAuth() + + init { + context.cookieJar.insertCookies(DOMAIN_AUTHORIZED, "nw=1", "sl=dm_2") + context.cookieJar.insertCookies(DOMAIN_UNAUTHORIZED, "nw=1", "sl=dm_2") + paginator.firstPage = 0 + searchPaginator.firstPage = 0 + } + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = mapTags(), + availableContentTypes = EnumSet.of( + ContentType.DOUJINSHI, + ContentType.MANGA, + ContentType.ARTIST_CG, + ContentType.GAME_CG, + ContentType.COMICS, + ContentType.IMAGE_SET, + ContentType.OTHER, + ), + availableLocales = setOf( + Locale.JAPANESE, + Locale.ENGLISH, + Locale.CHINESE, + Locale("nl"), + Locale.FRENCH, + Locale.GERMAN, + Locale("hu"), + Locale.ITALIAN, + Locale("kr"), + Locale("pl"), + Locale("pt"), + Locale("ru"), + Locale("es"), + Locale("th"), + Locale("vi"), + ), + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { + return getListPage(page, order, filter, updateDm = false) + } + + private suspend fun getListPage( + page: Int, + order: SortOrder, + filter: MangaListFilter, + updateDm: Boolean, + ): List { + val next = synchronized(nextPages) { + nextPages[filter.hashCode()]?.getOrDefault(page, 0L) ?: 0L + } + + if (page > 0 && next == 0L) { + assert(false) { "Page timestamp not found" } + return emptyList() + } + + val url = urlBuilder() + url.addEncodedQueryParameter("next", next.toString()) + url.addQueryParameter("f_search", filter.toSearchQuery()) + + val fCats = filter.types.toFCats() + if (fCats != 0) { + url.addEncodedQueryParameter("f_cats", (1023 - fCats).toString()) + } + if (updateDm) { + // by unknown reason cookie "sl=dm_2" is ignored, so, we should request it again + url.addQueryParameter("inline_set", "dm_e") + } + url.addQueryParameter("advsearch", "1") + if (config[suspiciousContentKey]) { + url.addQueryParameter("f_sh", "on") + } + val body = webClient.httpGet(url.build()).parseHtml().body() + val root = body.selectFirst("table.itg")?.selectFirst("tbody") + if (root == null) { + if (updateDm) { + if (body.getElementsContainingText("No hits found").isNotEmpty()) { + return emptyList() + } else { + body.parseFailed("Cannot find root") + } + } else { + return getListPage(page, order, filter, updateDm = true) + } + } + val nextTimestamp = getNextTimestamp(body) + synchronized(nextPages) { + nextPages.getOrPut(filter.hashCode()) { + MutableIntLongMap() + }.put(page + 1, nextTimestamp) + } + + return root.children().mapNotNull { tr -> + if (tr.childrenSize() != 2) return@mapNotNull null + val (td1, td2) = tr.children() + val gLink = td2.selectFirstOrThrow("div.glink") + val a = gLink.parents().select("a").first() ?: gLink.parseFailed("link not found") + val href = a.attrAsRelativeUrl("href") + val tagsDiv = gLink.nextElementSibling() ?: gLink.parseFailed("tags div not found") + val rawTitle = gLink.text() + val author = tagsDiv.getElementsContainingOwnText("artist:").first() + ?.nextElementSibling()?.textOrNull() + Manga( + id = generateUid(href), + title = rawTitle.cleanupTitle(), + altTitles = emptySet(), + url = href, + publicUrl = a.absUrl("href"), + rating = td2.selectFirst("div.ir")?.parseRating() ?: RATING_UNKNOWN, + contentRating = ContentRating.ADULT, + coverUrl = td1.selectFirst("img")?.attrAsAbsoluteUrlOrNull("src"), + tags = tagsDiv.parseTags(), + state = when { + rawTitle.contains("(ongoing)", ignoreCase = true) -> MangaState.ONGOING + else -> null + }, + authors = setOfNotNull(author), + source = source, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val root = doc.body().selectFirstOrThrow("div.gm") + val cover = root.getElementById("gd1")?.children()?.first() + val title = root.getElementById("gd2") + val tagList = root.getElementById("taglist") + val tabs = doc.body().selectFirst("table.ptt")?.selectFirst("tr") + val gd3 = root.getElementById("gd3") + val lang = gd3 + ?.selectFirst("tr:contains(Language)") + ?.selectFirst(".gdt2")?.ownTextOrNull() + val uploadDate = gd3 + ?.selectFirst("tr:contains(Posted)") + ?.selectFirst(".gdt2")?.ownTextOrNull() + .let { SimpleDateFormat("yyyy-MM-dd HH:mm", sourceLocale).parseSafe(it) } + val uploader = gd3 + ?.getElementsByAttributeValueContaining("href", "/uploader/") + ?.firstOrNull() + ?.ownTextOrNull() + val tags = tagList?.parseTags().orEmpty() + + return manga.copy( + title = title?.getElementById("gn")?.text()?.cleanupTitle() ?: manga.title, + altTitles = setOfNotNull(title?.getElementById("gj")?.text()?.cleanupTitle()?.nullIfEmpty()), + publicUrl = doc.baseUri().ifEmpty { manga.publicUrl }, + rating = root.getElementById("rating_label")?.text() + ?.substringAfterLast(' ') + ?.toFloatOrNull() + ?.div(5f) ?: manga.rating, + largeCoverUrl = cover?.styleValueOrNull("background")?.cssUrl(), + tags = manga.tags + tags, + description = tagList?.select("tr")?.joinToString("
") { tr -> + val (tc, td) = tr.children() + val subTags = td.select("a").joinToString { it.html() } + "${tc.html()} $subTags" + }, + chapters = tabs?.select("a")?.findLast { a -> + a.text().toIntOrNull() != null + }?.let { a -> + val count = a.text().toInt() + val chapters = ChaptersListBuilder(count) + for (i in 1..count) { + val url = "${manga.url}?p=${i - 1}" + chapters += MangaChapter( + id = generateUid(url), + title = null, + number = i.toFloat(), + volume = 0, + url = url, + uploadDate = uploadDate, + source = source, + scanlator = uploader, + branch = lang, + ) + } + chapters.toList() + }, + ) + } + + override suspend fun getPages(chapter: MangaChapter): List { + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + val root = doc.body().requireElementById("gdt") + return root.select("a").map { a -> + val url = a.attrAsRelativeUrl("href") + MangaPage( + id = generateUid(url), + url = url, + preview = a.children().firstOrNull()?.extractPreview(), + source = source, + ) + } + } + + override suspend fun getPageUrl(page: MangaPage): String { + val doc = webClient.httpGet(page.url.toAbsoluteUrl(domain)).parseHtml() + return doc.body().requireElementById("img").attrAsAbsoluteUrl("src") + } + + @Suppress("SpellCheckingInspection") + private val tags: String + get() = "ahegao,anal,angel,apron,bandages,bbw,bdsm,beauty mark,big areolae,big ass,big breasts,big clit,big lips," + + "big nipples,bikini,blackmail,bloomers,blowjob,bodysuit,bondage,breast expansion,bukkake,bunny girl,business suit," + + "catgirl,centaur,cheating,chinese dress,christmas,collar,corset,cosplaying,cowgirl,crossdressing,cunnilingus," + + "dark skin,daughter,deepthroat,defloration,demon girl,double penetration,dougi,dragon,drunk,elf,exhibitionism,farting," + + "females only,femdom,filming,fingering,fishnets,footjob,fox girl,furry,futanari,garter belt,ghost,giantess," + + "glasses,gloves,goblin,gothic lolita,growth,guro,gyaru,hair buns,hairy,hairy armpits,handjob,harem,hidden sex," + + "horns,huge breasts,humiliation,impregnation,incest,inverted nipples,kemonomimi,kimono,kissing,lactation," + + "latex,leg lock,leotard,lingerie,lizard girl,maid,masked face,masturbation,midget,miko,milf,mind break," + + "mind control,monster girl,mother,muscle,nakadashi,netorare,nose hook,nun,nurse,oil,paizuri,panda girl," + + "pantyhose,piercing,pixie cut,policewoman,ponytail,pregnant,rape,rimjob,robot,scat,lolicon,schoolgirl uniform," + + "sex toys,shemale,sister,small breasts,smell,sole dickgirl,sole female,squirting,stockings,sundress,sweating," + + "swimsuit,swinging,tail,tall girl,teacher,tentacles,thigh high boots,tomboy,transformation,twins,twintails," + + "unusual pupils,urination,vore,vtuber,widow,wings,witch,wolf girl,x-ray,yuri,zombie,sole male,males only,yaoi," + + "tomgirl,tall man,oni,shotacon,prostate massage,policeman,males only,huge penis,fox boy,feminization,dog boy,dickgirl on male,big penis" + + private fun mapTags(): Set { + val tagElements = tags.split(",") + val result = ArraySet(tagElements.size) + for (tag in tagElements) { + val el = tag.trim() + if (el.isEmpty()) continue + result += MangaTag( + title = el.toTitleCase(Locale.ENGLISH), + key = el, + source = source, + ) + } + return result + } + + override fun intercept(chain: Interceptor.Chain): Response { + val response = chain.proceed(chain.request()) + if (response.headersContentLength(BANNED_RESPONSE_LENGTH) <= BANNED_RESPONSE_LENGTH) { + val text = response.peekBody(BANNED_RESPONSE_LENGTH).use { it.string() } + if (text.contains("IP address has been temporarily banned", ignoreCase = true)) { + val hours = Regex("([0-9]+) hours?").find(text)?.groupValues?.getOrNull(1)?.toLongOrNull() ?: 0 + val minutes = Regex("([0-9]+) minutes?").find(text)?.groupValues?.getOrNull(1)?.toLongOrNull() ?: 0 + val seconds = Regex("([0-9]+) seconds?").find(text)?.groupValues?.getOrNull(1)?.toLongOrNull() ?: 0 + response.closeQuietly() + throw TooManyRequestExceptions( + url = response.request.url.toString(), + retryAfter = TimeUnit.HOURS.toMillis(hours) + + TimeUnit.MINUTES.toMillis(minutes) + + TimeUnit.SECONDS.toMillis(seconds), + ) + } + } + val imageRect = response.request.url.fragment?.split(',') + if (imageRect != null && imageRect.size == 4) { + // rect: top,left,right,bottom + return context.redrawImageResponse(response) { bitmap -> + val srcRect = Rect( + left = imageRect[0].toInt(), + top = imageRect[1].toInt(), + right = imageRect[2].toInt(), + bottom = imageRect[3].toInt(), + ) + val dstRect = Rect(0, 0, srcRect.width, srcRect.height) + val result = context.createBitmap(dstRect.width, dstRect.height) + result.drawBitmap(bitmap, srcRect, dstRect) + result + } + } + return response + } + + private fun Locale.toLanguagePath() = when (language) { + else -> getDisplayLanguage(Locale.ENGLISH).lowercase() + } + + override suspend fun getUsername(): String { + val doc = webClient.httpGet("https://forums.$DOMAIN_UNAUTHORIZED/").parseHtml().body() + val username = doc.getElementById("userlinks") + ?.getElementsByAttributeValueContaining("href", "showuser=") + ?.firstOrNull() + ?.ownText() + ?: if (doc.getElementById("userlinksguest") != null) { + throw AuthRequiredException(source) + } else { + doc.parseFailed() + } + return username + } + + override fun onCreateConfig(keys: MutableCollection>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + keys.add(suspiciousContentKey) + } + + override suspend fun getRelatedManga(seed: Manga): List { + val query = seed.title + return getListPage( + page = 0, + order = defaultSortOrder, + filter = MangaListFilter(query = query), + ) + } + + private fun isAuthorized(domain: String): Boolean { + val cookies = context.cookieJar.getCookies(domain).mapToSet { x -> x.name } + return authCookies.all { it in cookies } + } + + private fun Element.parseRating(): Float { + return runCatching { + val style = requireNotNull(attr("style")) + val (v1, v2) = ratingPattern.findAll(style).toList() + var p1 = v1.groupValues.first().dropLast(2).toInt() + val p2 = v2.groupValues.first().dropLast(2).toInt() + if (p2 != -1) { + p1 += 8 + } + (80 - p1) / 80f + }.getOrDefault(RATING_UNKNOWN) + } + + private fun String.cleanupTitle(): String { + return replace(titleCleanupPattern, "") + .replace(spacesCleanupPattern, "") + } + + private fun Element.parseTags(): Set { + + fun Element.parseTag() = textOrNull()?.let { + MangaTag(title = it.toTitleCase(Locale.ENGLISH), key = it, source = source) + } + + val result = ArraySet() + for (prefix in TAG_PREFIXES) { + getElementsByAttributeValueStarting("id", "ta_$prefix").mapNotNullTo(result, Element::parseTag) + getElementsByAttributeValueStarting("title", prefix).mapNotNullTo(result, Element::parseTag) + } + return result + } + + private fun Element.extractPreview(): String? { + val bg = backgroundOrNull() ?: return null + return buildString { + append(bg.url) + append('#') + // rect: left,top,right,bottom + append(bg.left) + append(',') + append(bg.top) + append(',') + append(bg.right) + append(',') + append(bg.bottom) + } + } + + private fun getNextTimestamp(root: Element): Long { + return root.getElementById("unext") + ?.attrAsAbsoluteUrlOrNull("href") + ?.toHttpUrlOrNull() + ?.queryParameter("next") + ?.toLongOrNull() ?: 1 + } + + private fun MangaListFilter.toSearchQuery(): String? { + if (isEmpty()) { + return null + } + val joiner = StringUtil.StringJoiner(" ") + if (!query.isNullOrEmpty()) { + joiner.add(query) + } + for (tag in tags) { + if (tag.key.isNumeric()) { + continue + } + joiner.add("tag:\"") + joiner.append(tag.key) + joiner.append("\"$") + } + for (tag in tagsExclude) { + if (tag.key.isNumeric()) { + continue + } + joiner.add("-tag:\"") + joiner.append(tag.key) + joiner.append("\"$") + } + locale?.let { lc -> + joiner.add("language:\"") + joiner.append(lc.toLanguagePath()) + joiner.append("\"$") + } + if (!author.isNullOrEmpty()) { + joiner.add("artist:\"") + joiner.append(author) + joiner.append("\"$") + } + return joiner.complete().nullIfEmpty() + } + + private fun Collection.toFCats(): Int = fold(0) { acc, ct -> + val cat: Int = when (ct) { + ContentType.DOUJINSHI -> 2 + ContentType.MANGA -> 4 + ContentType.ARTIST_CG -> 8 + ContentType.GAME_CG -> 16 + ContentType.COMICS -> 512 + ContentType.IMAGE_SET -> 32 + else -> 449 // 1 or 64 or 128 or 256 + } + acc or cat + } + + private fun checkAuth(): Boolean { + val authorized = isAuthorized(DOMAIN_UNAUTHORIZED) + if (authorized) { + if (!isAuthorized(DOMAIN_AUTHORIZED)) { + context.cookieJar.copyCookies( + DOMAIN_UNAUTHORIZED, + DOMAIN_AUTHORIZED, + authCookies, + ) + context.cookieJar.insertCookies(DOMAIN_AUTHORIZED, "yay=louder") + } + return true + } + return false + } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/KdtScans.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/KdtScans.kt index 70f28585..21896c62 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/KdtScans.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/KdtScans.kt @@ -19,6 +19,7 @@ import org.koitharu.kotatsu.parsers.model.MangaTag import org.koitharu.kotatsu.parsers.model.RATING_UNKNOWN import org.koitharu.kotatsu.parsers.model.SortOrder import org.koitharu.kotatsu.parsers.util.attrAsRelativeUrl +import org.koitharu.kotatsu.parsers.util.attrOrNull import org.koitharu.kotatsu.parsers.util.generateUid import org.koitharu.kotatsu.parsers.util.mapChapters import org.koitharu.kotatsu.parsers.util.mapNotNullToSet @@ -28,6 +29,7 @@ import org.koitharu.kotatsu.parsers.util.parseFailed import org.koitharu.kotatsu.parsers.util.parseHtml import org.koitharu.kotatsu.parsers.util.selectFirstOrThrow import org.koitharu.kotatsu.parsers.util.src +import org.koitharu.kotatsu.parsers.util.textOrNull import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl import org.koitharu.kotatsu.parsers.util.toTitleCase import org.koitharu.kotatsu.parsers.util.urlEncoded @@ -36,232 +38,232 @@ import java.util.EnumSet @MangaSourceParser("KDTSCANS", "KdtScans", "en") internal class KdtScans(context: MangaLoaderContext) : - PagedMangaParser(context, MangaParserSource.KDTSCANS, 20) { + PagedMangaParser(context, MangaParserSource.KDTSCANS, 20) { - override val configKeyDomain = ConfigKey.Domain("www.silentquill.net") + override val configKeyDomain = ConfigKey.Domain("www.silentquill.net") - override val availableSortOrders: Set = EnumSet.of( - SortOrder.RELEVANCE, - SortOrder.UPDATED, - SortOrder.POPULARITY, - SortOrder.NEWEST, - SortOrder.ALPHABETICAL, - SortOrder.ALPHABETICAL_DESC, - ) + override val availableSortOrders: Set = EnumSet.of( + SortOrder.RELEVANCE, + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.NEWEST, + SortOrder.ALPHABETICAL, + SortOrder.ALPHABETICAL_DESC, + ) - override val filterCapabilities = MangaListFilterCapabilities( - isSearchSupported = true, - isMultipleTagsSupported = true, - isTagsExclusionSupported = true, - ) + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isMultipleTagsSupported = true, + isTagsExclusionSupported = true, + ) - override suspend fun getFilterOptions(): MangaListFilterOptions { - return MangaListFilterOptions( - availableTags = fetchAvailableTags(), - availableStates = EnumSet.of( - MangaState.ONGOING, - MangaState.FINISHED, - MangaState.PAUSED, - ), - availableContentTypes = EnumSet.of( - ContentType.MANGA, - ContentType.MANHWA, - ContentType.MANHUA, - ContentType.COMICS, - ContentType.NOVEL, - ), - ) - } + override suspend fun getFilterOptions(): MangaListFilterOptions { + return MangaListFilterOptions( + availableTags = fetchAvailableTags(), + availableStates = EnumSet.of( + MangaState.ONGOING, + MangaState.FINISHED, + MangaState.PAUSED, + ), + availableContentTypes = EnumSet.of( + ContentType.MANGA, + ContentType.MANHWA, + ContentType.MANHUA, + ContentType.COMICS, + ContentType.NOVEL, + ), + ) + } - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { - val url = buildString { - append("https://$domain/manga/?page=${page}") + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { + val url = buildString { + append("https://$domain/manga/?page=${page}") - filter.query?.let { - append("&s=${it.urlEncoded()}") - } + filter.query?.let { + append("&s=${it.urlEncoded()}") + } - val sortValue = when (order) { - SortOrder.UPDATED -> "update" - SortOrder.POPULARITY -> "popular" - SortOrder.NEWEST -> "latest" - SortOrder.ALPHABETICAL -> "title" - SortOrder.ALPHABETICAL_DESC -> "titlereverse" - else -> "" // Default/Relevance - } - if (sortValue.isNotEmpty()) { - append("&order=$sortValue") - } + val sortValue = when (order) { + SortOrder.UPDATED -> "update" + SortOrder.POPULARITY -> "popular" + SortOrder.NEWEST -> "latest" + SortOrder.ALPHABETICAL -> "title" + SortOrder.ALPHABETICAL_DESC -> "titlereverse" + else -> "" // Default/Relevance + } + if (sortValue.isNotEmpty()) { + append("&order=$sortValue") + } - filter.tags.forEach { tag -> - append("&genre[]=${tag.key}") - } + filter.tags.forEach { tag -> + append("&genre[]=${tag.key}") + } - filter.tagsExclude.forEach { tag -> - append("&genre[]=-${tag.key}") - } + filter.tagsExclude.forEach { tag -> + append("&genre[]=-${tag.key}") + } - filter.states.oneOrThrowIfMany().let { state -> - val stateValue = when (state) { - MangaState.ONGOING -> "ongoing" - MangaState.FINISHED -> "completed" - MangaState.PAUSED -> "hiatus" - else -> "" - } - if (stateValue.isNotEmpty()) { - append("&status=$stateValue") - } - } + filter.states.oneOrThrowIfMany().let { state -> + val stateValue = when (state) { + MangaState.ONGOING -> "ongoing" + MangaState.FINISHED -> "completed" + MangaState.PAUSED -> "hiatus" + else -> "" + } + if (stateValue.isNotEmpty()) { + append("&status=$stateValue") + } + } - filter.types.oneOrThrowIfMany()?.let { type -> - val typeValue = when (type) { - ContentType.MANGA -> "manga" - ContentType.MANHWA -> "manhwa" - ContentType.MANHUA -> "manhua" - ContentType.COMICS -> "comic" - ContentType.NOVEL -> "novel" - else -> "" - } - if (typeValue.isNotEmpty()) { - append("&type=$typeValue") - } - } - } - val doc = webClient.httpGet(url).parseHtml() - return parseMangaList(doc) - } + filter.types.oneOrThrowIfMany()?.let { type -> + val typeValue = when (type) { + ContentType.MANGA -> "manga" + ContentType.MANHWA -> "manhwa" + ContentType.MANHUA -> "manhua" + ContentType.COMICS -> "comic" + ContentType.NOVEL -> "novel" + else -> "" + } + if (typeValue.isNotEmpty()) { + append("&type=$typeValue") + } + } + } + val doc = webClient.httpGet(url).parseHtml() + return parseMangaList(doc) + } - private fun parseMangaList(doc: Document): List { - val elements = doc.select("div.listupd div.bs") + private fun parseMangaList(doc: Document): List { + val elements = doc.select("div.listupd div.bs") - if (elements.isEmpty()) { - return emptyList() - } + if (elements.isEmpty()) { + return emptyList() + } - return elements.map { div -> - val a = div.selectFirstOrThrow("a") - val href = a.attrAsRelativeUrl("href") - val img = div.selectFirst("img") - val title = a.attr("title").ifEmpty { - div.selectFirst(".tt")?.text().orEmpty() - } - val rating = div.selectFirst(".numscore")?.text()?.toFloatOrNull()?.div(10f) ?: RATING_UNKNOWN + return elements.map { div -> + val a = div.selectFirstOrThrow("a") + val href = a.attrAsRelativeUrl("href") + val img = div.selectFirst("img") + val title = a.attr("title").ifEmpty { + div.selectFirst(".tt")?.text().orEmpty() + } + val rating = div.selectFirst(".numscore")?.text()?.toFloatOrNull()?.div(10f) ?: RATING_UNKNOWN - Manga( - id = generateUid(href), - url = href, - publicUrl = href.toAbsoluteUrl(domain), - coverUrl = img?.src(), - title = title, - altTitles = emptySet(), - rating = rating, - tags = emptySet(), - authors = emptySet(), - state = parseStatus(div.selectFirst(".status")?.text().orEmpty()), - source = source, - contentRating = if (isNsfwSource) ContentRating.ADULT else null, - ) - } - } + Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + coverUrl = img?.src(), + title = title, + altTitles = emptySet(), + rating = rating, + tags = emptySet(), + authors = emptySet(), + state = parseStatus(div.selectFirst(".status")?.text().orEmpty()), + source = source, + contentRating = if (isNsfwSource) ContentRating.ADULT else null, + ) + } + } - override suspend fun getDetails(manga: Manga): Manga { - val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - val infoElement = - doc.selectFirst(".main-info, .postbody") ?: doc.parseFailed("Cannot find manga details element") + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val infoElement = + doc.selectFirst(".main-info, .postbody") ?: doc.parseFailed("Cannot find manga details element") - val statusText = - infoElement.selectFirst(".tsinfo .imptdt:contains(Status) i, .infotable tr:contains(Status) td:last-child") - ?.text() + val statusText = + infoElement.selectFirst(".tsinfo .imptdt:contains(Status) i, .infotable tr:contains(Status) td:last-child") + ?.text() - val chapters = doc.select("#chapterlist li").mapChapters(reversed = true) { i, li -> - val a = li.selectFirstOrThrow("a") - val href = a.attrAsRelativeUrl("href") - MangaChapter( - id = generateUid(href), - url = href, - title = a.selectFirst(".chapternum")?.text() ?: a.text(), - number = i + 1f, - uploadDate = parseChapterDate(li.selectFirst(".chapterdate")?.text()), - source = source, - volume = 0, - scanlator = null, - branch = null, - ) - } + val chapters = doc.select("#chapterlist li").mapChapters(reversed = true) { i, li -> + val a = li.selectFirstOrThrow("a") + val href = a.attrAsRelativeUrl("href") + MangaChapter( + id = generateUid(href), + url = href, + title = a.selectFirst(".chapternum")?.text() ?: a.text(), + number = i + 1f, + uploadDate = parseChapterDate(li.selectFirst(".chapterdate")?.text()), + source = source, + volume = 0, + scanlator = null, + branch = null, + ) + } - val genres = infoElement.select(".mgen a, .seriestugenre a").mapToSet { a -> - MangaTag( - key = a.attr("href").substringAfterLast("/").removeSuffix("/"), - title = a.text(), - source = source, - ) - } + val genres = infoElement.select(".mgen a, .seriestugenre a").mapToSet { a -> + MangaTag( + key = a.attr("href").substringAfterLast("/").removeSuffix("/"), + title = a.text(), + source = source, + ) + } - val typeTag = infoElement.selectFirst(".tsinfo .imptdt:contains(Type) a")?.text()?.let { typeText -> - MangaTag( - key = typeText.lowercase(), - title = typeText.trim(), - source = source, - ) - } + val typeTag = infoElement.selectFirst(".tsinfo .imptdt:contains(Type) a")?.text()?.let { typeText -> + MangaTag( + key = typeText.lowercase(), + title = typeText.trim(), + source = source, + ) + } - val allTags = genres.toMutableSet() - typeTag?.let { allTags.add(it) } + val allTags = genres.toMutableSet() + typeTag?.let { allTags.add(it) } - return manga.copy( - title = infoElement.selectFirst("h1.entry-title")?.text() ?: manga.title, - authors = infoElement.select(".tsinfo .imptdt:contains(Author) i, .infotable tr:contains(Author) td:last-child") - .mapToSet { it.text() }, - description = infoElement.select(".desc, .entry-content[itemprop=description]") - .joinToString("\n") { it.text() }, - state = parseStatus(statusText.orEmpty()), - tags = allTags, - chapters = chapters, - ) - } + return manga.copy( + title = infoElement.selectFirst("h1.entry-title")?.text() ?: manga.title, + authors = infoElement.select(".tsinfo .imptdt:contains(Author) i, .infotable tr:contains(Author) td:last-child") + .mapToSet { it.text() }, + description = infoElement.select(".desc, .entry-content[itemprop=description]") + .joinToString("\n") { it.text() }, + state = parseStatus(statusText.orEmpty()), + tags = allTags, + chapters = chapters, + ) + } - override suspend fun getPages(chapter: MangaChapter): List { - val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() - return doc.select("#readerarea img").map { img -> - val url = img.attr("data-src").ifEmpty { img.src().orEmpty() } - MangaPage( - id = generateUid(url), - url = url, - preview = null, - source = source, - ) - } - } + override suspend fun getPages(chapter: MangaChapter): List { + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + return doc.select("#readerarea img").map { img -> + val url = img.attr("data-src").ifEmpty { img.src().orEmpty() } + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } - private fun parseStatus(status: String): MangaState? { - return when { - status.contains("ongoing", ignoreCase = true) -> MangaState.ONGOING - status.contains("completed", ignoreCase = true) -> MangaState.FINISHED - status.contains("hiatus", ignoreCase = true) -> MangaState.PAUSED - status.contains("dropped", ignoreCase = true) -> MangaState.ABANDONED - status.contains("canceled", ignoreCase = true) -> MangaState.ABANDONED - else -> null - } - } + private fun parseStatus(status: String): MangaState? { + return when { + status.contains("ongoing", ignoreCase = true) -> MangaState.ONGOING + status.contains("completed", ignoreCase = true) -> MangaState.FINISHED + status.contains("hiatus", ignoreCase = true) -> MangaState.PAUSED + status.contains("dropped", ignoreCase = true) -> MangaState.ABANDONED + status.contains("canceled", ignoreCase = true) -> MangaState.ABANDONED + else -> null + } + } - private fun parseChapterDate(date: String?): Long { - return try { - SimpleDateFormat("MMMM dd, yyyy", sourceLocale).parse(date?.trim()).time - } catch (_: Exception) { - 0L - } - } + private fun parseChapterDate(date: String?): Long { + return try { + SimpleDateFormat("MMMM dd, yyyy", sourceLocale).parse(date?.trim()).time + } catch (_: Exception) { + 0L + } + } - private suspend fun fetchAvailableTags(): Set { - val doc = webClient.httpGet("https://$domain/manga/").parseHtml() - return doc.select("ul.genrez li").mapNotNullToSet { li -> - val key = li.selectFirst("input").attr("value") ?: return@mapNotNullToSet null - val title = li.selectFirst("label").text().toTitleCase() - MangaTag( - key = key, - title = title, - source = source, - ) - } - } + private suspend fun fetchAvailableTags(): Set { + val doc = webClient.httpGet("https://$domain/manga/").parseHtml() + return doc.select("ul.genrez li").mapNotNullToSet { li -> + val key = li.selectFirst("input")?.attrOrNull("value") ?: return@mapNotNullToSet null + val title = li.selectFirst("label")?.textOrNull()?.toTitleCase() ?: return@mapNotNullToSet null + MangaTag( + key = key, + title = title, + source = source, + ) + } + } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/es/DragonTranslationParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/es/DragonTranslationParser.kt index a32048cc..9db301a7 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/es/DragonTranslationParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/es/DragonTranslationParser.kt @@ -1,174 +1,184 @@ package org.koitharu.kotatsu.parsers.site.es +import org.koitharu.kotatsu.parsers.Broken import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaSourceParser import org.koitharu.kotatsu.parsers.config.ConfigKey import org.koitharu.kotatsu.parsers.core.PagedMangaParser import org.koitharu.kotatsu.parsers.model.* import org.koitharu.kotatsu.parsers.util.* -import org.koitharu.kotatsu.parsers.Broken import java.util.* @Broken // Website closed @MangaSourceParser("DRAGONTRANSLATION", "Dragon Translation", "es") -internal class DragonTranslationParser(context: MangaLoaderContext) : PagedMangaParser(context, MangaParserSource.DRAGONTRANSLATION, 30) { - - override val configKeyDomain = ConfigKey.Domain("dragontranslation.net") - - override fun onCreateConfig(keys: MutableCollection>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - } - - override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() - .add("referer", "no-referrer") - .build() - - override val availableSortOrders: Set = EnumSet.of(SortOrder.UPDATED) - - override val filterCapabilities: MangaListFilterCapabilities - get() = MangaListFilterCapabilities( - isSearchSupported = true, - ) - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = emptySet(), // cant find any URLs for fetch tags - availableContentTypes = EnumSet.of(ContentType.MANGA, ContentType.MANHWA, ContentType.MANHUA), - ) - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { - val url = buildString { - append("https://") - append(domain) - when { - !filter.query.isNullOrEmpty() -> { - append("/mangas?buscar=") - append(filter.query.urlEncoded()) - append("&page=") - append(page.toString()) - } - - else -> { - append("/mangas?page=") - append(page.toString()) - - val tag = filter.tags.oneOrThrowIfMany() - if (filter.tags.isNotEmpty()) { - append("&tag=") - append(tag?.key.orEmpty()) - } - - if (filter.types.isNotEmpty()) { - append("&type=") - when (filter.types.oneOrThrowIfMany()) { - ContentType.MANGA -> "manga" - ContentType.MANHWA -> "manhwa" - ContentType.MANHUA -> "manhua" - else -> "" - } - } - } - } - } - - val doc = webClient.httpGet(url).parseHtml() - val row = doc.select("div.row.gy-3").firstOrNull() ?: return emptyList() - return row.select("article.position-relative.card").mapNotNull { div -> - val href = div.selectFirst("a.lanzador")?.attrAsRelativeUrlOrNull("href") ?: return@mapNotNull null - val coverUrl = div.selectFirst("img.card-img-top.wp-post-image.lazy.loaded")?.src().orEmpty() - Manga( - id = generateUid(href), - url = href, - publicUrl = href, - coverUrl = coverUrl, - title = div.selectFirst("h2.card-title.fs-6.entry-title").text(), - altTitles = emptySet(), - rating = RATING_UNKNOWN, - tags = emptySet(), - authors = emptySet(), - state = null, - source = source, - contentRating = if (isNsfwSource) ContentRating.ADULT else null, - ) - } - } - - override suspend fun getDetails(manga: Manga): Manga { - val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - val statusText = doc.selectFirst("p:contains(Status:)")?.text() - val status = when { - statusText?.contains("publishing", ignoreCase = true) == true -> MangaState.ONGOING - else -> null - } - - val chapterElements = doc.select("ul.list-group a") - val totalChapters = chapterElements.size - - val chapters = chapterElements.mapIndexed { index, a -> - val href = a.attrAsRelativeUrl("href") - val title = a.text() - MangaChapter( - id = generateUid(href), - title = title, - number = totalChapters - index.toFloat(), - volume = 0, - url = href, - scanlator = null, - uploadDate = parseDate(a.selectFirst("span")?.text()), - branch = null, - source = source, - ) - } - - return manga.copy( - state = status, - chapters = chapters.reversed(), - ) - } - - override suspend fun getPages(chapter: MangaChapter): List { - val fullUrl = chapter.url.toAbsoluteUrl(domain) - val doc = webClient.httpGet(fullUrl).parseHtml() - return doc.select("div#chapter_imgs img").map { img -> - val url = img.attr("src") - MangaPage( - id = generateUid(url), - url = url, - preview = null, - source = source, - ) - } - } - - private fun parseDate(dateText: String?): Long { - if (dateText == null) return 0 - - val number = dateText.filter { it.isDigit() }.toIntOrNull() ?: return 0 - val now = System.currentTimeMillis() - - return when { - dateText.contains("minutos") -> { - now - (number * 60 * 1000L) - } - dateText.contains("horas") -> { - now - (number * 60 * 60 * 1000L) - } - dateText.contains("días") -> { - now - (number * 24 * 60 * 60 * 1000L) - } - dateText.contains("día") -> { - now - (number * 24 * 60 * 60 * 1000L) - } - dateText.contains("semanas") -> { - now - (number * 7 * 24 * 60 * 60 * 1000L) - } - dateText.contains("meses") -> { - now - (number * 30 * 24 * 60 * 60 * 1000L) - } - dateText.contains("años") -> { - now - (number * 365 * 24 * 60 * 60 * 1000L) - } - else -> 0L - } - } +internal class DragonTranslationParser(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.DRAGONTRANSLATION, 30) { + + override val configKeyDomain = ConfigKey.Domain("dragontranslation.net") + + override fun onCreateConfig(keys: MutableCollection>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() + .add("referer", "no-referrer") + .build() + + override val availableSortOrders: Set = EnumSet.of(SortOrder.UPDATED) + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isSearchSupported = true, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = emptySet(), // cant find any URLs for fetch tags + availableContentTypes = EnumSet.of(ContentType.MANGA, ContentType.MANHWA, ContentType.MANHUA), + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { + val url = buildString { + append("https://") + append(domain) + when { + !filter.query.isNullOrEmpty() -> { + append("/mangas?buscar=") + append(filter.query.urlEncoded()) + append("&page=") + append(page.toString()) + } + + else -> { + append("/mangas?page=") + append(page.toString()) + + val tag = filter.tags.oneOrThrowIfMany() + if (filter.tags.isNotEmpty()) { + append("&tag=") + append(tag?.key.orEmpty()) + } + + if (filter.types.isNotEmpty()) { + append("&type=") + append( + when (filter.types.oneOrThrowIfMany()) { + ContentType.MANGA -> "manga" + ContentType.MANHWA -> "manhwa" + ContentType.MANHUA -> "manhua" + else -> "" + }, + ) + } + } + } + } + + val doc = webClient.httpGet(url).parseHtml() + val row = doc.select("div.row.gy-3").firstOrNull() ?: return emptyList() + return row.select("article.position-relative.card").mapNotNull { div -> + val href = div.selectFirst("a.lanzador")?.attrAsRelativeUrlOrNull("href") ?: return@mapNotNull null + val coverUrl = div.selectFirst("img.card-img-top.wp-post-image.lazy.loaded")?.src().orEmpty() + Manga( + id = generateUid(href), + url = href, + publicUrl = href, + coverUrl = coverUrl, + title = div.selectFirst("h2.card-title.fs-6.entry-title")?.text().orEmpty(), + altTitles = emptySet(), + rating = RATING_UNKNOWN, + tags = emptySet(), + authors = emptySet(), + state = null, + source = source, + contentRating = if (isNsfwSource) ContentRating.ADULT else null, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val statusText = doc.selectFirst("p:contains(Status:)")?.text() + val status = when { + statusText?.contains("publishing", ignoreCase = true) == true -> MangaState.ONGOING + else -> null + } + + val chapterElements = doc.select("ul.list-group a") + val totalChapters = chapterElements.size + + val chapters = chapterElements.mapIndexed { index, a -> + val href = a.attrAsRelativeUrl("href") + val title = a.text() + MangaChapter( + id = generateUid(href), + title = title, + number = totalChapters - index.toFloat(), + volume = 0, + url = href, + scanlator = null, + uploadDate = parseDate(a.selectFirst("span")?.text()), + branch = null, + source = source, + ) + } + + return manga.copy( + state = status, + chapters = chapters.reversed(), + ) + } + + override suspend fun getPages(chapter: MangaChapter): List { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + return doc.select("div#chapter_imgs img").map { img -> + val url = img.attr("src") + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } + + private fun parseDate(dateText: String?): Long { + if (dateText == null) return 0 + + val number = dateText.filter { it.isDigit() }.toIntOrNull() ?: return 0 + val now = System.currentTimeMillis() + + return when { + dateText.contains("minutos") -> { + now - (number * 60 * 1000L) + } + + dateText.contains("horas") -> { + now - (number * 60 * 60 * 1000L) + } + + dateText.contains("días") -> { + now - (number * 24 * 60 * 60 * 1000L) + } + + dateText.contains("día") -> { + now - (number * 24 * 60 * 60 * 1000L) + } + + dateText.contains("semanas") -> { + now - (number * 7 * 24 * 60 * 60 * 1000L) + } + + dateText.contains("meses") -> { + now - (number * 30 * 24 * 60 * 60 * 1000L) + } + + dateText.contains("años") -> { + now - (number * 365 * 24 * 60 * 60 * 1000L) + } + + else -> 0L + } + } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/pt/LuratoonScansParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/pt/LuratoonScansParser.kt index 81ce3582..7c61dc3a 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/pt/LuratoonScansParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/pt/LuratoonScansParser.kt @@ -20,143 +20,143 @@ import java.util.zip.ZipInputStream @Broken // Not dead but changed template @MangaSourceParser("RANDOMSCANS", "LuratoonScan", "pt") internal class LuratoonScansParser(context: MangaLoaderContext) : - SinglePageMangaParser(context, MangaParserSource.RANDOMSCANS), - Interceptor { + SinglePageMangaParser(context, MangaParserSource.RANDOMSCANS), + Interceptor { - override val configKeyDomain = ConfigKey.Domain("luratoons.net") + override val configKeyDomain = ConfigKey.Domain("luratoons.net") - override fun getRequestHeaders(): Headers = Headers.Builder().add("User-Agent", config[userAgentKey]).build() + override fun getRequestHeaders(): Headers = Headers.Builder().add("User-Agent", config[userAgentKey]).build() - override val availableSortOrders = setOf(SortOrder.ALPHABETICAL) + override val availableSortOrders = setOf(SortOrder.ALPHABETICAL) - override val filterCapabilities: MangaListFilterCapabilities - get() = MangaListFilterCapabilities() + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities() - override suspend fun getFilterOptions() = MangaListFilterOptions() + override suspend fun getFilterOptions() = MangaListFilterOptions() - override suspend fun getList(order: SortOrder, filter: MangaListFilter): List { - require(filter.query.isNullOrEmpty()) { ErrorMessages.SEARCH_NOT_SUPPORTED } - val url = urlBuilder() - val tag = filter.tags.oneOrThrowIfMany() - if (tag == null) { - url.addPathSegment("todas-as-obras") - } else { - url.addPathSegment("pesquisar").addQueryParameter("category", tag.key) - } - val doc = webClient.httpGet(url.build()).parseHtml() - return doc.selectFirstOrThrow(".todas__as__obras").select(".comics__all__box").map { div -> - val a = div.selectFirstOrThrow("a") - val href = a.attrAsRelativeUrl("href") - Manga( - id = generateUid(href), - url = href, - publicUrl = href.toAbsoluteUrl(div.host ?: domain), - coverUrl = div.selectFirst("img")?.src().orEmpty(), - title = div.text(), - altTitles = emptySet(), - rating = RATING_UNKNOWN, - tags = emptySet(), - authors = emptySet(), - state = null, - source = source, - contentRating = null, - ) - } - } + override suspend fun getList(order: SortOrder, filter: MangaListFilter): List { + require(filter.query.isNullOrEmpty()) { ErrorMessages.SEARCH_NOT_SUPPORTED } + val url = urlBuilder() + val tag = filter.tags.oneOrThrowIfMany() + if (tag == null) { + url.addPathSegment("todas-as-obras") + } else { + url.addPathSegment("pesquisar").addQueryParameter("category", tag.key) + } + val doc = webClient.httpGet(url.build()).parseHtml() + return doc.selectFirstOrThrow(".todas__as__obras").select(".comics__all__box").map { div -> + val a = div.selectFirstOrThrow("a") + val href = a.attrAsRelativeUrl("href") + Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(div.host ?: domain), + coverUrl = div.selectFirst("img")?.src().orEmpty(), + title = div.text(), + altTitles = emptySet(), + rating = RATING_UNKNOWN, + tags = emptySet(), + authors = emptySet(), + state = null, + source = source, + contentRating = null, + ) + } + } - override suspend fun getDetails(manga: Manga): Manga { - val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml().body() - val summaryContainer = doc.selectFirstOrThrow(".sumario__container") - // 1 de Maio de 2024 às 20:15 - val dateFormat = SimpleDateFormat("dd 'de' MMM 'de' YYYY 'às' HH:mm", sourceLocale) - val author = summaryContainer.getElementsContainingOwnText("Autor(es)").firstOrNull() - ?.nextElementSibling()?.textOrNull() - return manga.copy( - title = doc.selectFirst("h1.desc__titulo__comic")?.textOrNull() ?: manga.title, - altTitles = setOfNotNull( - summaryContainer.getElementsContainingOwnText("Alternativo").firstOrNull() - ?.nextElementSibling()?.textOrNull(), - ), - tags = summaryContainer.getElementsByAttributeValueContaining("href", "?category=").mapToSet { - MangaTag( - title = it.text().toTitleCase(sourceLocale), - key = it.attr("href").substringAfterLast('='), - source = source, - ) - }, - state = when (summaryContainer.getElementsContainingOwnText("Status").firstOrNull() - ?.nextElementSibling()?.text()?.lowercase()) { - "em lançamento" -> MangaState.ONGOING - "hiato" -> MangaState.PAUSED - "finalizado" -> MangaState.FINISHED - else -> null - }, - authors = setOfNotNull(author), - largeCoverUrl = doc.selectFirst("img.sumario__img")?.attrAsAbsoluteUrlOrNull("src"), - description = summaryContainer.selectFirst(".sumario__sinopse__texto")?.html(), - chapters = doc.selectFirstOrThrow("ul.capitulos__lista") - .select("li") - .mapChapters(reversed = true) { _, li -> - val href = li.parent()?.attrAsRelativeUrlOrNull("href") ?: return@mapChapters null - val span = li.selectFirstOrThrow(".numero__capitulo") - MangaChapter( - id = generateUid(href), - title = span.text(), - number = 0.0f, - volume = 0, - url = href, - scanlator = null, - uploadDate = dateFormat.parseSafe(span.nextElementSibling()?.text()), - branch = null, - source = source, - ) - }, - ) - } + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml().body() + val summaryContainer = doc.selectFirstOrThrow(".sumario__container") + // 1 de Maio de 2024 às 20:15 + val dateFormat = SimpleDateFormat("dd 'de' MMM 'de' YYYY 'às' HH:mm", sourceLocale) + val author = summaryContainer.getElementsContainingOwnText("Autor(es)").firstOrNull() + ?.nextElementSibling()?.textOrNull() + return manga.copy( + title = doc.selectFirst("h1.desc__titulo__comic")?.textOrNull() ?: manga.title, + altTitles = setOfNotNull( + summaryContainer.getElementsContainingOwnText("Alternativo").firstOrNull() + ?.nextElementSibling()?.textOrNull(), + ), + tags = summaryContainer.getElementsByAttributeValueContaining("href", "?category=").mapToSet { + MangaTag( + title = it.text().toTitleCase(sourceLocale), + key = it.attr("href").substringAfterLast('='), + source = source, + ) + }, + state = when (summaryContainer.getElementsContainingOwnText("Status").firstOrNull() + ?.nextElementSibling()?.text()?.lowercase()) { + "em lançamento" -> MangaState.ONGOING + "hiato" -> MangaState.PAUSED + "finalizado" -> MangaState.FINISHED + else -> null + }, + authors = setOfNotNull(author), + largeCoverUrl = doc.selectFirst("img.sumario__img")?.attrAsAbsoluteUrlOrNull("src"), + description = summaryContainer.selectFirst(".sumario__sinopse__texto")?.html(), + chapters = doc.selectFirstOrThrow("ul.capitulos__lista") + .select("li") + .mapChapters(reversed = true) { _, li -> + val href = li.parent()?.attrAsRelativeUrlOrNull("href") ?: return@mapChapters null + val span = li.selectFirstOrThrow(".numero__capitulo") + MangaChapter( + id = generateUid(href), + title = span.text(), + number = 0.0f, + volume = 0, + url = href, + scanlator = null, + uploadDate = dateFormat.parseSafe(span.nextElementSibling()?.text()), + branch = null, + source = source, + ) + }, + ) + } - override suspend fun getPages(chapter: MangaChapter): List { - val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() - val regex = Regex("const\\s+urls\\s*=\\s*(\\[.*])") - val urls = doc.select("script").firstNotNullOf { - regex.find(it.data())?.groupValues?.getOrNull(1) - } - val ja = JSONArray(urls) - return (0 until ja.length()).map { i -> - val url = ja.getString(i) - MangaPage( - id = generateUid(url), - url = url, - preview = null, - source = source, - ) - } - } + override suspend fun getPages(chapter: MangaChapter): List { + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + val regex = Regex("const\\s+urls\\s*=\\s*(\\[.*])") + val urls = doc.select("script").firstNotNullOf { + regex.find(it.data())?.groupValues?.getOrNull(1) + } + val ja = JSONArray(urls) + return (0 until ja.length()).map { i -> + val url = ja.getString(i) + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } - override fun intercept(chain: Interceptor.Chain): Response { - val response = chain.proceed(chain.request()) - if (response.mimeType == "application/octet-stream") { - val (bytes, name) = response.use { resp -> - ZipInputStream(resp.requireBody().byteStream()).use { - val entry = it.nextEntry - it.readBytes() to entry?.name - } - } - val type = if (name?.endsWith(".avif", ignoreCase = true) == true) { - "image/avif" - } else { - "image/*" - }.toMediaTypeOrNull() - return response.newBuilder() - .setHeader("Content-Type", type?.toString()) - .body(bytes.toResponseBody(type)) - .build() - } else { - return response - } - } + override fun intercept(chain: Interceptor.Chain): Response { + val response = chain.proceed(chain.request()) + if (response.mimeType == "application/octet-stream") { + val (bytes, name) = response.use { resp -> + ZipInputStream(resp.body.byteStream()).use { + val entry = it.nextEntry + it.readBytes() to entry?.name + } + } + val type = if (name?.endsWith(".avif", ignoreCase = true) == true) { + "image/avif" + } else { + "image/*" + }.toMediaTypeOrNull() + return response.newBuilder() + .setHeader("Content-Type", type?.toString()) + .body(bytes.toResponseBody(type)) + .build() + } else { + return response + } + } - override fun onCreateConfig(keys: MutableCollection>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - } + override fun onCreateConfig(keys: MutableCollection>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ru/grouple/GroupleParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ru/grouple/GroupleParser.kt index c8ca941d..db49b772 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ru/grouple/GroupleParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ru/grouple/GroupleParser.kt @@ -12,7 +12,6 @@ import okhttp3.HttpUrl.Companion.toHttpUrlOrNull import okhttp3.Interceptor import okhttp3.Response import okhttp3.internal.closeQuietly -import okhttp3.internal.headersContentLength import okio.IOException import org.json.JSONArray import org.jsoup.nodes.Element @@ -41,540 +40,541 @@ private const val COPYRIGHT_ALERT = "Запрещена публикация п private const val NO_CHAPTERS = "В этой манге еще нет ни одной главы" internal abstract class GroupleParser( - context: MangaLoaderContext, - source: MangaParserSource, - private val siteId: Int, + context: MangaLoaderContext, + source: MangaParserSource, + private val siteId: Int, ) : AbstractMangaParser(context, source), MangaParserAuthProvider, Interceptor { - @Volatile - private var cachedPagesServer: String? = null - - override val userAgentKey = ConfigKey.UserAgent( - "Mozilla/5.0 (X11; U; UNICOS lcLinux; en-US) Gecko/20140730 (KHTML, like Gecko, Safari/419.3) Arora/0.8.0", - ) - private val splitTranslationsKey = ConfigKey.SplitByTranslations(false) - private val tagsIndex = suspendLazy(initializer = ::fetchTagsMap) - - override fun getRequestHeaders(): Headers = Headers.Builder() - .add("User-Agent", config[userAgentKey]) - .add("Accept-Language", "ru,en-US;q=0.7,en;q=0.3") - .build() - - override val availableSortOrders: Set = EnumSet.of( - SortOrder.UPDATED, - SortOrder.POPULARITY, - SortOrder.NEWEST, - SortOrder.RATING, - SortOrder.ALPHABETICAL, - SortOrder.ADDED, - ) - - override val authUrl: String - get() { - val targetUri = "https://${domain}/".urlEncoded() - return "https://3.grouple.co/internal/auth/sso?siteId=$siteId&=targetUri=$targetUri" - } - - override suspend fun isAuthorized(): Boolean = hasAuthCookie() - - override val filterCapabilities: MangaListFilterCapabilities - get() = MangaListFilterCapabilities( - isMultipleTagsSupported = true, - isTagsExclusionSupported = true, - isSearchSupported = true, - isSearchWithFiltersSupported = true, - isYearRangeSupported = true, - ) - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = fetchAvailableTags(), - availableStates = EnumSet.of(MangaState.FINISHED, MangaState.ABANDONED, MangaState.UPCOMING), - ) - - override suspend fun getList(offset: Int, order: SortOrder, filter: MangaListFilter): List { - val domain = domain - val root = if (filter.isEmpty()) { - webClient.httpGet( - "https://$domain/list?sortType=${ - getSortKey(order) - }&offset=${offset upBy PAGE_SIZE}", - ).parseHtml().body().let { doc -> (doc.getElementById("mangaBox") ?: doc.getElementById("mangaResults")) } - } else { - advancedSearch(offset, order, filter).parseHtml() - } - val tiles = root.selectFirst("div.tiles.row") - if (tiles == null) { - if (!root.getElementsContainingOwnText(NOTHING_FOUND).isNullOrEmpty()) { - return emptyList() - } - root.parseFailed("No tiles found") - } - return tiles.select("div.tile").mapNotNull(::parseManga) - } - - override suspend fun getDetails(manga: Manga): Manga { - val response = webClient.httpGet(manga.url.toAbsoluteUrl(domain)) - val doc = response.parseHtml() - val root = doc.body().requireElementById("mangaBox").run { - selectFirst("div.leftContent") ?: this - } - val dateFormat = SimpleDateFormat("dd.MM.yy", Locale.US) - val coverImg = root.selectFirst("div.subject-cover")?.selectFirst("img") - val translations = if (config[splitTranslationsKey]) { - root.selectFirst("div.translator-selection") - ?.select(".translator-selection-item") - ?.associate { - it.id().removePrefix("tr-").toLong() to it.selectFirst(".translator-selection-name")?.textOrNull() - } - } else { - null - } - val newSource = getSource(response.request.url) - val chaptersList = root.getElementById("chapters-list") - var isRestricted = false - if (chaptersList == null && root.getElementsContainingOwnText(NO_CHAPTERS).isEmpty()) { - if (root.getElementsContainingOwnText(COPYRIGHT_ALERT).isNotEmpty()) { - isRestricted = true - } else { - root.parseFailed("No chapters found") - } - } - val hashRegex = Regex("window.user_hash\\s*=\\s*\'([^\']+)\'") - val userHash = doc.select("script").firstNotNullOfOrNull { it.html().findGroupValue(hashRegex) } - val hasNsfwAlert = root.select(".alert-warning").any { it.ownText().contains(NSFW_ALERT) } - return manga.copy( - source = newSource, - title = doc.metaValue("name") ?: manga.title, - altTitles = root.selectFirst(".all-names-popover")?.select(".name")?.mapNotNullToSet { - it.textOrNull() - } ?: manga.altTitles, - publicUrl = response.request.url.toString(), - description = root.selectFirst("div.manga-description")?.html(), - largeCoverUrl = coverImg?.attrAsAbsoluteUrlOrNull("data-full"), - coverUrl = manga.coverUrl - ?: coverImg?.attrAsAbsoluteUrlOrNull("data-thumb")?.replace("_p.", "."), - tags = root.selectFirstOrThrow("div.subject-meta") - .getElementsByAttributeValueContaining("href", "/list/genre/").mapTo(manga.tags.toMutableSet()) { a -> - MangaTag( - title = a.text().toTitleCase(), - key = a.attr("href").removeSuffix('/').substringAfterLast('/'), - source = source, - ) - }, - state = if (isRestricted) { - MangaState.RESTRICTED - } else { - manga.state - }, - authors = root.select(".elem_author,.elem_illustrator,.elem_screenwriter") - .select("a.person-link") - .mapNotNullToSet { it.textOrNull() } + manga.authors, - contentRating = (if (hasNsfwAlert) ContentRating.SUGGESTIVE else ContentRating.SAFE) - .coerceAtLeast(manga.contentRating ?: ContentRating.SAFE), - chapters = chaptersList?.select("a.chapter-link") - ?.flatMapChapters(reversed = true) { a -> - val tr = a.selectFirstParent("tr") ?: return@flatMapChapters emptyList() - val href = a.attrAsRelativeUrl("href") - val number = tr.attr("data-num").toFloatOrNull()?.div(10f) ?: 0f - val volume = tr.attr("data-vol").toIntOrNull() ?: 0 - if (translations.isNullOrEmpty() || a.attr("data-translations").isEmpty()) { - var translators = "" - val translatorElement = a.attr("title") - if (!translatorElement.isNullOrBlank()) { - translators = translatorElement.replace("(Переводчик),", "&").removeSuffix(" (Переводчик)") - } - listOf( - MangaChapter( - id = generateUid(href), - title = a.text().removePrefix(manga.title).trim().nullIfEmpty(), - number = number, - volume = volume, - url = href.withQueryParam("d", userHash), - uploadDate = dateFormat.parseSafe(tr.selectFirst("td.date")?.text()), - scanlator = translators, - source = newSource, - branch = null, - ), - ) - } else { - val translationData = JSONArray(a.attr("data-translations")) - translationData.mapJSON { jo -> - val personId = jo.getLong("personId") - val link = href.withQueryParam("tran", personId.toString()) - MangaChapter( - id = generateUid(link), - title = a.text().removePrefix(manga.title).trim(), - number = number, - volume = volume, - url = link.withQueryParam("d", userHash), - uploadDate = dateFormat.parseSafe(jo.getStringOrNull("dateCreated")), - scanlator = null, - source = newSource, - branch = translations[personId], - ) - } - } - }.orEmpty(), - ) - } - - override suspend fun getPages(chapter: MangaChapter): List { - if (chapter.source != source && chapter.source is MangaParserSource) { // handle redirects between websites - return context.newParserInstance(chapter.source).getPages(chapter) - } - val url = chapter.url.toAbsoluteUrl(domain).toHttpUrl().newBuilder().setQueryParameter("mtr", "1").build() - val doc = webClient.httpGet(url).parseHtml() - val scripts = doc.select("script") - for (script in scripts) { - val data = script.html() - var pos = data.indexOf("rm_h.readerDoInit(") - if (pos != -1) { - parsePagesV2(data, pos)?.let { return it } - } - pos = data.indexOf("rm_h.readerInit( 0,") - if (pos != -1) { - parsePagesV1(data, pos)?.let { return it } - } - pos = data.indexOf(".readerInit(") - if (pos != -1) { - parsePagesV3(data, pos).let { return it } - } - } - doc.parseFailed("Pages list not found at ${chapter.url}") - } - - override suspend fun getPageUrl(page: MangaPage): String { - val parts = page.url.split('|') - if (parts.size < 2) { - return page.url - } - val path = parts.last() - // fast path - cachedPagesServer?.let { host -> - val url = concatUrl("https://$host/", path) - if (tryHead(url)) { - return url - } else { - cachedPagesServer = null - } - } - // slow path - val candidates = HashSet((parts.size - 1) * 2) - for (i in 0 until parts.size - 1) { - val server = parts[i].trim().ifEmpty { "https://$domain/" } - candidates.add(concatUrl(server, path)) - candidates.add(concatUrl(server, path.substringBeforeLast('?'))) - } - return try { - channelFlow { - for (url in candidates) { - launch { - if (tryHead(url)) { - send(url) - } - } - } - }.first().also { - cachedPagesServer = it.toHttpUrlOrNull()?.host - } - } catch (e: NoSuchElementException) { - candidates.randomOrNull() ?: throw ParseException("No page url candidates", page.url, e) - } - } - - private suspend fun fetchAvailableTags(): Set { - val doc = webClient.httpGet("https://${domain}/list/genres/sort_name").parseHtml() - val root = doc.body().getElementById("mangaBox")?.selectFirst("div.leftContent")?.selectFirst("table.table") - ?: doc.parseFailed("Cannot find root") - return root.select("a.element-link").mapToSet { a -> - MangaTag( - title = a.text().toTitleCase(), - key = a.attr("href").substringAfterLast('/'), - source = source, - ) - } - } - - override suspend fun getUsername(): String { - val root = webClient.httpGet("https://grouple.co/").parseHtml().body() - val element = root.selectFirst("img.user-avatar") ?: throw AuthRequiredException(source) - val res = element.parent()?.text() - return if (res.isNullOrEmpty()) { - root.parseFailed("Cannot find username") - } else res - } - - override fun intercept(chain: Interceptor.Chain): Response { - val request = chain.request() - if (!request.header(HEADER_ACCEPT).isNullOrEmpty()) { - return chain.proceed(request).checkIfAuthRequired() - } - val ext = request.url.pathSegments.lastOrNull()?.substringAfterLast('.', "")?.lowercase(Locale.ROOT) - return if (ext == "jpg" || ext == "jpeg" || ext == "png" || ext == "webp") { - chain.proceed( - request.newBuilder().header(HEADER_ACCEPT, "image/webp,image/png;q=0.9,image/jpeg,*/*;q=0.8").build(), - ) - } else { - chain.proceed(request).checkIfAuthRequired() - } - } - - override fun onCreateConfig(keys: MutableCollection>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - keys.add(splitTranslationsKey) - } - - override suspend fun getRelatedManga(seed: Manga): List { - val doc = webClient.httpGet(seed.url.toAbsoluteUrl(domain)).parseHtml() - val root = doc.body().requireElementById("mangaBox").select("h4").first { it.ownText() == RELATED_TITLE } - .nextElementSibling() ?: doc.parseFailed("Cannot find root") - return root.select("div.tile").mapNotNull(::parseManga) - } - - protected open fun getSource(url: HttpUrl): MangaSource = when (url.host) { - in SeiMangaParser.domains -> MangaParserSource.SEIMANGA - in MintMangaParser.domains -> MangaParserSource.MINTMANGA - in ReadmangaParser.domains -> MangaParserSource.READMANGA_RU - in SelfMangaParser.domains -> MangaParserSource.SELFMANGA - in UsagiParser.domains -> MangaParserSource.USAGI - else -> source - } - - private fun getSortKey(sortOrder: SortOrder) = when (sortOrder) { - SortOrder.ALPHABETICAL -> "name" - SortOrder.POPULARITY -> "rate" - SortOrder.UPDATED -> "updated" - SortOrder.ADDED, - SortOrder.NEWEST, - -> "created" - - SortOrder.RATING -> "votes" - else -> "rate" - } - - private suspend fun advancedSearch(offset: Int, order: SortOrder, filter: MangaListFilter): Response { - val tagsMap = tagsIndex.get() - val url = urlBuilder() - .addPathSegment("search") - .addPathSegment("advancedResults") - url.addQueryParameter("q", filter.query) - url.addQueryParameter("offset", offset.toString()) - filter.tags.forEach { tag -> - val tagId = requireNotNull(tagsMap[tag.title.lowercase()]) { "Tag ${tag.title} not found" } - url.addQueryParameter(tagId, "in") - } - filter.tagsExclude.forEach { tag -> - val tagId = requireNotNull(tagsMap[tag.title.lowercase()]) { "Tag ${tag.title} not found" } - url.addQueryParameter(tagId, "ex") - } - url.addQueryParameter( - "years", - buildString { - append(filter.yearFrom.ifZero { YEAR_MIN }) - append(',') - append(filter.yearTo.ifZero { YEAR_MAX }) - }, - ) - url.addQueryParameter( - "sortType", - when (order) { - SortOrder.RATING -> "USER_RATING" - SortOrder.ALPHABETICAL -> "NAME" - SortOrder.ADDED -> "YEAR" - SortOrder.POPULARITY -> "POPULARITY" - SortOrder.NEWEST -> "DATE_CREATE" - SortOrder.UPDATED -> "DATE_UPDATE" - else -> "RATING" - }, - ) - filter.states.forEach { state -> - when (state) { - MangaState.FINISHED -> "s_completed" - MangaState.ABANDONED -> "s_abandoned_popular" - MangaState.UPCOMING -> "s_wait_upload" - else -> null - }?.let { - url.addQueryParameter(it, "in") - } - } - - return webClient.httpGet(url.build()) - } - - private suspend fun tryHead(url: String): Boolean = runCatchingCancellable { - webClient.httpHead(url).use { response -> - response.isSuccessful && !response.isPumpkin() && response.headersContentLength() >= MIN_IMAGE_SIZE - } - }.getOrDefault(false) - - private fun Response.isPumpkin(): Boolean = request.url.host == "upload.wikimedia.org" - - private fun parseManga(node: Element): Manga? { - val imgDiv = node.selectFirst("div.img") ?: return null - val descDiv = node.selectFirst("div.desc") ?: return null - if (descDiv.selectFirst("i.fa-user") != null || descDiv.selectFirst("i.fa-external-link") != null) { - return null // skip author - } - val href = imgDiv.selectFirst("a")?.attrAsAbsoluteUrlOrNull("href") ?: return null - val title = descDiv.selectFirst("h3")?.selectFirst("a")?.text() ?: return null - val tileInfo = descDiv.selectFirst("div.tile-info") - val relUrl = href.toRelativeUrl(domain) - if (relUrl.contains("://")) { - return null - } - val author = tileInfo?.selectFirst("a.person-link")?.text() - return Manga( - id = generateUid(relUrl), - url = relUrl, - publicUrl = href, - title = title, - altTitles = setOfNotNull(descDiv.selectFirst("h5")?.textOrNull()), - coverUrl = imgDiv.selectFirst("img.lazy")?.attrAsAbsoluteUrlOrNull("data-original")?.replace("_p.", "."), - rating = runCatching { - node.selectFirst(".compact-rate")?.attr("title")?.toFloatOrNull()?.div(5f) - }.getOrNull() ?: RATING_UNKNOWN, - authors = setOfNotNull(author), - contentRating = if (isNsfwSource) ContentRating.ADULT else null, - tags = runCatching { - tileInfo?.select("a.element-link")?.mapToSet { - MangaTag( - title = it.text().toTitleCase(), - key = it.attr("href").substringAfterLast('/'), - source = source, - ) - } - }.getOrNull().orEmpty(), - state = when { - node.selectFirst("div.tags")?.selectFirst("span.mangaCompleted") != null -> MangaState.FINISHED - - else -> null - }, - source = source, - ) - } - - private fun parsePagesV1(data: String, pos: Int): List? { - val json = data.substring(pos).substringAfter('(').substringBefore('\n').substringBeforeLast(')') - if (json.isEmpty()) { - return null - } - val ja = JSONArray("[$json]") - val pages = ja.getJSONArray(1) - val servers = ja.getJSONArray(3).mapJSON { it.getString("path") } - val serversStr = servers.joinToString("|") - return (0 until pages.length()).map { i -> - val page = pages.getJSONArray(i) - val primaryServer = page.getString(0) - val url = page.getString(2) - val fullSrc = if ("$primaryServer|$serversStr|$url".contains("one-way.work")) { - // domain that does not need a token - "$primaryServer|$serversStr|${url}".substringBefore("?") - } else { - "$primaryServer|$serversStr|$url" - } - MangaPage( - id = generateUid(url), - url = fullSrc, - preview = null, - source = source, - ) - } - } - - private fun parsePagesV2(data: String, pos: Int): List? { - val json = data.substring(pos).substringAfter('(').substringBefore('\n').substringBeforeLast(')') - if (json.isEmpty()) { - return null - } - val ja = JSONArray("[$json]") - val pages = ja.getJSONArray(0) - val servers = ja.getJSONArray(2).mapJSON { it.getString("path") } - val serversStr = servers.joinToString("|") - return (0 until pages.length()).map { i -> - val page = pages.getJSONArray(i) - val primaryServer = page.getString(0) - val url = page.getString(2) - val fullSrc = if ("$primaryServer|$serversStr|$url".contains("one-way.work")) { - // domain that does not need a token - "$primaryServer|$serversStr|${url}".substringBefore("?") - } else { - "$primaryServer|$serversStr|$url" - } - MangaPage( - id = generateUid(url), - url = fullSrc, - preview = null, - source = source, - ) - } - } - - private fun parsePagesV3(data: String, pos: Int): List { - val json = JSONArray(data.substring(pos).substringBetween("(", ")").substringBeforeLast(',')) - return (0 until json.length()).map { i -> - val ja = json.getJSONArray(i) - val server = ja.getString(0).ifEmpty { "https://$domain" } - val url = ja.getString(2) - val fullUrl = concatUrl(server, url) - MangaPage( - id = generateUid(url), - url = if (fullUrl.contains("one-way.work")) { - // domain that does not need a token - fullUrl.substringBefore("?") - } else { - fullUrl - }, - preview = null, - source = source, - ) - } - } - - private suspend fun fetchTagsMap(): ScatterMap { - val url = "https://$domain/search/advanced" - val properties = - webClient.httpGet(url).parseHtml().body().selectFirst("form.search-form")?.select("div.form-group") - ?.find { it.selectFirst("li.property") != null } - ?.select("li.property") - ?: throw ParseException("Genres filter element not found", url) - val result = MutableScatterMap(properties.size) - properties.forEach { li -> - val name = li.text().lowercase() - val id = li.selectFirstOrThrow("input").id() - result[name] = id - } - return result - } - - private fun String.withQueryParam(name: String, value: String?): String { - if (value == null) return this - return toAbsoluteUrl(domain) - .toHttpUrl() - .newBuilder() - .setQueryParameter(name, value) - .build() - .toString() - .toRelativeUrl(domain) - } - - @Throws(IOException::class) - private fun Response.checkIfAuthRequired(): Response { - val lastPathSegment = request.url.pathSegments.lastOrNull() - if (lastPathSegment == "login") { - closeQuietly() - throw AuthRequiredException(source) - } - if (code == HttpURLConnection.HTTP_NOT_FOUND) { - if (!hasAuthCookie()) { - closeQuietly() - throw AuthRequiredException(source) - } else { - return newBuilder().code(HttpURLConnection.HTTP_OK).build() - } - } - return this - } - - private fun hasAuthCookie() = context.cookieJar.getCookies(domain).any { it.name == "gwt" } + @Volatile + private var cachedPagesServer: String? = null + + override val userAgentKey = ConfigKey.UserAgent( + "Mozilla/5.0 (X11; U; UNICOS lcLinux; en-US) Gecko/20140730 (KHTML, like Gecko, Safari/419.3) Arora/0.8.0", + ) + private val splitTranslationsKey = ConfigKey.SplitByTranslations(false) + private val tagsIndex = suspendLazy(initializer = ::fetchTagsMap) + + override fun getRequestHeaders(): Headers = Headers.Builder() + .add("User-Agent", config[userAgentKey]) + .add("Accept-Language", "ru,en-US;q=0.7,en;q=0.3") + .build() + + override val availableSortOrders: Set = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.NEWEST, + SortOrder.RATING, + SortOrder.ALPHABETICAL, + SortOrder.ADDED, + ) + + override val authUrl: String + get() { + val targetUri = "https://${domain}/".urlEncoded() + return "https://3.grouple.co/internal/auth/sso?siteId=$siteId&=targetUri=$targetUri" + } + + override suspend fun isAuthorized(): Boolean = hasAuthCookie() + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isMultipleTagsSupported = true, + isTagsExclusionSupported = true, + isSearchSupported = true, + isSearchWithFiltersSupported = true, + isYearRangeSupported = true, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = fetchAvailableTags(), + availableStates = EnumSet.of(MangaState.FINISHED, MangaState.ABANDONED, MangaState.UPCOMING), + ) + + override suspend fun getList(offset: Int, order: SortOrder, filter: MangaListFilter): List { + val domain = domain + val root = if (filter.isEmpty()) { + webClient.httpGet( + "https://$domain/list?sortType=${ + getSortKey(order) + }&offset=${offset upBy PAGE_SIZE}", + ).parseHtml().body().let { doc -> (doc.getElementById("mangaBox") ?: doc.getElementById("mangaResults")) } + } else { + advancedSearch(offset, order, filter).parseHtml() + } + checkNotNull(root) { "Root not found" } + val tiles = root.selectFirst("div.tiles.row") + if (tiles == null) { + if (root.getElementsContainingOwnText(NOTHING_FOUND).isNotEmpty()) { + return emptyList() + } + root.parseFailed("No tiles found") + } + return tiles.select("div.tile").mapNotNull(::parseManga) + } + + override suspend fun getDetails(manga: Manga): Manga { + val response = webClient.httpGet(manga.url.toAbsoluteUrl(domain)) + val doc = response.parseHtml() + val root = doc.body().requireElementById("mangaBox").run { + selectFirst("div.leftContent") ?: this + } + val dateFormat = SimpleDateFormat("dd.MM.yy", Locale.US) + val coverImg = root.selectFirst("div.subject-cover")?.selectFirst("img") + val translations = if (config[splitTranslationsKey]) { + root.selectFirst("div.translator-selection") + ?.select(".translator-selection-item") + ?.associate { + it.id().removePrefix("tr-").toLong() to it.selectFirst(".translator-selection-name")?.textOrNull() + } + } else { + null + } + val newSource = getSource(response.request.url) + val chaptersList = root.getElementById("chapters-list") + var isRestricted = false + if (chaptersList == null && root.getElementsContainingOwnText(NO_CHAPTERS).isEmpty()) { + if (root.getElementsContainingOwnText(COPYRIGHT_ALERT).isNotEmpty()) { + isRestricted = true + } else { + root.parseFailed("No chapters found") + } + } + val hashRegex = Regex("window.user_hash\\s*=\\s*\'([^\']+)\'") + val userHash = doc.select("script").firstNotNullOfOrNull { it.html().findGroupValue(hashRegex) } + val hasNsfwAlert = root.select(".alert-warning").any { it.ownText().contains(NSFW_ALERT) } + return manga.copy( + source = newSource, + title = doc.metaValue("name") ?: manga.title, + altTitles = root.selectFirst(".all-names-popover")?.select(".name")?.mapNotNullToSet { + it.textOrNull() + } ?: manga.altTitles, + publicUrl = response.request.url.toString(), + description = root.selectFirst("div.manga-description")?.html(), + largeCoverUrl = coverImg?.attrAsAbsoluteUrlOrNull("data-full"), + coverUrl = manga.coverUrl + ?: coverImg?.attrAsAbsoluteUrlOrNull("data-thumb")?.replace("_p.", "."), + tags = root.selectFirstOrThrow("div.subject-meta") + .getElementsByAttributeValueContaining("href", "/list/genre/").mapTo(manga.tags.toMutableSet()) { a -> + MangaTag( + title = a.text().toTitleCase(), + key = a.attr("href").removeSuffix('/').substringAfterLast('/'), + source = source, + ) + }, + state = if (isRestricted) { + MangaState.RESTRICTED + } else { + manga.state + }, + authors = root.select(".elem_author,.elem_illustrator,.elem_screenwriter") + .select("a.person-link") + .mapNotNullToSet { it.textOrNull() } + manga.authors, + contentRating = (if (hasNsfwAlert) ContentRating.SUGGESTIVE else ContentRating.SAFE) + .coerceAtLeast(manga.contentRating ?: ContentRating.SAFE), + chapters = chaptersList?.select("a.chapter-link") + ?.flatMapChapters(reversed = true) { a -> + val tr = a.selectFirstParent("tr") ?: return@flatMapChapters emptyList() + val href = a.attrAsRelativeUrl("href") + val number = tr.attr("data-num").toFloatOrNull()?.div(10f) ?: 0f + val volume = tr.attr("data-vol").toIntOrNull() ?: 0 + if (translations.isNullOrEmpty() || a.attr("data-translations").isEmpty()) { + var translators = "" + val translatorElement = a.attr("title") + if (translatorElement.isNotBlank()) { + translators = translatorElement.replace("(Переводчик),", "&").removeSuffix(" (Переводчик)") + } + listOf( + MangaChapter( + id = generateUid(href), + title = a.text().removePrefix(manga.title).trim().nullIfEmpty(), + number = number, + volume = volume, + url = href.withQueryParam("d", userHash), + uploadDate = dateFormat.parseSafe(tr.selectFirst("td.date")?.text()), + scanlator = translators, + source = newSource, + branch = null, + ), + ) + } else { + val translationData = JSONArray(a.attr("data-translations")) + translationData.mapJSON { jo -> + val personId = jo.getLong("personId") + val link = href.withQueryParam("tran", personId.toString()) + MangaChapter( + id = generateUid(link), + title = a.text().removePrefix(manga.title).trim(), + number = number, + volume = volume, + url = link.withQueryParam("d", userHash), + uploadDate = dateFormat.parseSafe(jo.getStringOrNull("dateCreated")), + scanlator = null, + source = newSource, + branch = translations[personId], + ) + } + } + }.orEmpty(), + ) + } + + override suspend fun getPages(chapter: MangaChapter): List { + if (chapter.source != source && chapter.source is MangaParserSource) { // handle redirects between websites + return context.newParserInstance(chapter.source).getPages(chapter) + } + val url = chapter.url.toAbsoluteUrl(domain).toHttpUrl().newBuilder().setQueryParameter("mtr", "1").build() + val doc = webClient.httpGet(url).parseHtml() + val scripts = doc.select("script") + for (script in scripts) { + val data = script.html() + var pos = data.indexOf("rm_h.readerDoInit(") + if (pos != -1) { + parsePagesV2(data, pos)?.let { return it } + } + pos = data.indexOf("rm_h.readerInit( 0,") + if (pos != -1) { + parsePagesV1(data, pos)?.let { return it } + } + pos = data.indexOf(".readerInit(") + if (pos != -1) { + parsePagesV3(data, pos).let { return it } + } + } + doc.parseFailed("Pages list not found at ${chapter.url}") + } + + override suspend fun getPageUrl(page: MangaPage): String { + val parts = page.url.split('|') + if (parts.size < 2) { + return page.url + } + val path = parts.last() + // fast path + cachedPagesServer?.let { host -> + val url = concatUrl("https://$host/", path) + if (tryHead(url)) { + return url + } else { + cachedPagesServer = null + } + } + // slow path + val candidates = HashSet((parts.size - 1) * 2) + for (i in 0 until parts.size - 1) { + val server = parts[i].trim().ifEmpty { "https://$domain/" } + candidates.add(concatUrl(server, path)) + candidates.add(concatUrl(server, path.substringBeforeLast('?'))) + } + return try { + channelFlow { + for (url in candidates) { + launch { + if (tryHead(url)) { + send(url) + } + } + } + }.first().also { + cachedPagesServer = it.toHttpUrlOrNull()?.host + } + } catch (e: NoSuchElementException) { + candidates.randomOrNull() ?: throw ParseException("No page url candidates", page.url, e) + } + } + + private suspend fun fetchAvailableTags(): Set { + val doc = webClient.httpGet("https://${domain}/list/genres/sort_name").parseHtml() + val root = doc.body().getElementById("mangaBox")?.selectFirst("div.leftContent")?.selectFirst("table.table") + ?: doc.parseFailed("Cannot find root") + return root.select("a.element-link").mapToSet { a -> + MangaTag( + title = a.text().toTitleCase(), + key = a.attr("href").substringAfterLast('/'), + source = source, + ) + } + } + + override suspend fun getUsername(): String { + val root = webClient.httpGet("https://grouple.co/").parseHtml().body() + val element = root.selectFirst("img.user-avatar") ?: throw AuthRequiredException(source) + val res = element.parent()?.text() + return if (res.isNullOrEmpty()) { + root.parseFailed("Cannot find username") + } else res + } + + override fun intercept(chain: Interceptor.Chain): Response { + val request = chain.request() + if (!request.header(HEADER_ACCEPT).isNullOrEmpty()) { + return chain.proceed(request).checkIfAuthRequired() + } + val ext = request.url.pathSegments.lastOrNull()?.substringAfterLast('.', "")?.lowercase(Locale.ROOT) + return if (ext == "jpg" || ext == "jpeg" || ext == "png" || ext == "webp") { + chain.proceed( + request.newBuilder().header(HEADER_ACCEPT, "image/webp,image/png;q=0.9,image/jpeg,*/*;q=0.8").build(), + ) + } else { + chain.proceed(request).checkIfAuthRequired() + } + } + + override fun onCreateConfig(keys: MutableCollection>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + keys.add(splitTranslationsKey) + } + + override suspend fun getRelatedManga(seed: Manga): List { + val doc = webClient.httpGet(seed.url.toAbsoluteUrl(domain)).parseHtml() + val root = doc.body().requireElementById("mangaBox").select("h4").first { it.ownText() == RELATED_TITLE } + .nextElementSibling() ?: doc.parseFailed("Cannot find root") + return root.select("div.tile").mapNotNull(::parseManga) + } + + protected open fun getSource(url: HttpUrl): MangaSource = when (url.host) { + in SeiMangaParser.domains -> MangaParserSource.SEIMANGA + in MintMangaParser.domains -> MangaParserSource.MINTMANGA + in ReadmangaParser.domains -> MangaParserSource.READMANGA_RU + in SelfMangaParser.domains -> MangaParserSource.SELFMANGA + in UsagiParser.domains -> MangaParserSource.USAGI + else -> source + } + + private fun getSortKey(sortOrder: SortOrder) = when (sortOrder) { + SortOrder.ALPHABETICAL -> "name" + SortOrder.POPULARITY -> "rate" + SortOrder.UPDATED -> "updated" + SortOrder.ADDED, + SortOrder.NEWEST, + -> "created" + + SortOrder.RATING -> "votes" + else -> "rate" + } + + private suspend fun advancedSearch(offset: Int, order: SortOrder, filter: MangaListFilter): Response { + val tagsMap = tagsIndex.get() + val url = urlBuilder() + .addPathSegment("search") + .addPathSegment("advancedResults") + url.addQueryParameter("q", filter.query) + url.addQueryParameter("offset", offset.toString()) + filter.tags.forEach { tag -> + val tagId = requireNotNull(tagsMap[tag.title.lowercase()]) { "Tag ${tag.title} not found" } + url.addQueryParameter(tagId, "in") + } + filter.tagsExclude.forEach { tag -> + val tagId = requireNotNull(tagsMap[tag.title.lowercase()]) { "Tag ${tag.title} not found" } + url.addQueryParameter(tagId, "ex") + } + url.addQueryParameter( + "years", + buildString { + append(filter.yearFrom.ifZero { YEAR_MIN }) + append(',') + append(filter.yearTo.ifZero { YEAR_MAX }) + }, + ) + url.addQueryParameter( + "sortType", + when (order) { + SortOrder.RATING -> "USER_RATING" + SortOrder.ALPHABETICAL -> "NAME" + SortOrder.ADDED -> "YEAR" + SortOrder.POPULARITY -> "POPULARITY" + SortOrder.NEWEST -> "DATE_CREATE" + SortOrder.UPDATED -> "DATE_UPDATE" + else -> "RATING" + }, + ) + filter.states.forEach { state -> + when (state) { + MangaState.FINISHED -> "s_completed" + MangaState.ABANDONED -> "s_abandoned_popular" + MangaState.UPCOMING -> "s_wait_upload" + else -> null + }?.let { + url.addQueryParameter(it, "in") + } + } + + return webClient.httpGet(url.build()) + } + + private suspend fun tryHead(url: String): Boolean = runCatchingCancellable { + webClient.httpHead(url).use { response -> + response.isSuccessful && !response.isPumpkin() && response.headersContentLength() >= MIN_IMAGE_SIZE + } + }.getOrDefault(false) + + private fun Response.isPumpkin(): Boolean = request.url.host == "upload.wikimedia.org" + + private fun parseManga(node: Element): Manga? { + val imgDiv = node.selectFirst("div.img") ?: return null + val descDiv = node.selectFirst("div.desc") ?: return null + if (descDiv.selectFirst("i.fa-user") != null || descDiv.selectFirst("i.fa-external-link") != null) { + return null // skip author + } + val href = imgDiv.selectFirst("a")?.attrAsAbsoluteUrlOrNull("href") ?: return null + val title = descDiv.selectFirst("h3")?.selectFirst("a")?.text() ?: return null + val tileInfo = descDiv.selectFirst("div.tile-info") + val relUrl = href.toRelativeUrl(domain) + if (relUrl.contains("://")) { + return null + } + val author = tileInfo?.selectFirst("a.person-link")?.text() + return Manga( + id = generateUid(relUrl), + url = relUrl, + publicUrl = href, + title = title, + altTitles = setOfNotNull(descDiv.selectFirst("h5")?.textOrNull()), + coverUrl = imgDiv.selectFirst("img.lazy")?.attrAsAbsoluteUrlOrNull("data-original")?.replace("_p.", "."), + rating = runCatching { + node.selectFirst(".compact-rate")?.attr("title")?.toFloatOrNull()?.div(5f) + }.getOrNull() ?: RATING_UNKNOWN, + authors = setOfNotNull(author), + contentRating = if (isNsfwSource) ContentRating.ADULT else null, + tags = runCatching { + tileInfo?.select("a.element-link")?.mapToSet { + MangaTag( + title = it.text().toTitleCase(), + key = it.attr("href").substringAfterLast('/'), + source = source, + ) + } + }.getOrNull().orEmpty(), + state = when { + node.selectFirst("div.tags")?.selectFirst("span.mangaCompleted") != null -> MangaState.FINISHED + + else -> null + }, + source = source, + ) + } + + private fun parsePagesV1(data: String, pos: Int): List? { + val json = data.substring(pos).substringAfter('(').substringBefore('\n').substringBeforeLast(')') + if (json.isEmpty()) { + return null + } + val ja = JSONArray("[$json]") + val pages = ja.getJSONArray(1) + val servers = ja.getJSONArray(3).mapJSON { it.getString("path") } + val serversStr = servers.joinToString("|") + return (0 until pages.length()).map { i -> + val page = pages.getJSONArray(i) + val primaryServer = page.getString(0) + val url = page.getString(2) + val fullSrc = if ("$primaryServer|$serversStr|$url".contains("one-way.work")) { + // domain that does not need a token + "$primaryServer|$serversStr|${url}".substringBefore("?") + } else { + "$primaryServer|$serversStr|$url" + } + MangaPage( + id = generateUid(url), + url = fullSrc, + preview = null, + source = source, + ) + } + } + + private fun parsePagesV2(data: String, pos: Int): List? { + val json = data.substring(pos).substringAfter('(').substringBefore('\n').substringBeforeLast(')') + if (json.isEmpty()) { + return null + } + val ja = JSONArray("[$json]") + val pages = ja.getJSONArray(0) + val servers = ja.getJSONArray(2).mapJSON { it.getString("path") } + val serversStr = servers.joinToString("|") + return (0 until pages.length()).map { i -> + val page = pages.getJSONArray(i) + val primaryServer = page.getString(0) + val url = page.getString(2) + val fullSrc = if ("$primaryServer|$serversStr|$url".contains("one-way.work")) { + // domain that does not need a token + "$primaryServer|$serversStr|${url}".substringBefore("?") + } else { + "$primaryServer|$serversStr|$url" + } + MangaPage( + id = generateUid(url), + url = fullSrc, + preview = null, + source = source, + ) + } + } + + private fun parsePagesV3(data: String, pos: Int): List { + val json = JSONArray(data.substring(pos).substringBetween("(", ")").substringBeforeLast(',')) + return (0 until json.length()).map { i -> + val ja = json.getJSONArray(i) + val server = ja.getString(0).ifEmpty { "https://$domain" } + val url = ja.getString(2) + val fullUrl = concatUrl(server, url) + MangaPage( + id = generateUid(url), + url = if (fullUrl.contains("one-way.work")) { + // domain that does not need a token + fullUrl.substringBefore("?") + } else { + fullUrl + }, + preview = null, + source = source, + ) + } + } + + private suspend fun fetchTagsMap(): ScatterMap { + val url = "https://$domain/search/advanced" + val properties = + webClient.httpGet(url).parseHtml().body().selectFirst("form.search-form")?.select("div.form-group") + ?.find { it.selectFirst("li.property") != null } + ?.select("li.property") + ?: throw ParseException("Genres filter element not found", url) + val result = MutableScatterMap(properties.size) + properties.forEach { li -> + val name = li.text().lowercase() + val id = li.selectFirstOrThrow("input").id() + result[name] = id + } + return result + } + + private fun String.withQueryParam(name: String, value: String?): String { + if (value == null) return this + return toAbsoluteUrl(domain) + .toHttpUrl() + .newBuilder() + .setQueryParameter(name, value) + .build() + .toString() + .toRelativeUrl(domain) + } + + @Throws(IOException::class) + private fun Response.checkIfAuthRequired(): Response { + val lastPathSegment = request.url.pathSegments.lastOrNull() + if (lastPathSegment == "login") { + closeQuietly() + throw AuthRequiredException(source) + } + if (code == HttpURLConnection.HTTP_NOT_FOUND) { + if (!hasAuthCookie()) { + closeQuietly() + throw AuthRequiredException(source) + } else { + return newBuilder().code(HttpURLConnection.HTTP_OK).build() + } + } + return this + } + + private fun hasAuthCookie() = context.cookieJar.getCookies(domain).any { it.name == "gwt" } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/OkHttp.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/OkHttp.kt index 7151352c..88a6e876 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/OkHttp.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/OkHttp.kt @@ -4,67 +4,52 @@ package org.koitharu.kotatsu.parsers.util import kotlinx.coroutines.suspendCancellableCoroutine import okhttp3.* +import okhttp3.internal.toLongOrDefault import kotlin.contracts.InvocationKind import kotlin.contracts.contract public suspend fun Call.await(): Response = suspendCancellableCoroutine { continuation -> - val callback = ContinuationCallCallback(this, continuation) - enqueue(callback) - continuation.invokeOnCancellation(callback) + val callback = ContinuationCallCallback(this, continuation) + enqueue(callback) + continuation.invokeOnCancellation(callback) } public val Response.mimeType: String? - get() = header("content-type")?.substringBefore(';')?.trim()?.nullIfEmpty()?.lowercase() + get() = header("content-type")?.substringBefore(';')?.trim()?.nullIfEmpty()?.lowercase() public val HttpUrl.isHttpOrHttps: Boolean - get() = scheme.equals("https", ignoreCase = true) || scheme.equals("http", ignoreCase = true) + get() = scheme.equals("https", ignoreCase = true) || scheme.equals("http", ignoreCase = true) public fun Headers.Builder.mergeWith(other: Headers, replaceExisting: Boolean): Headers.Builder { - for ((name, value) in other) { - if (replaceExisting || this[name] == null) { - this[name] = value - } - } - return this + for ((name, value) in other) { + if (replaceExisting || this[name] == null) { + this[name] = value + } + } + return this } public fun Response.copy(): Response = newBuilder() - .body(peekBody(Long.MAX_VALUE)) - .build() + .body(peekBody(Long.MAX_VALUE)) + .build() public fun Response.Builder.setHeader(name: String, value: String?): Response.Builder = if (value == null) { - removeHeader(name) + removeHeader(name) } else { - header(name, value) + header(name, value) } public inline fun Response.map(mapper: (ResponseBody) -> ResponseBody): Response { - contract { - callsInPlace(mapper, InvocationKind.AT_MOST_ONCE) - } - return body?.use { responseBody -> - newBuilder() - .body(mapper(responseBody)) - .build() - } ?: this + contract { + callsInPlace(mapper, InvocationKind.AT_MOST_ONCE) + } + return body.use { responseBody -> + newBuilder() + .body(mapper(responseBody)) + .build() + } } -public fun Cookie.newBuilder(): Cookie.Builder = Cookie.Builder().also { c -> - c.name(name) - c.value(value) - if (persistent) { - c.expiresAt(expiresAt) - } - if (hostOnly) { - c.hostOnlyDomain(domain) - } else { - c.domain(domain) - } - c.path(path) - if (secure) { - c.secure() - } - if (httpOnly) { - c.httpOnly() - } -} +public fun Response.headersContentLength( + defaultValue: Long = -1, +): Long = headers["Content-Length"]?.toLongOrDefault(defaultValue) ?: defaultValue diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Parse.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Parse.kt index b37b9680..242dd50c 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Parse.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Parse.kt @@ -8,7 +8,6 @@ import org.json.JSONArray import org.json.JSONObject import org.jsoup.Jsoup import org.jsoup.nodes.Document -import org.koitharu.kotatsu.parsers.ErrorMessages import org.koitharu.kotatsu.parsers.InternalParsersApi import java.text.DateFormat @@ -22,9 +21,9 @@ internal const val SCHEME_HTTPS = "https" */ // TODO suspend public fun Response.parseHtml(): Document = use { response -> - val body = response.requireBody() - val charset = body.contentType()?.charset()?.name() - Jsoup.parse(body.byteStream(), charset, response.request.url.toString()) + val body = response.body + val charset = body.contentType()?.charset()?.name() + Jsoup.parse(body.byteStream(), charset, response.request.url.toString()) } /** @@ -33,7 +32,7 @@ public fun Response.parseHtml(): Document = use { response -> * @see [parseHtml] */ public fun Response.parseJson(): JSONObject = use { response -> - JSONObject(response.requireBody().string()) + JSONObject(response.body.string()) } /** @@ -42,15 +41,15 @@ public fun Response.parseJson(): JSONObject = use { response -> * @see [parseHtml] */ public fun Response.parseJsonArray(): JSONArray = use { response -> - JSONArray(response.requireBody().string()) + JSONArray(response.body.string()) } public fun Response.parseRaw(): String = use { response -> - response.requireBody().string() + response.body.string() } public fun Response.parseBytes(): ByteArray = use { response -> - response.requireBody().bytes() + response.body.bytes() } /** @@ -58,10 +57,10 @@ public fun Response.parseBytes(): ByteArray = use { response -> * @return an url relative to the [domain] or absolute, if domain is mismatching */ public fun String.toRelativeUrl(domain: String): String { - if (isEmpty() || startsWith("/")) { - return this - } - return replace(Regex("^[^/]{2,6}://${Regex.escape(domain)}+/", RegexOption.IGNORE_CASE), "/") + if (isEmpty() || startsWith("/")) { + return this + } + return replace(Regex("^[^/]{2,6}://${Regex.escape(domain)}+/", RegexOption.IGNORE_CASE), "/") } /** @@ -69,36 +68,35 @@ public fun String.toRelativeUrl(domain: String): String { * @return an absolute url with [domain] if this is relative */ public fun String.toAbsoluteUrl(domain: String): String = when { - startsWith("//") -> "$SCHEME_HTTPS:$this" - startsWith('/') -> "$SCHEME_HTTPS://$domain$this" - REGEX_SCHEME_PREFIX.containsMatchIn(this) -> this - else -> "$SCHEME_HTTPS://$domain/$this" + startsWith("//") -> "$SCHEME_HTTPS:$this" + startsWith('/') -> "$SCHEME_HTTPS://$domain$this" + REGEX_SCHEME_PREFIX.containsMatchIn(this) -> this + else -> "$SCHEME_HTTPS://$domain/$this" } public fun concatUrl(host: String, path: String): String { - val hostWithSlash = host.endsWith('/') - val pathWithSlash = path.startsWith('/') - val hostWithScheme = if (host.startsWith("//")) "https:$host" else host - return when { - hostWithSlash && pathWithSlash -> hostWithScheme + path.drop(1) - !hostWithSlash && !pathWithSlash -> "$hostWithScheme/$path" - else -> hostWithScheme + path - } + val hostWithSlash = host.endsWith('/') + val pathWithSlash = path.startsWith('/') + val hostWithScheme = if (host.startsWith("//")) "https:$host" else host + return when { + hostWithSlash && pathWithSlash -> hostWithScheme + path.drop(1) + !hostWithSlash && !pathWithSlash -> "$hostWithScheme/$path" + else -> hostWithScheme + path + } } @InternalParsersApi public fun DateFormat.parseSafe(str: String?): Long = if (str.isNullOrEmpty()) { - 0L + 0L } else { - runCatching { - parse(str)?.time ?: 0L - }.onFailure { - if (javaClass.desiredAssertionStatus()) { - throw AssertionError("Cannot parse date $str", it) - } - }.getOrDefault(0L) + runCatching { + parse(str)?.time ?: 0L + }.onFailure { + if (javaClass.desiredAssertionStatus()) { + throw AssertionError("Cannot parse date $str", it) + } + }.getOrDefault(0L) } -public fun Response.requireBody(): ResponseBody = requireNotNull(body) { - ErrorMessages.RESPONSE_NULL_BODY -} +@Deprecated("Useless since OkHttp 5.0", replaceWith = ReplaceWith("body")) +public fun Response.requireBody(): ResponseBody = body diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/RelatedMangaFinder.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/RelatedMangaFinder.kt index 8a8e1dfe..73e89b42 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/RelatedMangaFinder.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/RelatedMangaFinder.kt @@ -3,51 +3,55 @@ package org.koitharu.kotatsu.parsers.util import kotlinx.coroutines.* import org.koitharu.kotatsu.parsers.MangaParser import org.koitharu.kotatsu.parsers.model.Manga +import org.koitharu.kotatsu.parsers.model.MangaListFilter import org.koitharu.kotatsu.parsers.model.SortOrder -import org.koitharu.kotatsu.parsers.model.search.MangaSearchQuery -import org.koitharu.kotatsu.parsers.model.search.QueryCriteria -import org.koitharu.kotatsu.parsers.model.search.SearchableField public class RelatedMangaFinder( - private val parsers: Collection, + private val parsers: Collection, ) { - public suspend operator fun invoke(seed: Manga): List = withContext(Dispatchers.Default) { - coroutineScope { - parsers.singleOrNull()?.let { parser -> - findRelatedImpl(this, parser, seed) - } ?: parsers.map { parser -> - async { - findRelatedImpl(this, parser, seed) - } - }.awaitAll().flatten() - } - } + public suspend operator fun invoke(seed: Manga): List = withContext(Dispatchers.Default) { + coroutineScope { + parsers.singleOrNull()?.let { parser -> + findRelatedImpl(this, parser, seed) + } ?: parsers.map { parser -> + async { + findRelatedImpl(this, parser, seed) + } + }.awaitAll().flatten() + } + } - private suspend fun findRelatedImpl(scope: CoroutineScope, parser: MangaParser, seed: Manga): List { - val words = HashSet() - words += seed.title.splitByWhitespace() - seed.altTitles.forEach { - words += it.splitByWhitespace() - } - if (words.isEmpty()) { - return emptyList() - } - val results = words.map { keyword -> - scope.async { - val result = parser.getList( - MangaSearchQuery.Builder() - .order(SortOrder.RELEVANCE) - .criterion(QueryCriteria.Match(SearchableField.TITLE_NAME, keyword)) - .build(), - ) - result.filter { it.id != seed.id && it.containKeyword(keyword) } - } - }.awaitAll() - return results.minBy { if (it.isEmpty()) Int.MAX_VALUE else it.size } - } + private suspend fun findRelatedImpl(scope: CoroutineScope, parser: MangaParser, seed: Manga): List { + val words = HashSet() + words += seed.title.splitByWhitespace() + seed.altTitles.forEach { + words += it.splitByWhitespace() + } + if (words.isEmpty()) { + return emptyList() + } + val results = words.map { keyword -> + scope.async { + val result = parser.getList( + 0, + if (SortOrder.RELEVANCE in parser.availableSortOrders) { + SortOrder.RELEVANCE + } else { + parser.availableSortOrders.first() + }, + MangaListFilter( + query = keyword, + ), + ) + result.filter { it.id != seed.id && it.containKeyword(keyword) } + } + }.awaitAll() + return results.minBy { if (it.isEmpty()) Int.MAX_VALUE else it.size } + } - private fun Manga.containKeyword(keyword: String): Boolean { - return title.contains(keyword, ignoreCase = true) || altTitle?.contains(keyword, ignoreCase = true) == true - } + private fun Manga.containKeyword(keyword: String): Boolean { + return title.contains(keyword, ignoreCase = true) + || altTitles.any { it.contains(keyword, ignoreCase = true) } + } }