Update dependencies
This commit is contained in:
@@ -1,5 +1,5 @@
|
||||
plugins {
|
||||
kotlin("jvm") version "2.0.20"
|
||||
kotlin("jvm") version "2.2.10"
|
||||
}
|
||||
|
||||
repositories {
|
||||
@@ -14,5 +14,5 @@ dependencies {
|
||||
implementation(gradleApi())
|
||||
implementation("org.simpleframework:simple-xml:2.7.1")
|
||||
implementation("com.soywiz.korlibs.korte:korte-jvm:4.0.10")
|
||||
implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.10.1")
|
||||
implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.10.2")
|
||||
}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
distributionBase=GRADLE_USER_HOME
|
||||
distributionPath=wrapper/dists
|
||||
distributionUrl=https\://services.gradle.org/distributions/gradle-7.4-bin.zip
|
||||
distributionSha256Sum=bd71102213493060956ec229d946beee57158dbd89d0e62b91bca0fa2c5f3531
|
||||
distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.3-bin.zip
|
||||
zipStoreBase=GRADLE_USER_HOME
|
||||
zipStorePath=wrapper/dists
|
||||
|
||||
@@ -1,13 +1,13 @@
|
||||
[versions]
|
||||
kotlin = "2.0.20"
|
||||
ksp = "2.0.20-1.0.25"
|
||||
kotlin = "2.2.10"
|
||||
ksp = "2.2.10-2.0.2"
|
||||
coroutines = "1.10.2"
|
||||
junit = "5.10.1"
|
||||
okhttp = "4.12.0"
|
||||
okio = "3.11.0"
|
||||
okhttp = "5.1.0"
|
||||
okio = "3.16.0"
|
||||
json = "20240303"
|
||||
androidx-collection = "1.5.0"
|
||||
jsoup = "1.19.1"
|
||||
jsoup = "1.21.2"
|
||||
quickjs = "1.1.0"
|
||||
|
||||
[plugins]
|
||||
|
||||
4
gradle/wrapper/gradle-wrapper.properties
vendored
4
gradle/wrapper/gradle-wrapper.properties
vendored
@@ -1,7 +1,7 @@
|
||||
#Wed Aug 27 01:56:37 ICT 2025
|
||||
distributionBase=GRADLE_USER_HOME
|
||||
distributionPath=wrapper/dists
|
||||
distributionSha256Sum=20f1b1176237254a6fc204d8434196fa11a4cfb387567519c61556e8710aed78
|
||||
distributionUrl=https\://services.gradle.org/distributions/gradle-8.13-bin.zip
|
||||
distributionSha256Sum=bd71102213493060956ec229d946beee57158dbd89d0e62b91bca0fa2c5f3531
|
||||
distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.3-bin.zip
|
||||
zipStoreBase=GRADLE_USER_HOME
|
||||
zipStorePath=wrapper/dists
|
||||
|
||||
@@ -12,92 +12,92 @@ import java.io.Writer
|
||||
import java.util.*
|
||||
|
||||
class ParserProcessor(
|
||||
private val codeGenerator: CodeGenerator,
|
||||
private val logger: KSPLogger,
|
||||
private val options: Map<String, String>,
|
||||
private val codeGenerator: CodeGenerator,
|
||||
private val logger: KSPLogger,
|
||||
private val options: Map<String, String>,
|
||||
) : SymbolProcessor {
|
||||
private val availableLocales = Locale.getAvailableLocales().toSet()
|
||||
private val sourceNamePattern = Regex("[A-Z_][A-Z0-9_]{3,}")
|
||||
private val availableLocales = Locale.getAvailableLocales().toSet()
|
||||
private val sourceNamePattern = Regex("[A-Z_][A-Z0-9_]{3,}")
|
||||
|
||||
override fun process(resolver: Resolver): List<KSAnnotated> {
|
||||
val symbols = resolver.getSymbolsWithAnnotation("org.koitharu.kotatsu.parsers.MangaSourceParser")
|
||||
val ret = symbols.filterNot { it.validate() }.toList()
|
||||
if (!symbols.iterator().hasNext()) {
|
||||
return ret
|
||||
}
|
||||
val dependencies = Dependencies.ALL_FILES
|
||||
val factoryFile =
|
||||
try {
|
||||
codeGenerator.createNewFile(
|
||||
dependencies = dependencies,
|
||||
packageName = "org.koitharu.kotatsu.parsers",
|
||||
fileName = "MangaParserFactory",
|
||||
)
|
||||
} catch (e: FileAlreadyExistsException) {
|
||||
logger.warn(e.toString(), null)
|
||||
null
|
||||
}
|
||||
val sourcesFile =
|
||||
try {
|
||||
codeGenerator.createNewFile(
|
||||
dependencies = dependencies,
|
||||
packageName = "org.koitharu.kotatsu.parsers.model",
|
||||
fileName = "MangaSource",
|
||||
)
|
||||
} catch (e: FileAlreadyExistsException) {
|
||||
logger.warn(e.toString(), null)
|
||||
null
|
||||
}
|
||||
val totalCount = sourcesFile?.writer().use { sourcesWriter ->
|
||||
factoryFile?.writer().use { factoryWriter ->
|
||||
writeContent(sourcesWriter, factoryWriter, symbols)
|
||||
}
|
||||
}
|
||||
writeSummary(totalCount)
|
||||
return ret
|
||||
}
|
||||
override fun process(resolver: Resolver): List<KSAnnotated> {
|
||||
val symbols = resolver.getSymbolsWithAnnotation("org.koitharu.kotatsu.parsers.MangaSourceParser")
|
||||
val ret = symbols.filterNot { it.validate() }.toList()
|
||||
if (!symbols.iterator().hasNext()) {
|
||||
return ret
|
||||
}
|
||||
val dependencies = Dependencies.ALL_FILES
|
||||
val factoryFile =
|
||||
try {
|
||||
codeGenerator.createNewFile(
|
||||
dependencies = dependencies,
|
||||
packageName = "org.koitharu.kotatsu.parsers",
|
||||
fileName = "MangaParserFactory",
|
||||
)
|
||||
} catch (e: FileAlreadyExistsException) {
|
||||
logger.warn(e.toString(), null)
|
||||
null
|
||||
}
|
||||
val sourcesFile =
|
||||
try {
|
||||
codeGenerator.createNewFile(
|
||||
dependencies = dependencies,
|
||||
packageName = "org.koitharu.kotatsu.parsers.model",
|
||||
fileName = "MangaSource",
|
||||
)
|
||||
} catch (e: FileAlreadyExistsException) {
|
||||
logger.warn(e.toString(), null)
|
||||
null
|
||||
}
|
||||
val totalCount = sourcesFile?.writer().use { sourcesWriter ->
|
||||
factoryFile?.writer().use { factoryWriter ->
|
||||
writeContent(sourcesWriter, factoryWriter, symbols)
|
||||
}
|
||||
}
|
||||
writeSummary(totalCount)
|
||||
return ret
|
||||
}
|
||||
|
||||
private fun writeContent(
|
||||
sourcesWriter: Writer?,
|
||||
factoryWriter: Writer?,
|
||||
symbols: Sequence<KSAnnotated>,
|
||||
): Int {
|
||||
if (sourcesWriter == null && factoryWriter == null) {
|
||||
return 0
|
||||
}
|
||||
factoryWriter?.write(
|
||||
"""
|
||||
private fun writeContent(
|
||||
sourcesWriter: Writer?,
|
||||
factoryWriter: Writer?,
|
||||
symbols: Sequence<KSAnnotated>,
|
||||
): Int {
|
||||
if (sourcesWriter == null && factoryWriter == null) {
|
||||
return 0
|
||||
}
|
||||
factoryWriter?.write(
|
||||
"""
|
||||
package org.koitharu.kotatsu.parsers
|
||||
|
||||
import org.koitharu.kotatsu.parsers.model.MangaParserSource
|
||||
import org.koitharu.kotatsu.parsers.core.MangaParserWrapper
|
||||
|
||||
internal fun MangaParserSource.newParser(context: MangaLoaderContext): MangaParser = when (this) {
|
||||
|
||||
|
||||
""".trimIndent(),
|
||||
)
|
||||
sourcesWriter?.write(
|
||||
"""
|
||||
)
|
||||
sourcesWriter?.write(
|
||||
"""
|
||||
package org.koitharu.kotatsu.parsers.model
|
||||
|
||||
|
||||
public enum class MangaParserSource(
|
||||
public val title: String,
|
||||
public val locale: String,
|
||||
public val contentType: ContentType,
|
||||
public val isBroken: Boolean,
|
||||
): MangaSource {
|
||||
|
||||
|
||||
""".trimIndent(),
|
||||
)
|
||||
)
|
||||
|
||||
val visitor = ParserVisitor(sourcesWriter, factoryWriter)
|
||||
val totalCount = symbols
|
||||
.filter { it is KSClassDeclaration && it.validate() }
|
||||
.onEach { it.accept(visitor, Unit) }
|
||||
.count()
|
||||
val visitor = ParserVisitor(sourcesWriter, factoryWriter)
|
||||
val totalCount = symbols
|
||||
.filter { it is KSClassDeclaration && it.validate() }
|
||||
.onEach { it.accept(visitor, Unit) }
|
||||
.count()
|
||||
|
||||
factoryWriter?.write(
|
||||
"""
|
||||
factoryWriter?.write(
|
||||
"""
|
||||
MangaParserSource.DUMMY -> throw NotImplementedError("Manga parser ${'$'}name cannot be instantiated")
|
||||
}.let {
|
||||
require(it.source == this) {
|
||||
@@ -106,83 +106,83 @@ class ParserProcessor(
|
||||
MangaParserWrapper(it)
|
||||
}
|
||||
""".trimIndent(),
|
||||
)
|
||||
sourcesWriter?.write(
|
||||
"""
|
||||
)
|
||||
sourcesWriter?.write(
|
||||
"""
|
||||
DUMMY("Dummy", "", ContentType.OTHER, false),
|
||||
;
|
||||
}
|
||||
""".trimIndent(),
|
||||
)
|
||||
return totalCount
|
||||
}
|
||||
)
|
||||
return totalCount
|
||||
}
|
||||
|
||||
private fun writeSummary(totalCount: Int) {
|
||||
val file = File(options["summaryOutputDir"] ?: return, "summary.yaml")
|
||||
file.writeText("total: $totalCount")
|
||||
}
|
||||
private fun writeSummary(totalCount: Int) {
|
||||
val file = File(options["summaryOutputDir"] ?: return, "summary.yaml")
|
||||
file.writeText("total: $totalCount")
|
||||
}
|
||||
|
||||
private inner class ParserVisitor(
|
||||
private val sourcesWriter: Writer?,
|
||||
private val factoryWriter: Writer?,
|
||||
) : KSVisitorVoid() {
|
||||
private val titles = HashMap<String, String>()
|
||||
private inner class ParserVisitor(
|
||||
private val sourcesWriter: Writer?,
|
||||
private val factoryWriter: Writer?,
|
||||
) : KSVisitorVoid() {
|
||||
private val titles = HashMap<String, String>()
|
||||
|
||||
override fun visitClassDeclaration(
|
||||
classDeclaration: KSClassDeclaration,
|
||||
data: Unit,
|
||||
) {
|
||||
if (classDeclaration.classKind != ClassKind.CLASS || classDeclaration.isAbstract()) {
|
||||
logger.error("Only non-abstract can be annotated with @MangaSourceParser", classDeclaration)
|
||||
}
|
||||
val annotation = classDeclaration.annotations.single { it.shortName.asString() == "MangaSourceParser" }
|
||||
val deprecation = classDeclaration.annotations.singleOrNull { it.shortName.asString() == "Deprecated" }
|
||||
val isBroken = classDeclaration.annotations.any { it.shortName.asString() == "Broken" }
|
||||
val name = annotation.arguments.single { it.name?.asString() == "name" }.value as String
|
||||
val title = annotation.arguments.single { it.name?.asString() == "title" }.value as String
|
||||
val locale = annotation.arguments.single { it.name?.asString() == "locale" }.value as String
|
||||
val type = annotation.arguments.single { it.name?.asString() == "type" }.value
|
||||
val localeString = "\"$locale\""
|
||||
val localeObj = if (locale.isEmpty()) null else Locale(locale)
|
||||
val localeTitle = localeObj?.getDisplayLanguage(localeObj)
|
||||
if (localeObj != null && localeObj !in availableLocales) {
|
||||
logger.error("Manga source $name has wrong locale: $localeTitle")
|
||||
}
|
||||
override fun visitClassDeclaration(
|
||||
classDeclaration: KSClassDeclaration,
|
||||
data: Unit,
|
||||
) {
|
||||
if (classDeclaration.classKind != ClassKind.CLASS || classDeclaration.isAbstract()) {
|
||||
logger.error("Only non-abstract can be annotated with @MangaSourceParser", classDeclaration)
|
||||
}
|
||||
val annotation = classDeclaration.annotations.single { it.shortName.asString() == "MangaSourceParser" }
|
||||
val deprecation = classDeclaration.annotations.singleOrNull { it.shortName.asString() == "Deprecated" }
|
||||
val isBroken = classDeclaration.annotations.any { it.shortName.asString() == "Broken" }
|
||||
val name = annotation.arguments.single { it.name?.asString() == "name" }.value as String
|
||||
val title = annotation.arguments.single { it.name?.asString() == "title" }.value as String
|
||||
val locale = annotation.arguments.single { it.name?.asString() == "locale" }.value as String
|
||||
val type = annotation.arguments.single { it.name?.asString() == "type" }.value
|
||||
val localeString = "\"$locale\""
|
||||
val localeObj = if (locale.isEmpty()) null else Locale(locale)
|
||||
val localeTitle = localeObj?.getDisplayLanguage(localeObj)
|
||||
if (localeObj != null && localeObj !in availableLocales) {
|
||||
logger.error("Manga source $name has wrong locale: $localeTitle")
|
||||
}
|
||||
|
||||
if (!sourceNamePattern.matches(name)) {
|
||||
logger.error("Manga source name must be uppercase: $name")
|
||||
}
|
||||
if (!sourceNamePattern.matches(name)) {
|
||||
logger.error("Manga source name must be uppercase: $name")
|
||||
}
|
||||
|
||||
val constructor = classDeclaration.primaryConstructor
|
||||
if (constructor == null || constructor.parameters.count { !it.hasDefault } != 1) {
|
||||
logger.error(
|
||||
"Class with @MangaSourceParser must have a primary constructor with one parameter",
|
||||
classDeclaration,
|
||||
)
|
||||
}
|
||||
val className = checkNotNull(classDeclaration.qualifiedName?.asString()) { "Class name is null" }
|
||||
val constructor = classDeclaration.primaryConstructor
|
||||
if (constructor == null || constructor.parameters.count { !it.hasDefault } != 1) {
|
||||
logger.error(
|
||||
"Class with @MangaSourceParser must have a primary constructor with one parameter",
|
||||
classDeclaration,
|
||||
)
|
||||
}
|
||||
val className = checkNotNull(classDeclaration.qualifiedName?.asString()) { "Class name is null" }
|
||||
|
||||
val prevTitleClass = titles.put(title, className)
|
||||
if (prevTitleClass != null) {
|
||||
logger.warn("Source title duplication: \"$title\" is assigned to both $prevTitleClass and $className")
|
||||
}
|
||||
val prevTitleClass = titles.put(title, className)
|
||||
if (prevTitleClass != null) {
|
||||
logger.warn("Source title duplication: \"$title\" is assigned to both $prevTitleClass and $className")
|
||||
}
|
||||
|
||||
factoryWriter?.write("\tMangaParserSource.$name -> $className(context)\n")
|
||||
val deprecationString =
|
||||
if (deprecation != null) {
|
||||
val reason =
|
||||
deprecation.arguments
|
||||
.find { it.name?.asString() == "message" }
|
||||
?.value
|
||||
?.toString() ?: "Unknown reason"
|
||||
"@Deprecated(\"$reason\") "
|
||||
} else {
|
||||
""
|
||||
}
|
||||
val localeComment = localeTitle?.toTitleCase(localeObj)?.let { " /* $it */" }.orEmpty()
|
||||
sourcesWriter?.write(
|
||||
"\t$deprecationString$name(\"$title\", $localeString$localeComment, ContentType.$type, $isBroken),\n",
|
||||
)
|
||||
}
|
||||
}
|
||||
factoryWriter?.write("\tMangaParserSource.$name -> $className(context)\n")
|
||||
val deprecationString =
|
||||
if (deprecation != null) {
|
||||
val reason =
|
||||
deprecation.arguments
|
||||
.find { it.name?.asString() == "message" }
|
||||
?.value
|
||||
?.toString() ?: "Unknown reason"
|
||||
"@Deprecated(\"$reason\") "
|
||||
} else {
|
||||
""
|
||||
}
|
||||
val localeComment = localeTitle?.toTitleCase(localeObj)?.let { " /* $it */" }.orEmpty()
|
||||
sourcesWriter?.write(
|
||||
"\t$deprecationString$name(\"$title\", $localeString$localeComment, $type, $isBroken),\n",
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,18 +2,17 @@ package org.koitharu.kotatsu.parsers
|
||||
|
||||
public object ErrorMessages {
|
||||
|
||||
public const val FILTER_MULTIPLE_STATES_NOT_SUPPORTED: String = "Multiple states are not supported by this source"
|
||||
public const val FILTER_MULTIPLE_GENRES_NOT_SUPPORTED: String = "Multiple genres are not supported by this source"
|
||||
public const val FILTER_MULTIPLE_CONTENT_RATING_NOT_SUPPORTED: String =
|
||||
"Multiple Content ratings are not supported by this source"
|
||||
public const val FILTER_MULTIPLE_CONTENT_TYPES_NOT_SUPPORTED: String =
|
||||
"Multiple Content types are not supported by this source"
|
||||
public const val FILTER_MULTIPLE_DEMOGRAPHICS_NOT_SUPPORTED: String =
|
||||
"Multiple Demographics are not supported by this source"
|
||||
public const val FILTER_BOTH_LOCALE_GENRES_NOT_SUPPORTED: String =
|
||||
"Filtering by both genres and locale is not supported by this source"
|
||||
public const val FILTER_BOTH_STATES_GENRES_NOT_SUPPORTED: String =
|
||||
"Filtering by both genres and states is not supported by this source"
|
||||
public const val SEARCH_NOT_SUPPORTED: String = "Search is not supported by this source"
|
||||
public const val RESPONSE_NULL_BODY: String = "Response has no body"
|
||||
public const val FILTER_MULTIPLE_STATES_NOT_SUPPORTED: String = "Multiple states are not supported by this source"
|
||||
public const val FILTER_MULTIPLE_GENRES_NOT_SUPPORTED: String = "Multiple genres are not supported by this source"
|
||||
public const val FILTER_MULTIPLE_CONTENT_RATING_NOT_SUPPORTED: String =
|
||||
"Multiple Content ratings are not supported by this source"
|
||||
public const val FILTER_MULTIPLE_CONTENT_TYPES_NOT_SUPPORTED: String =
|
||||
"Multiple Content types are not supported by this source"
|
||||
public const val FILTER_MULTIPLE_DEMOGRAPHICS_NOT_SUPPORTED: String =
|
||||
"Multiple Demographics are not supported by this source"
|
||||
public const val FILTER_BOTH_LOCALE_GENRES_NOT_SUPPORTED: String =
|
||||
"Filtering by both genres and locale is not supported by this source"
|
||||
public const val FILTER_BOTH_STATES_GENRES_NOT_SUPPORTED: String =
|
||||
"Filtering by both genres and states is not supported by this source"
|
||||
public const val SEARCH_NOT_SUPPORTED: String = "Search is not supported by this source"
|
||||
}
|
||||
|
||||
@@ -9,39 +9,39 @@ import java.net.HttpURLConnection.HTTP_UNAVAILABLE
|
||||
|
||||
public object CloudFlareHelper {
|
||||
|
||||
public const val PROTECTION_NOT_DETECTED: Int = 0
|
||||
public const val PROTECTION_CAPTCHA: Int = 1
|
||||
public const val PROTECTION_BLOCKED: Int = 2
|
||||
public const val PROTECTION_NOT_DETECTED: Int = 0
|
||||
public const val PROTECTION_CAPTCHA: Int = 1
|
||||
public const val PROTECTION_BLOCKED: Int = 2
|
||||
|
||||
private const val CF_CLEARANCE = "cf_clearance"
|
||||
private const val CF_CLEARANCE = "cf_clearance"
|
||||
|
||||
public fun checkResponseForProtection(response: Response): Int {
|
||||
if (response.code != HTTP_FORBIDDEN && response.code != HTTP_UNAVAILABLE) {
|
||||
return PROTECTION_NOT_DETECTED
|
||||
}
|
||||
val content = if (response.body != null) {
|
||||
response.peekBody(Long.MAX_VALUE).use {
|
||||
Jsoup.parse(it.byteStream(), Charsets.UTF_8.name(), response.request.url.toString())
|
||||
}
|
||||
} else {
|
||||
return PROTECTION_NOT_DETECTED
|
||||
}
|
||||
return when {
|
||||
content.selectFirst("h2[data-translate=\"blocked_why_headline\"]") != null -> PROTECTION_BLOCKED
|
||||
content.getElementById("challenge-error-title") != null || content.getElementById("challenge-error-text") != null -> PROTECTION_CAPTCHA
|
||||
public fun checkResponseForProtection(response: Response): Int {
|
||||
if (response.code != HTTP_FORBIDDEN && response.code != HTTP_UNAVAILABLE) {
|
||||
return PROTECTION_NOT_DETECTED
|
||||
}
|
||||
val content = try {
|
||||
response.peekBody(Long.MAX_VALUE).use {
|
||||
Jsoup.parse(it.byteStream(), Charsets.UTF_8.name(), response.request.url.toString())
|
||||
}
|
||||
} catch (_: IllegalStateException) {
|
||||
return PROTECTION_NOT_DETECTED
|
||||
}
|
||||
return when {
|
||||
content.selectFirst("h2[data-translate=\"blocked_why_headline\"]") != null -> PROTECTION_BLOCKED
|
||||
content.getElementById("challenge-error-title") != null || content.getElementById("challenge-error-text") != null -> PROTECTION_CAPTCHA
|
||||
|
||||
else -> PROTECTION_NOT_DETECTED
|
||||
}
|
||||
}
|
||||
else -> PROTECTION_NOT_DETECTED
|
||||
}
|
||||
}
|
||||
|
||||
public fun getClearanceCookie(cookieJar: CookieJar, url: String): String? {
|
||||
return cookieJar.loadForRequest(url.toHttpUrl()).find { it.name == CF_CLEARANCE }?.value
|
||||
}
|
||||
public fun getClearanceCookie(cookieJar: CookieJar, url: String): String? {
|
||||
return cookieJar.loadForRequest(url.toHttpUrl()).find { it.name == CF_CLEARANCE }?.value
|
||||
}
|
||||
|
||||
public fun isCloudFlareCookie(name: String): Boolean {
|
||||
return name.startsWith("cf_")
|
||||
|| name.startsWith("_cf")
|
||||
|| name.startsWith("__cf")
|
||||
|| name == "csrftoken"
|
||||
}
|
||||
public fun isCloudFlareCookie(name: String): Boolean {
|
||||
return name.startsWith("cf_")
|
||||
|| name.startsWith("_cf")
|
||||
|| name.startsWith("__cf")
|
||||
|| name == "csrftoken"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,7 +7,6 @@ import okhttp3.HttpUrl.Companion.toHttpUrlOrNull
|
||||
import okhttp3.Interceptor
|
||||
import okhttp3.Response
|
||||
import okhttp3.internal.closeQuietly
|
||||
import okhttp3.internal.headersContentLength
|
||||
import org.jsoup.internal.StringUtil
|
||||
import org.jsoup.nodes.Element
|
||||
import org.koitharu.kotatsu.parsers.MangaLoaderContext
|
||||
@@ -28,470 +27,471 @@ import java.util.concurrent.TimeUnit
|
||||
private const val DOMAIN_UNAUTHORIZED = "e-hentai.org"
|
||||
private const val DOMAIN_AUTHORIZED = "exhentai.org"
|
||||
private val TAG_PREFIXES = arrayOf("male:", "female:", "other:")
|
||||
private const val BANNED_RESPONSE_LENGTH = 256L
|
||||
|
||||
@MangaSourceParser("EXHENTAI", "ExHentai", type = ContentType.HENTAI)
|
||||
internal class ExHentaiParser(
|
||||
context: MangaLoaderContext,
|
||||
context: MangaLoaderContext,
|
||||
) : PagedMangaParser(context, MangaParserSource.EXHENTAI, pageSize = 25), MangaParserAuthProvider, Interceptor {
|
||||
|
||||
override val availableSortOrders: Set<SortOrder> = setOf(SortOrder.NEWEST)
|
||||
override val availableSortOrders: Set<SortOrder> = setOf(SortOrder.NEWEST)
|
||||
|
||||
override val configKeyDomain: ConfigKey.Domain
|
||||
get() {
|
||||
val isAuthorized = checkAuth()
|
||||
return ConfigKey.Domain(
|
||||
if (isAuthorized) DOMAIN_AUTHORIZED else DOMAIN_UNAUTHORIZED,
|
||||
if (isAuthorized) DOMAIN_UNAUTHORIZED else DOMAIN_AUTHORIZED,
|
||||
)
|
||||
}
|
||||
override val configKeyDomain: ConfigKey.Domain
|
||||
get() {
|
||||
val isAuthorized = checkAuth()
|
||||
return ConfigKey.Domain(
|
||||
if (isAuthorized) DOMAIN_AUTHORIZED else DOMAIN_UNAUTHORIZED,
|
||||
if (isAuthorized) DOMAIN_UNAUTHORIZED else DOMAIN_AUTHORIZED,
|
||||
)
|
||||
}
|
||||
|
||||
override val authUrl: String
|
||||
get() = "https://${domain}/bounce_login.php"
|
||||
override val authUrl: String
|
||||
get() = "https://${domain}/bounce_login.php"
|
||||
|
||||
private val ratingPattern = Regex("-?[0-9]+px")
|
||||
private val titleCleanupPattern = Regex("(\\[.*?]|\\([C0-9]*\\))")
|
||||
private val spacesCleanupPattern = Regex("(^\\s+|\\s+\$|\\s+(?=\\s))")
|
||||
private val authCookies = arrayOf("ipb_member_id", "ipb_pass_hash")
|
||||
private val suspiciousContentKey = ConfigKey.ShowSuspiciousContent(false)
|
||||
private val nextPages = MutableIntObjectMap<MutableIntLongMap>()
|
||||
private val ratingPattern = Regex("-?[0-9]+px")
|
||||
private val titleCleanupPattern = Regex("(\\[.*?]|\\([C0-9]*\\))")
|
||||
private val spacesCleanupPattern = Regex("(^\\s+|\\s+\$|\\s+(?=\\s))")
|
||||
private val authCookies = arrayOf("ipb_member_id", "ipb_pass_hash")
|
||||
private val suspiciousContentKey = ConfigKey.ShowSuspiciousContent(false)
|
||||
private val nextPages = MutableIntObjectMap<MutableIntLongMap>()
|
||||
|
||||
override val filterCapabilities: MangaListFilterCapabilities
|
||||
get() = MangaListFilterCapabilities(
|
||||
isMultipleTagsSupported = true,
|
||||
isTagsExclusionSupported = true,
|
||||
isSearchSupported = true,
|
||||
isSearchWithFiltersSupported = true,
|
||||
isAuthorSearchSupported = true,
|
||||
)
|
||||
override val filterCapabilities: MangaListFilterCapabilities
|
||||
get() = MangaListFilterCapabilities(
|
||||
isMultipleTagsSupported = true,
|
||||
isTagsExclusionSupported = true,
|
||||
isSearchSupported = true,
|
||||
isSearchWithFiltersSupported = true,
|
||||
isAuthorSearchSupported = true,
|
||||
)
|
||||
|
||||
override suspend fun isAuthorized(): Boolean = checkAuth()
|
||||
override suspend fun isAuthorized(): Boolean = checkAuth()
|
||||
|
||||
init {
|
||||
context.cookieJar.insertCookies(DOMAIN_AUTHORIZED, "nw=1", "sl=dm_2")
|
||||
context.cookieJar.insertCookies(DOMAIN_UNAUTHORIZED, "nw=1", "sl=dm_2")
|
||||
paginator.firstPage = 0
|
||||
searchPaginator.firstPage = 0
|
||||
}
|
||||
init {
|
||||
context.cookieJar.insertCookies(DOMAIN_AUTHORIZED, "nw=1", "sl=dm_2")
|
||||
context.cookieJar.insertCookies(DOMAIN_UNAUTHORIZED, "nw=1", "sl=dm_2")
|
||||
paginator.firstPage = 0
|
||||
searchPaginator.firstPage = 0
|
||||
}
|
||||
|
||||
override suspend fun getFilterOptions() = MangaListFilterOptions(
|
||||
availableTags = mapTags(),
|
||||
availableContentTypes = EnumSet.of(
|
||||
ContentType.DOUJINSHI,
|
||||
ContentType.MANGA,
|
||||
ContentType.ARTIST_CG,
|
||||
ContentType.GAME_CG,
|
||||
ContentType.COMICS,
|
||||
ContentType.IMAGE_SET,
|
||||
ContentType.OTHER,
|
||||
),
|
||||
availableLocales = setOf(
|
||||
Locale.JAPANESE,
|
||||
Locale.ENGLISH,
|
||||
Locale.CHINESE,
|
||||
Locale("nl"),
|
||||
Locale.FRENCH,
|
||||
Locale.GERMAN,
|
||||
Locale("hu"),
|
||||
Locale.ITALIAN,
|
||||
Locale("kr"),
|
||||
Locale("pl"),
|
||||
Locale("pt"),
|
||||
Locale("ru"),
|
||||
Locale("es"),
|
||||
Locale("th"),
|
||||
Locale("vi"),
|
||||
),
|
||||
)
|
||||
override suspend fun getFilterOptions() = MangaListFilterOptions(
|
||||
availableTags = mapTags(),
|
||||
availableContentTypes = EnumSet.of(
|
||||
ContentType.DOUJINSHI,
|
||||
ContentType.MANGA,
|
||||
ContentType.ARTIST_CG,
|
||||
ContentType.GAME_CG,
|
||||
ContentType.COMICS,
|
||||
ContentType.IMAGE_SET,
|
||||
ContentType.OTHER,
|
||||
),
|
||||
availableLocales = setOf(
|
||||
Locale.JAPANESE,
|
||||
Locale.ENGLISH,
|
||||
Locale.CHINESE,
|
||||
Locale("nl"),
|
||||
Locale.FRENCH,
|
||||
Locale.GERMAN,
|
||||
Locale("hu"),
|
||||
Locale.ITALIAN,
|
||||
Locale("kr"),
|
||||
Locale("pl"),
|
||||
Locale("pt"),
|
||||
Locale("ru"),
|
||||
Locale("es"),
|
||||
Locale("th"),
|
||||
Locale("vi"),
|
||||
),
|
||||
)
|
||||
|
||||
override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> {
|
||||
return getListPage(page, order, filter, updateDm = false)
|
||||
}
|
||||
override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> {
|
||||
return getListPage(page, order, filter, updateDm = false)
|
||||
}
|
||||
|
||||
private suspend fun getListPage(
|
||||
page: Int,
|
||||
order: SortOrder,
|
||||
filter: MangaListFilter,
|
||||
updateDm: Boolean,
|
||||
): List<Manga> {
|
||||
val next = synchronized(nextPages) {
|
||||
nextPages[filter.hashCode()]?.getOrDefault(page, 0L) ?: 0L
|
||||
}
|
||||
private suspend fun getListPage(
|
||||
page: Int,
|
||||
order: SortOrder,
|
||||
filter: MangaListFilter,
|
||||
updateDm: Boolean,
|
||||
): List<Manga> {
|
||||
val next = synchronized(nextPages) {
|
||||
nextPages[filter.hashCode()]?.getOrDefault(page, 0L) ?: 0L
|
||||
}
|
||||
|
||||
if (page > 0 && next == 0L) {
|
||||
assert(false) { "Page timestamp not found" }
|
||||
return emptyList()
|
||||
}
|
||||
if (page > 0 && next == 0L) {
|
||||
assert(false) { "Page timestamp not found" }
|
||||
return emptyList()
|
||||
}
|
||||
|
||||
val url = urlBuilder()
|
||||
url.addEncodedQueryParameter("next", next.toString())
|
||||
url.addQueryParameter("f_search", filter.toSearchQuery())
|
||||
val url = urlBuilder()
|
||||
url.addEncodedQueryParameter("next", next.toString())
|
||||
url.addQueryParameter("f_search", filter.toSearchQuery())
|
||||
|
||||
val fCats = filter.types.toFCats()
|
||||
if (fCats != 0) {
|
||||
url.addEncodedQueryParameter("f_cats", (1023 - fCats).toString())
|
||||
}
|
||||
if (updateDm) {
|
||||
// by unknown reason cookie "sl=dm_2" is ignored, so, we should request it again
|
||||
url.addQueryParameter("inline_set", "dm_e")
|
||||
}
|
||||
url.addQueryParameter("advsearch", "1")
|
||||
if (config[suspiciousContentKey]) {
|
||||
url.addQueryParameter("f_sh", "on")
|
||||
}
|
||||
val body = webClient.httpGet(url.build()).parseHtml().body()
|
||||
val root = body.selectFirst("table.itg")?.selectFirst("tbody")
|
||||
if (root == null) {
|
||||
if (updateDm) {
|
||||
if (body.getElementsContainingText("No hits found").isNotEmpty()) {
|
||||
return emptyList()
|
||||
} else {
|
||||
body.parseFailed("Cannot find root")
|
||||
}
|
||||
} else {
|
||||
return getListPage(page, order, filter, updateDm = true)
|
||||
}
|
||||
}
|
||||
val nextTimestamp = getNextTimestamp(body)
|
||||
synchronized(nextPages) {
|
||||
nextPages.getOrPut(filter.hashCode()) {
|
||||
MutableIntLongMap()
|
||||
}.put(page + 1, nextTimestamp)
|
||||
}
|
||||
val fCats = filter.types.toFCats()
|
||||
if (fCats != 0) {
|
||||
url.addEncodedQueryParameter("f_cats", (1023 - fCats).toString())
|
||||
}
|
||||
if (updateDm) {
|
||||
// by unknown reason cookie "sl=dm_2" is ignored, so, we should request it again
|
||||
url.addQueryParameter("inline_set", "dm_e")
|
||||
}
|
||||
url.addQueryParameter("advsearch", "1")
|
||||
if (config[suspiciousContentKey]) {
|
||||
url.addQueryParameter("f_sh", "on")
|
||||
}
|
||||
val body = webClient.httpGet(url.build()).parseHtml().body()
|
||||
val root = body.selectFirst("table.itg")?.selectFirst("tbody")
|
||||
if (root == null) {
|
||||
if (updateDm) {
|
||||
if (body.getElementsContainingText("No hits found").isNotEmpty()) {
|
||||
return emptyList()
|
||||
} else {
|
||||
body.parseFailed("Cannot find root")
|
||||
}
|
||||
} else {
|
||||
return getListPage(page, order, filter, updateDm = true)
|
||||
}
|
||||
}
|
||||
val nextTimestamp = getNextTimestamp(body)
|
||||
synchronized(nextPages) {
|
||||
nextPages.getOrPut(filter.hashCode()) {
|
||||
MutableIntLongMap()
|
||||
}.put(page + 1, nextTimestamp)
|
||||
}
|
||||
|
||||
return root.children().mapNotNull { tr ->
|
||||
if (tr.childrenSize() != 2) return@mapNotNull null
|
||||
val (td1, td2) = tr.children()
|
||||
val gLink = td2.selectFirstOrThrow("div.glink")
|
||||
val a = gLink.parents().select("a").first() ?: gLink.parseFailed("link not found")
|
||||
val href = a.attrAsRelativeUrl("href")
|
||||
val tagsDiv = gLink.nextElementSibling() ?: gLink.parseFailed("tags div not found")
|
||||
val rawTitle = gLink.text()
|
||||
val author = tagsDiv.getElementsContainingOwnText("artist:").first()
|
||||
?.nextElementSibling()?.textOrNull()
|
||||
Manga(
|
||||
id = generateUid(href),
|
||||
title = rawTitle.cleanupTitle(),
|
||||
altTitles = emptySet(),
|
||||
url = href,
|
||||
publicUrl = a.absUrl("href"),
|
||||
rating = td2.selectFirst("div.ir")?.parseRating() ?: RATING_UNKNOWN,
|
||||
contentRating = ContentRating.ADULT,
|
||||
coverUrl = td1.selectFirst("img")?.attrAsAbsoluteUrlOrNull("src"),
|
||||
tags = tagsDiv.parseTags(),
|
||||
state = when {
|
||||
rawTitle.contains("(ongoing)", ignoreCase = true) -> MangaState.ONGOING
|
||||
else -> null
|
||||
},
|
||||
authors = setOfNotNull(author),
|
||||
source = source,
|
||||
)
|
||||
}
|
||||
}
|
||||
return root.children().mapNotNull { tr ->
|
||||
if (tr.childrenSize() != 2) return@mapNotNull null
|
||||
val (td1, td2) = tr.children()
|
||||
val gLink = td2.selectFirstOrThrow("div.glink")
|
||||
val a = gLink.parents().select("a").first() ?: gLink.parseFailed("link not found")
|
||||
val href = a.attrAsRelativeUrl("href")
|
||||
val tagsDiv = gLink.nextElementSibling() ?: gLink.parseFailed("tags div not found")
|
||||
val rawTitle = gLink.text()
|
||||
val author = tagsDiv.getElementsContainingOwnText("artist:").first()
|
||||
?.nextElementSibling()?.textOrNull()
|
||||
Manga(
|
||||
id = generateUid(href),
|
||||
title = rawTitle.cleanupTitle(),
|
||||
altTitles = emptySet(),
|
||||
url = href,
|
||||
publicUrl = a.absUrl("href"),
|
||||
rating = td2.selectFirst("div.ir")?.parseRating() ?: RATING_UNKNOWN,
|
||||
contentRating = ContentRating.ADULT,
|
||||
coverUrl = td1.selectFirst("img")?.attrAsAbsoluteUrlOrNull("src"),
|
||||
tags = tagsDiv.parseTags(),
|
||||
state = when {
|
||||
rawTitle.contains("(ongoing)", ignoreCase = true) -> MangaState.ONGOING
|
||||
else -> null
|
||||
},
|
||||
authors = setOfNotNull(author),
|
||||
source = source,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
override suspend fun getDetails(manga: Manga): Manga {
|
||||
val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml()
|
||||
val root = doc.body().selectFirstOrThrow("div.gm")
|
||||
val cover = root.getElementById("gd1")?.children()?.first()
|
||||
val title = root.getElementById("gd2")
|
||||
val tagList = root.getElementById("taglist")
|
||||
val tabs = doc.body().selectFirst("table.ptt")?.selectFirst("tr")
|
||||
val gd3 = root.getElementById("gd3")
|
||||
val lang = gd3
|
||||
?.selectFirst("tr:contains(Language)")
|
||||
?.selectFirst(".gdt2")?.ownTextOrNull()
|
||||
val uploadDate = gd3
|
||||
?.selectFirst("tr:contains(Posted)")
|
||||
?.selectFirst(".gdt2")?.ownTextOrNull()
|
||||
.let { SimpleDateFormat("yyyy-MM-dd HH:mm", sourceLocale).parseSafe(it) }
|
||||
val uploader = gd3
|
||||
?.getElementsByAttributeValueContaining("href", "/uploader/")
|
||||
?.firstOrNull()
|
||||
?.ownTextOrNull()
|
||||
val tags = tagList?.parseTags().orEmpty()
|
||||
override suspend fun getDetails(manga: Manga): Manga {
|
||||
val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml()
|
||||
val root = doc.body().selectFirstOrThrow("div.gm")
|
||||
val cover = root.getElementById("gd1")?.children()?.first()
|
||||
val title = root.getElementById("gd2")
|
||||
val tagList = root.getElementById("taglist")
|
||||
val tabs = doc.body().selectFirst("table.ptt")?.selectFirst("tr")
|
||||
val gd3 = root.getElementById("gd3")
|
||||
val lang = gd3
|
||||
?.selectFirst("tr:contains(Language)")
|
||||
?.selectFirst(".gdt2")?.ownTextOrNull()
|
||||
val uploadDate = gd3
|
||||
?.selectFirst("tr:contains(Posted)")
|
||||
?.selectFirst(".gdt2")?.ownTextOrNull()
|
||||
.let { SimpleDateFormat("yyyy-MM-dd HH:mm", sourceLocale).parseSafe(it) }
|
||||
val uploader = gd3
|
||||
?.getElementsByAttributeValueContaining("href", "/uploader/")
|
||||
?.firstOrNull()
|
||||
?.ownTextOrNull()
|
||||
val tags = tagList?.parseTags().orEmpty()
|
||||
|
||||
return manga.copy(
|
||||
title = title?.getElementById("gn")?.text()?.cleanupTitle() ?: manga.title,
|
||||
altTitles = setOfNotNull(title?.getElementById("gj")?.text()?.cleanupTitle()?.nullIfEmpty()),
|
||||
publicUrl = doc.baseUri().ifEmpty { manga.publicUrl },
|
||||
rating = root.getElementById("rating_label")?.text()
|
||||
?.substringAfterLast(' ')
|
||||
?.toFloatOrNull()
|
||||
?.div(5f) ?: manga.rating,
|
||||
largeCoverUrl = cover?.styleValueOrNull("background")?.cssUrl(),
|
||||
tags = manga.tags + tags,
|
||||
description = tagList?.select("tr")?.joinToString("<br>") { tr ->
|
||||
val (tc, td) = tr.children()
|
||||
val subTags = td.select("a").joinToString { it.html() }
|
||||
"<b>${tc.html()}</b> $subTags"
|
||||
},
|
||||
chapters = tabs?.select("a")?.findLast { a ->
|
||||
a.text().toIntOrNull() != null
|
||||
}?.let { a ->
|
||||
val count = a.text().toInt()
|
||||
val chapters = ChaptersListBuilder(count)
|
||||
for (i in 1..count) {
|
||||
val url = "${manga.url}?p=${i - 1}"
|
||||
chapters += MangaChapter(
|
||||
id = generateUid(url),
|
||||
title = null,
|
||||
number = i.toFloat(),
|
||||
volume = 0,
|
||||
url = url,
|
||||
uploadDate = uploadDate,
|
||||
source = source,
|
||||
scanlator = uploader,
|
||||
branch = lang,
|
||||
)
|
||||
}
|
||||
chapters.toList()
|
||||
},
|
||||
)
|
||||
}
|
||||
return manga.copy(
|
||||
title = title?.getElementById("gn")?.text()?.cleanupTitle() ?: manga.title,
|
||||
altTitles = setOfNotNull(title?.getElementById("gj")?.text()?.cleanupTitle()?.nullIfEmpty()),
|
||||
publicUrl = doc.baseUri().ifEmpty { manga.publicUrl },
|
||||
rating = root.getElementById("rating_label")?.text()
|
||||
?.substringAfterLast(' ')
|
||||
?.toFloatOrNull()
|
||||
?.div(5f) ?: manga.rating,
|
||||
largeCoverUrl = cover?.styleValueOrNull("background")?.cssUrl(),
|
||||
tags = manga.tags + tags,
|
||||
description = tagList?.select("tr")?.joinToString("<br>") { tr ->
|
||||
val (tc, td) = tr.children()
|
||||
val subTags = td.select("a").joinToString { it.html() }
|
||||
"<b>${tc.html()}</b> $subTags"
|
||||
},
|
||||
chapters = tabs?.select("a")?.findLast { a ->
|
||||
a.text().toIntOrNull() != null
|
||||
}?.let { a ->
|
||||
val count = a.text().toInt()
|
||||
val chapters = ChaptersListBuilder(count)
|
||||
for (i in 1..count) {
|
||||
val url = "${manga.url}?p=${i - 1}"
|
||||
chapters += MangaChapter(
|
||||
id = generateUid(url),
|
||||
title = null,
|
||||
number = i.toFloat(),
|
||||
volume = 0,
|
||||
url = url,
|
||||
uploadDate = uploadDate,
|
||||
source = source,
|
||||
scanlator = uploader,
|
||||
branch = lang,
|
||||
)
|
||||
}
|
||||
chapters.toList()
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
|
||||
val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml()
|
||||
val root = doc.body().requireElementById("gdt")
|
||||
return root.select("a").map { a ->
|
||||
val url = a.attrAsRelativeUrl("href")
|
||||
MangaPage(
|
||||
id = generateUid(url),
|
||||
url = url,
|
||||
preview = a.children().firstOrNull()?.extractPreview(),
|
||||
source = source,
|
||||
)
|
||||
}
|
||||
}
|
||||
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
|
||||
val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml()
|
||||
val root = doc.body().requireElementById("gdt")
|
||||
return root.select("a").map { a ->
|
||||
val url = a.attrAsRelativeUrl("href")
|
||||
MangaPage(
|
||||
id = generateUid(url),
|
||||
url = url,
|
||||
preview = a.children().firstOrNull()?.extractPreview(),
|
||||
source = source,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
override suspend fun getPageUrl(page: MangaPage): String {
|
||||
val doc = webClient.httpGet(page.url.toAbsoluteUrl(domain)).parseHtml()
|
||||
return doc.body().requireElementById("img").attrAsAbsoluteUrl("src")
|
||||
}
|
||||
override suspend fun getPageUrl(page: MangaPage): String {
|
||||
val doc = webClient.httpGet(page.url.toAbsoluteUrl(domain)).parseHtml()
|
||||
return doc.body().requireElementById("img").attrAsAbsoluteUrl("src")
|
||||
}
|
||||
|
||||
@Suppress("SpellCheckingInspection")
|
||||
private val tags: String
|
||||
get() = "ahegao,anal,angel,apron,bandages,bbw,bdsm,beauty mark,big areolae,big ass,big breasts,big clit,big lips," +
|
||||
"big nipples,bikini,blackmail,bloomers,blowjob,bodysuit,bondage,breast expansion,bukkake,bunny girl,business suit," +
|
||||
"catgirl,centaur,cheating,chinese dress,christmas,collar,corset,cosplaying,cowgirl,crossdressing,cunnilingus," +
|
||||
"dark skin,daughter,deepthroat,defloration,demon girl,double penetration,dougi,dragon,drunk,elf,exhibitionism,farting," +
|
||||
"females only,femdom,filming,fingering,fishnets,footjob,fox girl,furry,futanari,garter belt,ghost,giantess," +
|
||||
"glasses,gloves,goblin,gothic lolita,growth,guro,gyaru,hair buns,hairy,hairy armpits,handjob,harem,hidden sex," +
|
||||
"horns,huge breasts,humiliation,impregnation,incest,inverted nipples,kemonomimi,kimono,kissing,lactation," +
|
||||
"latex,leg lock,leotard,lingerie,lizard girl,maid,masked face,masturbation,midget,miko,milf,mind break," +
|
||||
"mind control,monster girl,mother,muscle,nakadashi,netorare,nose hook,nun,nurse,oil,paizuri,panda girl," +
|
||||
"pantyhose,piercing,pixie cut,policewoman,ponytail,pregnant,rape,rimjob,robot,scat,lolicon,schoolgirl uniform," +
|
||||
"sex toys,shemale,sister,small breasts,smell,sole dickgirl,sole female,squirting,stockings,sundress,sweating," +
|
||||
"swimsuit,swinging,tail,tall girl,teacher,tentacles,thigh high boots,tomboy,transformation,twins,twintails," +
|
||||
"unusual pupils,urination,vore,vtuber,widow,wings,witch,wolf girl,x-ray,yuri,zombie,sole male,males only,yaoi," +
|
||||
"tomgirl,tall man,oni,shotacon,prostate massage,policeman,males only,huge penis,fox boy,feminization,dog boy,dickgirl on male,big penis"
|
||||
@Suppress("SpellCheckingInspection")
|
||||
private val tags: String
|
||||
get() = "ahegao,anal,angel,apron,bandages,bbw,bdsm,beauty mark,big areolae,big ass,big breasts,big clit,big lips," +
|
||||
"big nipples,bikini,blackmail,bloomers,blowjob,bodysuit,bondage,breast expansion,bukkake,bunny girl,business suit," +
|
||||
"catgirl,centaur,cheating,chinese dress,christmas,collar,corset,cosplaying,cowgirl,crossdressing,cunnilingus," +
|
||||
"dark skin,daughter,deepthroat,defloration,demon girl,double penetration,dougi,dragon,drunk,elf,exhibitionism,farting," +
|
||||
"females only,femdom,filming,fingering,fishnets,footjob,fox girl,furry,futanari,garter belt,ghost,giantess," +
|
||||
"glasses,gloves,goblin,gothic lolita,growth,guro,gyaru,hair buns,hairy,hairy armpits,handjob,harem,hidden sex," +
|
||||
"horns,huge breasts,humiliation,impregnation,incest,inverted nipples,kemonomimi,kimono,kissing,lactation," +
|
||||
"latex,leg lock,leotard,lingerie,lizard girl,maid,masked face,masturbation,midget,miko,milf,mind break," +
|
||||
"mind control,monster girl,mother,muscle,nakadashi,netorare,nose hook,nun,nurse,oil,paizuri,panda girl," +
|
||||
"pantyhose,piercing,pixie cut,policewoman,ponytail,pregnant,rape,rimjob,robot,scat,lolicon,schoolgirl uniform," +
|
||||
"sex toys,shemale,sister,small breasts,smell,sole dickgirl,sole female,squirting,stockings,sundress,sweating," +
|
||||
"swimsuit,swinging,tail,tall girl,teacher,tentacles,thigh high boots,tomboy,transformation,twins,twintails," +
|
||||
"unusual pupils,urination,vore,vtuber,widow,wings,witch,wolf girl,x-ray,yuri,zombie,sole male,males only,yaoi," +
|
||||
"tomgirl,tall man,oni,shotacon,prostate massage,policeman,males only,huge penis,fox boy,feminization,dog boy,dickgirl on male,big penis"
|
||||
|
||||
private fun mapTags(): Set<MangaTag> {
|
||||
val tagElements = tags.split(",")
|
||||
val result = ArraySet<MangaTag>(tagElements.size)
|
||||
for (tag in tagElements) {
|
||||
val el = tag.trim()
|
||||
if (el.isEmpty()) continue
|
||||
result += MangaTag(
|
||||
title = el.toTitleCase(Locale.ENGLISH),
|
||||
key = el,
|
||||
source = source,
|
||||
)
|
||||
}
|
||||
return result
|
||||
}
|
||||
private fun mapTags(): Set<MangaTag> {
|
||||
val tagElements = tags.split(",")
|
||||
val result = ArraySet<MangaTag>(tagElements.size)
|
||||
for (tag in tagElements) {
|
||||
val el = tag.trim()
|
||||
if (el.isEmpty()) continue
|
||||
result += MangaTag(
|
||||
title = el.toTitleCase(Locale.ENGLISH),
|
||||
key = el,
|
||||
source = source,
|
||||
)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
override fun intercept(chain: Interceptor.Chain): Response {
|
||||
val response = chain.proceed(chain.request())
|
||||
if (response.headersContentLength() <= 256) {
|
||||
val text = response.peekBody(256).use { it.string() }
|
||||
if (text.contains("IP address has been temporarily banned", ignoreCase = true)) {
|
||||
val hours = Regex("([0-9]+) hours?").find(text)?.groupValues?.getOrNull(1)?.toLongOrNull() ?: 0
|
||||
val minutes = Regex("([0-9]+) minutes?").find(text)?.groupValues?.getOrNull(1)?.toLongOrNull() ?: 0
|
||||
val seconds = Regex("([0-9]+) seconds?").find(text)?.groupValues?.getOrNull(1)?.toLongOrNull() ?: 0
|
||||
response.closeQuietly()
|
||||
throw TooManyRequestExceptions(
|
||||
url = response.request.url.toString(),
|
||||
retryAfter = TimeUnit.HOURS.toMillis(hours)
|
||||
+ TimeUnit.MINUTES.toMillis(minutes)
|
||||
+ TimeUnit.SECONDS.toMillis(seconds),
|
||||
)
|
||||
}
|
||||
}
|
||||
val imageRect = response.request.url.fragment?.split(',')
|
||||
if (imageRect != null && imageRect.size == 4) {
|
||||
// rect: top,left,right,bottom
|
||||
return context.redrawImageResponse(response) { bitmap ->
|
||||
val srcRect = Rect(
|
||||
left = imageRect[0].toInt(),
|
||||
top = imageRect[1].toInt(),
|
||||
right = imageRect[2].toInt(),
|
||||
bottom = imageRect[3].toInt(),
|
||||
)
|
||||
val dstRect = Rect(0, 0, srcRect.width, srcRect.height)
|
||||
val result = context.createBitmap(dstRect.width, dstRect.height)
|
||||
result.drawBitmap(bitmap, srcRect, dstRect)
|
||||
result
|
||||
}
|
||||
}
|
||||
return response
|
||||
}
|
||||
override fun intercept(chain: Interceptor.Chain): Response {
|
||||
val response = chain.proceed(chain.request())
|
||||
if (response.headersContentLength(BANNED_RESPONSE_LENGTH) <= BANNED_RESPONSE_LENGTH) {
|
||||
val text = response.peekBody(BANNED_RESPONSE_LENGTH).use { it.string() }
|
||||
if (text.contains("IP address has been temporarily banned", ignoreCase = true)) {
|
||||
val hours = Regex("([0-9]+) hours?").find(text)?.groupValues?.getOrNull(1)?.toLongOrNull() ?: 0
|
||||
val minutes = Regex("([0-9]+) minutes?").find(text)?.groupValues?.getOrNull(1)?.toLongOrNull() ?: 0
|
||||
val seconds = Regex("([0-9]+) seconds?").find(text)?.groupValues?.getOrNull(1)?.toLongOrNull() ?: 0
|
||||
response.closeQuietly()
|
||||
throw TooManyRequestExceptions(
|
||||
url = response.request.url.toString(),
|
||||
retryAfter = TimeUnit.HOURS.toMillis(hours)
|
||||
+ TimeUnit.MINUTES.toMillis(minutes)
|
||||
+ TimeUnit.SECONDS.toMillis(seconds),
|
||||
)
|
||||
}
|
||||
}
|
||||
val imageRect = response.request.url.fragment?.split(',')
|
||||
if (imageRect != null && imageRect.size == 4) {
|
||||
// rect: top,left,right,bottom
|
||||
return context.redrawImageResponse(response) { bitmap ->
|
||||
val srcRect = Rect(
|
||||
left = imageRect[0].toInt(),
|
||||
top = imageRect[1].toInt(),
|
||||
right = imageRect[2].toInt(),
|
||||
bottom = imageRect[3].toInt(),
|
||||
)
|
||||
val dstRect = Rect(0, 0, srcRect.width, srcRect.height)
|
||||
val result = context.createBitmap(dstRect.width, dstRect.height)
|
||||
result.drawBitmap(bitmap, srcRect, dstRect)
|
||||
result
|
||||
}
|
||||
}
|
||||
return response
|
||||
}
|
||||
|
||||
private fun Locale.toLanguagePath() = when (language) {
|
||||
else -> getDisplayLanguage(Locale.ENGLISH).lowercase()
|
||||
}
|
||||
private fun Locale.toLanguagePath() = when (language) {
|
||||
else -> getDisplayLanguage(Locale.ENGLISH).lowercase()
|
||||
}
|
||||
|
||||
override suspend fun getUsername(): String {
|
||||
val doc = webClient.httpGet("https://forums.$DOMAIN_UNAUTHORIZED/").parseHtml().body()
|
||||
val username = doc.getElementById("userlinks")
|
||||
?.getElementsByAttributeValueContaining("href", "showuser=")
|
||||
?.firstOrNull()
|
||||
?.ownText()
|
||||
?: if (doc.getElementById("userlinksguest") != null) {
|
||||
throw AuthRequiredException(source)
|
||||
} else {
|
||||
doc.parseFailed()
|
||||
}
|
||||
return username
|
||||
}
|
||||
override suspend fun getUsername(): String {
|
||||
val doc = webClient.httpGet("https://forums.$DOMAIN_UNAUTHORIZED/").parseHtml().body()
|
||||
val username = doc.getElementById("userlinks")
|
||||
?.getElementsByAttributeValueContaining("href", "showuser=")
|
||||
?.firstOrNull()
|
||||
?.ownText()
|
||||
?: if (doc.getElementById("userlinksguest") != null) {
|
||||
throw AuthRequiredException(source)
|
||||
} else {
|
||||
doc.parseFailed()
|
||||
}
|
||||
return username
|
||||
}
|
||||
|
||||
override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) {
|
||||
super.onCreateConfig(keys)
|
||||
keys.add(userAgentKey)
|
||||
keys.add(suspiciousContentKey)
|
||||
}
|
||||
override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) {
|
||||
super.onCreateConfig(keys)
|
||||
keys.add(userAgentKey)
|
||||
keys.add(suspiciousContentKey)
|
||||
}
|
||||
|
||||
override suspend fun getRelatedManga(seed: Manga): List<Manga> {
|
||||
val query = seed.title
|
||||
return getListPage(
|
||||
page = 0,
|
||||
order = defaultSortOrder,
|
||||
filter = MangaListFilter(query = query),
|
||||
)
|
||||
}
|
||||
override suspend fun getRelatedManga(seed: Manga): List<Manga> {
|
||||
val query = seed.title
|
||||
return getListPage(
|
||||
page = 0,
|
||||
order = defaultSortOrder,
|
||||
filter = MangaListFilter(query = query),
|
||||
)
|
||||
}
|
||||
|
||||
private fun isAuthorized(domain: String): Boolean {
|
||||
val cookies = context.cookieJar.getCookies(domain).mapToSet { x -> x.name }
|
||||
return authCookies.all { it in cookies }
|
||||
}
|
||||
private fun isAuthorized(domain: String): Boolean {
|
||||
val cookies = context.cookieJar.getCookies(domain).mapToSet { x -> x.name }
|
||||
return authCookies.all { it in cookies }
|
||||
}
|
||||
|
||||
private fun Element.parseRating(): Float {
|
||||
return runCatching {
|
||||
val style = requireNotNull(attr("style"))
|
||||
val (v1, v2) = ratingPattern.findAll(style).toList()
|
||||
var p1 = v1.groupValues.first().dropLast(2).toInt()
|
||||
val p2 = v2.groupValues.first().dropLast(2).toInt()
|
||||
if (p2 != -1) {
|
||||
p1 += 8
|
||||
}
|
||||
(80 - p1) / 80f
|
||||
}.getOrDefault(RATING_UNKNOWN)
|
||||
}
|
||||
private fun Element.parseRating(): Float {
|
||||
return runCatching {
|
||||
val style = requireNotNull(attr("style"))
|
||||
val (v1, v2) = ratingPattern.findAll(style).toList()
|
||||
var p1 = v1.groupValues.first().dropLast(2).toInt()
|
||||
val p2 = v2.groupValues.first().dropLast(2).toInt()
|
||||
if (p2 != -1) {
|
||||
p1 += 8
|
||||
}
|
||||
(80 - p1) / 80f
|
||||
}.getOrDefault(RATING_UNKNOWN)
|
||||
}
|
||||
|
||||
private fun String.cleanupTitle(): String {
|
||||
return replace(titleCleanupPattern, "")
|
||||
.replace(spacesCleanupPattern, "")
|
||||
}
|
||||
private fun String.cleanupTitle(): String {
|
||||
return replace(titleCleanupPattern, "")
|
||||
.replace(spacesCleanupPattern, "")
|
||||
}
|
||||
|
||||
private fun Element.parseTags(): Set<MangaTag> {
|
||||
private fun Element.parseTags(): Set<MangaTag> {
|
||||
|
||||
fun Element.parseTag() = textOrNull()?.let {
|
||||
MangaTag(title = it.toTitleCase(Locale.ENGLISH), key = it, source = source)
|
||||
}
|
||||
fun Element.parseTag() = textOrNull()?.let {
|
||||
MangaTag(title = it.toTitleCase(Locale.ENGLISH), key = it, source = source)
|
||||
}
|
||||
|
||||
val result = ArraySet<MangaTag>()
|
||||
for (prefix in TAG_PREFIXES) {
|
||||
getElementsByAttributeValueStarting("id", "ta_$prefix").mapNotNullTo(result, Element::parseTag)
|
||||
getElementsByAttributeValueStarting("title", prefix).mapNotNullTo(result, Element::parseTag)
|
||||
}
|
||||
return result
|
||||
}
|
||||
val result = ArraySet<MangaTag>()
|
||||
for (prefix in TAG_PREFIXES) {
|
||||
getElementsByAttributeValueStarting("id", "ta_$prefix").mapNotNullTo(result, Element::parseTag)
|
||||
getElementsByAttributeValueStarting("title", prefix).mapNotNullTo(result, Element::parseTag)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
private fun Element.extractPreview(): String? {
|
||||
val bg = backgroundOrNull() ?: return null
|
||||
return buildString {
|
||||
append(bg.url)
|
||||
append('#')
|
||||
// rect: left,top,right,bottom
|
||||
append(bg.left)
|
||||
append(',')
|
||||
append(bg.top)
|
||||
append(',')
|
||||
append(bg.right)
|
||||
append(',')
|
||||
append(bg.bottom)
|
||||
}
|
||||
}
|
||||
private fun Element.extractPreview(): String? {
|
||||
val bg = backgroundOrNull() ?: return null
|
||||
return buildString {
|
||||
append(bg.url)
|
||||
append('#')
|
||||
// rect: left,top,right,bottom
|
||||
append(bg.left)
|
||||
append(',')
|
||||
append(bg.top)
|
||||
append(',')
|
||||
append(bg.right)
|
||||
append(',')
|
||||
append(bg.bottom)
|
||||
}
|
||||
}
|
||||
|
||||
private fun getNextTimestamp(root: Element): Long {
|
||||
return root.getElementById("unext")
|
||||
?.attrAsAbsoluteUrlOrNull("href")
|
||||
?.toHttpUrlOrNull()
|
||||
?.queryParameter("next")
|
||||
?.toLongOrNull() ?: 1
|
||||
}
|
||||
private fun getNextTimestamp(root: Element): Long {
|
||||
return root.getElementById("unext")
|
||||
?.attrAsAbsoluteUrlOrNull("href")
|
||||
?.toHttpUrlOrNull()
|
||||
?.queryParameter("next")
|
||||
?.toLongOrNull() ?: 1
|
||||
}
|
||||
|
||||
private fun MangaListFilter.toSearchQuery(): String? {
|
||||
if (isEmpty()) {
|
||||
return null
|
||||
}
|
||||
val joiner = StringUtil.StringJoiner(" ")
|
||||
if (!query.isNullOrEmpty()) {
|
||||
joiner.add(query)
|
||||
}
|
||||
for (tag in tags) {
|
||||
if (tag.key.isNumeric()) {
|
||||
continue
|
||||
}
|
||||
joiner.add("tag:\"")
|
||||
joiner.append(tag.key)
|
||||
joiner.append("\"$")
|
||||
}
|
||||
for (tag in tagsExclude) {
|
||||
if (tag.key.isNumeric()) {
|
||||
continue
|
||||
}
|
||||
joiner.add("-tag:\"")
|
||||
joiner.append(tag.key)
|
||||
joiner.append("\"$")
|
||||
}
|
||||
locale?.let { lc ->
|
||||
joiner.add("language:\"")
|
||||
joiner.append(lc.toLanguagePath())
|
||||
joiner.append("\"$")
|
||||
}
|
||||
if (!author.isNullOrEmpty()) {
|
||||
joiner.add("artist:\"")
|
||||
joiner.append(author)
|
||||
joiner.append("\"$")
|
||||
}
|
||||
return joiner.complete().nullIfEmpty()
|
||||
}
|
||||
private fun MangaListFilter.toSearchQuery(): String? {
|
||||
if (isEmpty()) {
|
||||
return null
|
||||
}
|
||||
val joiner = StringUtil.StringJoiner(" ")
|
||||
if (!query.isNullOrEmpty()) {
|
||||
joiner.add(query)
|
||||
}
|
||||
for (tag in tags) {
|
||||
if (tag.key.isNumeric()) {
|
||||
continue
|
||||
}
|
||||
joiner.add("tag:\"")
|
||||
joiner.append(tag.key)
|
||||
joiner.append("\"$")
|
||||
}
|
||||
for (tag in tagsExclude) {
|
||||
if (tag.key.isNumeric()) {
|
||||
continue
|
||||
}
|
||||
joiner.add("-tag:\"")
|
||||
joiner.append(tag.key)
|
||||
joiner.append("\"$")
|
||||
}
|
||||
locale?.let { lc ->
|
||||
joiner.add("language:\"")
|
||||
joiner.append(lc.toLanguagePath())
|
||||
joiner.append("\"$")
|
||||
}
|
||||
if (!author.isNullOrEmpty()) {
|
||||
joiner.add("artist:\"")
|
||||
joiner.append(author)
|
||||
joiner.append("\"$")
|
||||
}
|
||||
return joiner.complete().nullIfEmpty()
|
||||
}
|
||||
|
||||
private fun Collection<ContentType>.toFCats(): Int = fold(0) { acc, ct ->
|
||||
val cat: Int = when (ct) {
|
||||
ContentType.DOUJINSHI -> 2
|
||||
ContentType.MANGA -> 4
|
||||
ContentType.ARTIST_CG -> 8
|
||||
ContentType.GAME_CG -> 16
|
||||
ContentType.COMICS -> 512
|
||||
ContentType.IMAGE_SET -> 32
|
||||
else -> 449 // 1 or 64 or 128 or 256
|
||||
}
|
||||
acc or cat
|
||||
}
|
||||
private fun Collection<ContentType>.toFCats(): Int = fold(0) { acc, ct ->
|
||||
val cat: Int = when (ct) {
|
||||
ContentType.DOUJINSHI -> 2
|
||||
ContentType.MANGA -> 4
|
||||
ContentType.ARTIST_CG -> 8
|
||||
ContentType.GAME_CG -> 16
|
||||
ContentType.COMICS -> 512
|
||||
ContentType.IMAGE_SET -> 32
|
||||
else -> 449 // 1 or 64 or 128 or 256
|
||||
}
|
||||
acc or cat
|
||||
}
|
||||
|
||||
private fun checkAuth(): Boolean {
|
||||
val authorized = isAuthorized(DOMAIN_UNAUTHORIZED)
|
||||
if (authorized) {
|
||||
if (!isAuthorized(DOMAIN_AUTHORIZED)) {
|
||||
context.cookieJar.copyCookies(
|
||||
DOMAIN_UNAUTHORIZED,
|
||||
DOMAIN_AUTHORIZED,
|
||||
authCookies,
|
||||
)
|
||||
context.cookieJar.insertCookies(DOMAIN_AUTHORIZED, "yay=louder")
|
||||
}
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
private fun checkAuth(): Boolean {
|
||||
val authorized = isAuthorized(DOMAIN_UNAUTHORIZED)
|
||||
if (authorized) {
|
||||
if (!isAuthorized(DOMAIN_AUTHORIZED)) {
|
||||
context.cookieJar.copyCookies(
|
||||
DOMAIN_UNAUTHORIZED,
|
||||
DOMAIN_AUTHORIZED,
|
||||
authCookies,
|
||||
)
|
||||
context.cookieJar.insertCookies(DOMAIN_AUTHORIZED, "yay=louder")
|
||||
}
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,6 +19,7 @@ import org.koitharu.kotatsu.parsers.model.MangaTag
|
||||
import org.koitharu.kotatsu.parsers.model.RATING_UNKNOWN
|
||||
import org.koitharu.kotatsu.parsers.model.SortOrder
|
||||
import org.koitharu.kotatsu.parsers.util.attrAsRelativeUrl
|
||||
import org.koitharu.kotatsu.parsers.util.attrOrNull
|
||||
import org.koitharu.kotatsu.parsers.util.generateUid
|
||||
import org.koitharu.kotatsu.parsers.util.mapChapters
|
||||
import org.koitharu.kotatsu.parsers.util.mapNotNullToSet
|
||||
@@ -28,6 +29,7 @@ import org.koitharu.kotatsu.parsers.util.parseFailed
|
||||
import org.koitharu.kotatsu.parsers.util.parseHtml
|
||||
import org.koitharu.kotatsu.parsers.util.selectFirstOrThrow
|
||||
import org.koitharu.kotatsu.parsers.util.src
|
||||
import org.koitharu.kotatsu.parsers.util.textOrNull
|
||||
import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl
|
||||
import org.koitharu.kotatsu.parsers.util.toTitleCase
|
||||
import org.koitharu.kotatsu.parsers.util.urlEncoded
|
||||
@@ -36,232 +38,232 @@ import java.util.EnumSet
|
||||
|
||||
@MangaSourceParser("KDTSCANS", "KdtScans", "en")
|
||||
internal class KdtScans(context: MangaLoaderContext) :
|
||||
PagedMangaParser(context, MangaParserSource.KDTSCANS, 20) {
|
||||
PagedMangaParser(context, MangaParserSource.KDTSCANS, 20) {
|
||||
|
||||
override val configKeyDomain = ConfigKey.Domain("www.silentquill.net")
|
||||
override val configKeyDomain = ConfigKey.Domain("www.silentquill.net")
|
||||
|
||||
override val availableSortOrders: Set<SortOrder> = EnumSet.of(
|
||||
SortOrder.RELEVANCE,
|
||||
SortOrder.UPDATED,
|
||||
SortOrder.POPULARITY,
|
||||
SortOrder.NEWEST,
|
||||
SortOrder.ALPHABETICAL,
|
||||
SortOrder.ALPHABETICAL_DESC,
|
||||
)
|
||||
override val availableSortOrders: Set<SortOrder> = EnumSet.of(
|
||||
SortOrder.RELEVANCE,
|
||||
SortOrder.UPDATED,
|
||||
SortOrder.POPULARITY,
|
||||
SortOrder.NEWEST,
|
||||
SortOrder.ALPHABETICAL,
|
||||
SortOrder.ALPHABETICAL_DESC,
|
||||
)
|
||||
|
||||
override val filterCapabilities = MangaListFilterCapabilities(
|
||||
isSearchSupported = true,
|
||||
isMultipleTagsSupported = true,
|
||||
isTagsExclusionSupported = true,
|
||||
)
|
||||
override val filterCapabilities = MangaListFilterCapabilities(
|
||||
isSearchSupported = true,
|
||||
isMultipleTagsSupported = true,
|
||||
isTagsExclusionSupported = true,
|
||||
)
|
||||
|
||||
override suspend fun getFilterOptions(): MangaListFilterOptions {
|
||||
return MangaListFilterOptions(
|
||||
availableTags = fetchAvailableTags(),
|
||||
availableStates = EnumSet.of(
|
||||
MangaState.ONGOING,
|
||||
MangaState.FINISHED,
|
||||
MangaState.PAUSED,
|
||||
),
|
||||
availableContentTypes = EnumSet.of(
|
||||
ContentType.MANGA,
|
||||
ContentType.MANHWA,
|
||||
ContentType.MANHUA,
|
||||
ContentType.COMICS,
|
||||
ContentType.NOVEL,
|
||||
),
|
||||
)
|
||||
}
|
||||
override suspend fun getFilterOptions(): MangaListFilterOptions {
|
||||
return MangaListFilterOptions(
|
||||
availableTags = fetchAvailableTags(),
|
||||
availableStates = EnumSet.of(
|
||||
MangaState.ONGOING,
|
||||
MangaState.FINISHED,
|
||||
MangaState.PAUSED,
|
||||
),
|
||||
availableContentTypes = EnumSet.of(
|
||||
ContentType.MANGA,
|
||||
ContentType.MANHWA,
|
||||
ContentType.MANHUA,
|
||||
ContentType.COMICS,
|
||||
ContentType.NOVEL,
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> {
|
||||
val url = buildString {
|
||||
append("https://$domain/manga/?page=${page}")
|
||||
override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> {
|
||||
val url = buildString {
|
||||
append("https://$domain/manga/?page=${page}")
|
||||
|
||||
filter.query?.let {
|
||||
append("&s=${it.urlEncoded()}")
|
||||
}
|
||||
filter.query?.let {
|
||||
append("&s=${it.urlEncoded()}")
|
||||
}
|
||||
|
||||
val sortValue = when (order) {
|
||||
SortOrder.UPDATED -> "update"
|
||||
SortOrder.POPULARITY -> "popular"
|
||||
SortOrder.NEWEST -> "latest"
|
||||
SortOrder.ALPHABETICAL -> "title"
|
||||
SortOrder.ALPHABETICAL_DESC -> "titlereverse"
|
||||
else -> "" // Default/Relevance
|
||||
}
|
||||
if (sortValue.isNotEmpty()) {
|
||||
append("&order=$sortValue")
|
||||
}
|
||||
val sortValue = when (order) {
|
||||
SortOrder.UPDATED -> "update"
|
||||
SortOrder.POPULARITY -> "popular"
|
||||
SortOrder.NEWEST -> "latest"
|
||||
SortOrder.ALPHABETICAL -> "title"
|
||||
SortOrder.ALPHABETICAL_DESC -> "titlereverse"
|
||||
else -> "" // Default/Relevance
|
||||
}
|
||||
if (sortValue.isNotEmpty()) {
|
||||
append("&order=$sortValue")
|
||||
}
|
||||
|
||||
filter.tags.forEach { tag ->
|
||||
append("&genre[]=${tag.key}")
|
||||
}
|
||||
filter.tags.forEach { tag ->
|
||||
append("&genre[]=${tag.key}")
|
||||
}
|
||||
|
||||
filter.tagsExclude.forEach { tag ->
|
||||
append("&genre[]=-${tag.key}")
|
||||
}
|
||||
filter.tagsExclude.forEach { tag ->
|
||||
append("&genre[]=-${tag.key}")
|
||||
}
|
||||
|
||||
filter.states.oneOrThrowIfMany().let { state ->
|
||||
val stateValue = when (state) {
|
||||
MangaState.ONGOING -> "ongoing"
|
||||
MangaState.FINISHED -> "completed"
|
||||
MangaState.PAUSED -> "hiatus"
|
||||
else -> ""
|
||||
}
|
||||
if (stateValue.isNotEmpty()) {
|
||||
append("&status=$stateValue")
|
||||
}
|
||||
}
|
||||
filter.states.oneOrThrowIfMany().let { state ->
|
||||
val stateValue = when (state) {
|
||||
MangaState.ONGOING -> "ongoing"
|
||||
MangaState.FINISHED -> "completed"
|
||||
MangaState.PAUSED -> "hiatus"
|
||||
else -> ""
|
||||
}
|
||||
if (stateValue.isNotEmpty()) {
|
||||
append("&status=$stateValue")
|
||||
}
|
||||
}
|
||||
|
||||
filter.types.oneOrThrowIfMany()?.let { type ->
|
||||
val typeValue = when (type) {
|
||||
ContentType.MANGA -> "manga"
|
||||
ContentType.MANHWA -> "manhwa"
|
||||
ContentType.MANHUA -> "manhua"
|
||||
ContentType.COMICS -> "comic"
|
||||
ContentType.NOVEL -> "novel"
|
||||
else -> ""
|
||||
}
|
||||
if (typeValue.isNotEmpty()) {
|
||||
append("&type=$typeValue")
|
||||
}
|
||||
}
|
||||
}
|
||||
val doc = webClient.httpGet(url).parseHtml()
|
||||
return parseMangaList(doc)
|
||||
}
|
||||
filter.types.oneOrThrowIfMany()?.let { type ->
|
||||
val typeValue = when (type) {
|
||||
ContentType.MANGA -> "manga"
|
||||
ContentType.MANHWA -> "manhwa"
|
||||
ContentType.MANHUA -> "manhua"
|
||||
ContentType.COMICS -> "comic"
|
||||
ContentType.NOVEL -> "novel"
|
||||
else -> ""
|
||||
}
|
||||
if (typeValue.isNotEmpty()) {
|
||||
append("&type=$typeValue")
|
||||
}
|
||||
}
|
||||
}
|
||||
val doc = webClient.httpGet(url).parseHtml()
|
||||
return parseMangaList(doc)
|
||||
}
|
||||
|
||||
private fun parseMangaList(doc: Document): List<Manga> {
|
||||
val elements = doc.select("div.listupd div.bs")
|
||||
private fun parseMangaList(doc: Document): List<Manga> {
|
||||
val elements = doc.select("div.listupd div.bs")
|
||||
|
||||
if (elements.isEmpty()) {
|
||||
return emptyList()
|
||||
}
|
||||
if (elements.isEmpty()) {
|
||||
return emptyList()
|
||||
}
|
||||
|
||||
return elements.map { div ->
|
||||
val a = div.selectFirstOrThrow("a")
|
||||
val href = a.attrAsRelativeUrl("href")
|
||||
val img = div.selectFirst("img")
|
||||
val title = a.attr("title").ifEmpty {
|
||||
div.selectFirst(".tt")?.text().orEmpty()
|
||||
}
|
||||
val rating = div.selectFirst(".numscore")?.text()?.toFloatOrNull()?.div(10f) ?: RATING_UNKNOWN
|
||||
return elements.map { div ->
|
||||
val a = div.selectFirstOrThrow("a")
|
||||
val href = a.attrAsRelativeUrl("href")
|
||||
val img = div.selectFirst("img")
|
||||
val title = a.attr("title").ifEmpty {
|
||||
div.selectFirst(".tt")?.text().orEmpty()
|
||||
}
|
||||
val rating = div.selectFirst(".numscore")?.text()?.toFloatOrNull()?.div(10f) ?: RATING_UNKNOWN
|
||||
|
||||
Manga(
|
||||
id = generateUid(href),
|
||||
url = href,
|
||||
publicUrl = href.toAbsoluteUrl(domain),
|
||||
coverUrl = img?.src(),
|
||||
title = title,
|
||||
altTitles = emptySet(),
|
||||
rating = rating,
|
||||
tags = emptySet(),
|
||||
authors = emptySet(),
|
||||
state = parseStatus(div.selectFirst(".status")?.text().orEmpty()),
|
||||
source = source,
|
||||
contentRating = if (isNsfwSource) ContentRating.ADULT else null,
|
||||
)
|
||||
}
|
||||
}
|
||||
Manga(
|
||||
id = generateUid(href),
|
||||
url = href,
|
||||
publicUrl = href.toAbsoluteUrl(domain),
|
||||
coverUrl = img?.src(),
|
||||
title = title,
|
||||
altTitles = emptySet(),
|
||||
rating = rating,
|
||||
tags = emptySet(),
|
||||
authors = emptySet(),
|
||||
state = parseStatus(div.selectFirst(".status")?.text().orEmpty()),
|
||||
source = source,
|
||||
contentRating = if (isNsfwSource) ContentRating.ADULT else null,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
override suspend fun getDetails(manga: Manga): Manga {
|
||||
val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml()
|
||||
val infoElement =
|
||||
doc.selectFirst(".main-info, .postbody") ?: doc.parseFailed("Cannot find manga details element")
|
||||
override suspend fun getDetails(manga: Manga): Manga {
|
||||
val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml()
|
||||
val infoElement =
|
||||
doc.selectFirst(".main-info, .postbody") ?: doc.parseFailed("Cannot find manga details element")
|
||||
|
||||
val statusText =
|
||||
infoElement.selectFirst(".tsinfo .imptdt:contains(Status) i, .infotable tr:contains(Status) td:last-child")
|
||||
?.text()
|
||||
val statusText =
|
||||
infoElement.selectFirst(".tsinfo .imptdt:contains(Status) i, .infotable tr:contains(Status) td:last-child")
|
||||
?.text()
|
||||
|
||||
val chapters = doc.select("#chapterlist li").mapChapters(reversed = true) { i, li ->
|
||||
val a = li.selectFirstOrThrow("a")
|
||||
val href = a.attrAsRelativeUrl("href")
|
||||
MangaChapter(
|
||||
id = generateUid(href),
|
||||
url = href,
|
||||
title = a.selectFirst(".chapternum")?.text() ?: a.text(),
|
||||
number = i + 1f,
|
||||
uploadDate = parseChapterDate(li.selectFirst(".chapterdate")?.text()),
|
||||
source = source,
|
||||
volume = 0,
|
||||
scanlator = null,
|
||||
branch = null,
|
||||
)
|
||||
}
|
||||
val chapters = doc.select("#chapterlist li").mapChapters(reversed = true) { i, li ->
|
||||
val a = li.selectFirstOrThrow("a")
|
||||
val href = a.attrAsRelativeUrl("href")
|
||||
MangaChapter(
|
||||
id = generateUid(href),
|
||||
url = href,
|
||||
title = a.selectFirst(".chapternum")?.text() ?: a.text(),
|
||||
number = i + 1f,
|
||||
uploadDate = parseChapterDate(li.selectFirst(".chapterdate")?.text()),
|
||||
source = source,
|
||||
volume = 0,
|
||||
scanlator = null,
|
||||
branch = null,
|
||||
)
|
||||
}
|
||||
|
||||
val genres = infoElement.select(".mgen a, .seriestugenre a").mapToSet { a ->
|
||||
MangaTag(
|
||||
key = a.attr("href").substringAfterLast("/").removeSuffix("/"),
|
||||
title = a.text(),
|
||||
source = source,
|
||||
)
|
||||
}
|
||||
val genres = infoElement.select(".mgen a, .seriestugenre a").mapToSet { a ->
|
||||
MangaTag(
|
||||
key = a.attr("href").substringAfterLast("/").removeSuffix("/"),
|
||||
title = a.text(),
|
||||
source = source,
|
||||
)
|
||||
}
|
||||
|
||||
val typeTag = infoElement.selectFirst(".tsinfo .imptdt:contains(Type) a")?.text()?.let { typeText ->
|
||||
MangaTag(
|
||||
key = typeText.lowercase(),
|
||||
title = typeText.trim(),
|
||||
source = source,
|
||||
)
|
||||
}
|
||||
val typeTag = infoElement.selectFirst(".tsinfo .imptdt:contains(Type) a")?.text()?.let { typeText ->
|
||||
MangaTag(
|
||||
key = typeText.lowercase(),
|
||||
title = typeText.trim(),
|
||||
source = source,
|
||||
)
|
||||
}
|
||||
|
||||
val allTags = genres.toMutableSet()
|
||||
typeTag?.let { allTags.add(it) }
|
||||
val allTags = genres.toMutableSet()
|
||||
typeTag?.let { allTags.add(it) }
|
||||
|
||||
return manga.copy(
|
||||
title = infoElement.selectFirst("h1.entry-title")?.text() ?: manga.title,
|
||||
authors = infoElement.select(".tsinfo .imptdt:contains(Author) i, .infotable tr:contains(Author) td:last-child")
|
||||
.mapToSet { it.text() },
|
||||
description = infoElement.select(".desc, .entry-content[itemprop=description]")
|
||||
.joinToString("\n") { it.text() },
|
||||
state = parseStatus(statusText.orEmpty()),
|
||||
tags = allTags,
|
||||
chapters = chapters,
|
||||
)
|
||||
}
|
||||
return manga.copy(
|
||||
title = infoElement.selectFirst("h1.entry-title")?.text() ?: manga.title,
|
||||
authors = infoElement.select(".tsinfo .imptdt:contains(Author) i, .infotable tr:contains(Author) td:last-child")
|
||||
.mapToSet { it.text() },
|
||||
description = infoElement.select(".desc, .entry-content[itemprop=description]")
|
||||
.joinToString("\n") { it.text() },
|
||||
state = parseStatus(statusText.orEmpty()),
|
||||
tags = allTags,
|
||||
chapters = chapters,
|
||||
)
|
||||
}
|
||||
|
||||
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
|
||||
val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml()
|
||||
return doc.select("#readerarea img").map { img ->
|
||||
val url = img.attr("data-src").ifEmpty { img.src().orEmpty() }
|
||||
MangaPage(
|
||||
id = generateUid(url),
|
||||
url = url,
|
||||
preview = null,
|
||||
source = source,
|
||||
)
|
||||
}
|
||||
}
|
||||
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
|
||||
val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml()
|
||||
return doc.select("#readerarea img").map { img ->
|
||||
val url = img.attr("data-src").ifEmpty { img.src().orEmpty() }
|
||||
MangaPage(
|
||||
id = generateUid(url),
|
||||
url = url,
|
||||
preview = null,
|
||||
source = source,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
private fun parseStatus(status: String): MangaState? {
|
||||
return when {
|
||||
status.contains("ongoing", ignoreCase = true) -> MangaState.ONGOING
|
||||
status.contains("completed", ignoreCase = true) -> MangaState.FINISHED
|
||||
status.contains("hiatus", ignoreCase = true) -> MangaState.PAUSED
|
||||
status.contains("dropped", ignoreCase = true) -> MangaState.ABANDONED
|
||||
status.contains("canceled", ignoreCase = true) -> MangaState.ABANDONED
|
||||
else -> null
|
||||
}
|
||||
}
|
||||
private fun parseStatus(status: String): MangaState? {
|
||||
return when {
|
||||
status.contains("ongoing", ignoreCase = true) -> MangaState.ONGOING
|
||||
status.contains("completed", ignoreCase = true) -> MangaState.FINISHED
|
||||
status.contains("hiatus", ignoreCase = true) -> MangaState.PAUSED
|
||||
status.contains("dropped", ignoreCase = true) -> MangaState.ABANDONED
|
||||
status.contains("canceled", ignoreCase = true) -> MangaState.ABANDONED
|
||||
else -> null
|
||||
}
|
||||
}
|
||||
|
||||
private fun parseChapterDate(date: String?): Long {
|
||||
return try {
|
||||
SimpleDateFormat("MMMM dd, yyyy", sourceLocale).parse(date?.trim()).time
|
||||
} catch (_: Exception) {
|
||||
0L
|
||||
}
|
||||
}
|
||||
private fun parseChapterDate(date: String?): Long {
|
||||
return try {
|
||||
SimpleDateFormat("MMMM dd, yyyy", sourceLocale).parse(date?.trim()).time
|
||||
} catch (_: Exception) {
|
||||
0L
|
||||
}
|
||||
}
|
||||
|
||||
private suspend fun fetchAvailableTags(): Set<MangaTag> {
|
||||
val doc = webClient.httpGet("https://$domain/manga/").parseHtml()
|
||||
return doc.select("ul.genrez li").mapNotNullToSet { li ->
|
||||
val key = li.selectFirst("input").attr("value") ?: return@mapNotNullToSet null
|
||||
val title = li.selectFirst("label").text().toTitleCase()
|
||||
MangaTag(
|
||||
key = key,
|
||||
title = title,
|
||||
source = source,
|
||||
)
|
||||
}
|
||||
}
|
||||
private suspend fun fetchAvailableTags(): Set<MangaTag> {
|
||||
val doc = webClient.httpGet("https://$domain/manga/").parseHtml()
|
||||
return doc.select("ul.genrez li").mapNotNullToSet { li ->
|
||||
val key = li.selectFirst("input")?.attrOrNull("value") ?: return@mapNotNullToSet null
|
||||
val title = li.selectFirst("label")?.textOrNull()?.toTitleCase() ?: return@mapNotNullToSet null
|
||||
MangaTag(
|
||||
key = key,
|
||||
title = title,
|
||||
source = source,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,174 +1,184 @@
|
||||
package org.koitharu.kotatsu.parsers.site.es
|
||||
|
||||
import org.koitharu.kotatsu.parsers.Broken
|
||||
import org.koitharu.kotatsu.parsers.MangaLoaderContext
|
||||
import org.koitharu.kotatsu.parsers.MangaSourceParser
|
||||
import org.koitharu.kotatsu.parsers.config.ConfigKey
|
||||
import org.koitharu.kotatsu.parsers.core.PagedMangaParser
|
||||
import org.koitharu.kotatsu.parsers.model.*
|
||||
import org.koitharu.kotatsu.parsers.util.*
|
||||
import org.koitharu.kotatsu.parsers.Broken
|
||||
import java.util.*
|
||||
|
||||
@Broken // Website closed
|
||||
@MangaSourceParser("DRAGONTRANSLATION", "Dragon Translation", "es")
|
||||
internal class DragonTranslationParser(context: MangaLoaderContext) : PagedMangaParser(context, MangaParserSource.DRAGONTRANSLATION, 30) {
|
||||
internal class DragonTranslationParser(context: MangaLoaderContext) :
|
||||
PagedMangaParser(context, MangaParserSource.DRAGONTRANSLATION, 30) {
|
||||
|
||||
override val configKeyDomain = ConfigKey.Domain("dragontranslation.net")
|
||||
override val configKeyDomain = ConfigKey.Domain("dragontranslation.net")
|
||||
|
||||
override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) {
|
||||
super.onCreateConfig(keys)
|
||||
keys.add(userAgentKey)
|
||||
}
|
||||
override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) {
|
||||
super.onCreateConfig(keys)
|
||||
keys.add(userAgentKey)
|
||||
}
|
||||
|
||||
override fun getRequestHeaders() = super.getRequestHeaders().newBuilder()
|
||||
.add("referer", "no-referrer")
|
||||
.build()
|
||||
override fun getRequestHeaders() = super.getRequestHeaders().newBuilder()
|
||||
.add("referer", "no-referrer")
|
||||
.build()
|
||||
|
||||
override val availableSortOrders: Set<SortOrder> = EnumSet.of(SortOrder.UPDATED)
|
||||
override val availableSortOrders: Set<SortOrder> = EnumSet.of(SortOrder.UPDATED)
|
||||
|
||||
override val filterCapabilities: MangaListFilterCapabilities
|
||||
get() = MangaListFilterCapabilities(
|
||||
isSearchSupported = true,
|
||||
)
|
||||
override val filterCapabilities: MangaListFilterCapabilities
|
||||
get() = MangaListFilterCapabilities(
|
||||
isSearchSupported = true,
|
||||
)
|
||||
|
||||
override suspend fun getFilterOptions() = MangaListFilterOptions(
|
||||
availableTags = emptySet(), // cant find any URLs for fetch tags
|
||||
availableContentTypes = EnumSet.of(ContentType.MANGA, ContentType.MANHWA, ContentType.MANHUA),
|
||||
)
|
||||
override suspend fun getFilterOptions() = MangaListFilterOptions(
|
||||
availableTags = emptySet(), // cant find any URLs for fetch tags
|
||||
availableContentTypes = EnumSet.of(ContentType.MANGA, ContentType.MANHWA, ContentType.MANHUA),
|
||||
)
|
||||
|
||||
override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> {
|
||||
val url = buildString {
|
||||
append("https://")
|
||||
append(domain)
|
||||
when {
|
||||
!filter.query.isNullOrEmpty() -> {
|
||||
append("/mangas?buscar=")
|
||||
append(filter.query.urlEncoded())
|
||||
append("&page=")
|
||||
append(page.toString())
|
||||
}
|
||||
override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> {
|
||||
val url = buildString {
|
||||
append("https://")
|
||||
append(domain)
|
||||
when {
|
||||
!filter.query.isNullOrEmpty() -> {
|
||||
append("/mangas?buscar=")
|
||||
append(filter.query.urlEncoded())
|
||||
append("&page=")
|
||||
append(page.toString())
|
||||
}
|
||||
|
||||
else -> {
|
||||
append("/mangas?page=")
|
||||
append(page.toString())
|
||||
else -> {
|
||||
append("/mangas?page=")
|
||||
append(page.toString())
|
||||
|
||||
val tag = filter.tags.oneOrThrowIfMany()
|
||||
if (filter.tags.isNotEmpty()) {
|
||||
append("&tag=")
|
||||
append(tag?.key.orEmpty())
|
||||
}
|
||||
val tag = filter.tags.oneOrThrowIfMany()
|
||||
if (filter.tags.isNotEmpty()) {
|
||||
append("&tag=")
|
||||
append(tag?.key.orEmpty())
|
||||
}
|
||||
|
||||
if (filter.types.isNotEmpty()) {
|
||||
append("&type=")
|
||||
when (filter.types.oneOrThrowIfMany()) {
|
||||
ContentType.MANGA -> "manga"
|
||||
ContentType.MANHWA -> "manhwa"
|
||||
ContentType.MANHUA -> "manhua"
|
||||
else -> ""
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (filter.types.isNotEmpty()) {
|
||||
append("&type=")
|
||||
append(
|
||||
when (filter.types.oneOrThrowIfMany()) {
|
||||
ContentType.MANGA -> "manga"
|
||||
ContentType.MANHWA -> "manhwa"
|
||||
ContentType.MANHUA -> "manhua"
|
||||
else -> ""
|
||||
},
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
val doc = webClient.httpGet(url).parseHtml()
|
||||
val row = doc.select("div.row.gy-3").firstOrNull() ?: return emptyList()
|
||||
return row.select("article.position-relative.card").mapNotNull { div ->
|
||||
val href = div.selectFirst("a.lanzador")?.attrAsRelativeUrlOrNull("href") ?: return@mapNotNull null
|
||||
val coverUrl = div.selectFirst("img.card-img-top.wp-post-image.lazy.loaded")?.src().orEmpty()
|
||||
Manga(
|
||||
id = generateUid(href),
|
||||
url = href,
|
||||
publicUrl = href,
|
||||
coverUrl = coverUrl,
|
||||
title = div.selectFirst("h2.card-title.fs-6.entry-title").text(),
|
||||
altTitles = emptySet(),
|
||||
rating = RATING_UNKNOWN,
|
||||
tags = emptySet(),
|
||||
authors = emptySet(),
|
||||
state = null,
|
||||
source = source,
|
||||
contentRating = if (isNsfwSource) ContentRating.ADULT else null,
|
||||
)
|
||||
}
|
||||
}
|
||||
val doc = webClient.httpGet(url).parseHtml()
|
||||
val row = doc.select("div.row.gy-3").firstOrNull() ?: return emptyList()
|
||||
return row.select("article.position-relative.card").mapNotNull { div ->
|
||||
val href = div.selectFirst("a.lanzador")?.attrAsRelativeUrlOrNull("href") ?: return@mapNotNull null
|
||||
val coverUrl = div.selectFirst("img.card-img-top.wp-post-image.lazy.loaded")?.src().orEmpty()
|
||||
Manga(
|
||||
id = generateUid(href),
|
||||
url = href,
|
||||
publicUrl = href,
|
||||
coverUrl = coverUrl,
|
||||
title = div.selectFirst("h2.card-title.fs-6.entry-title")?.text().orEmpty(),
|
||||
altTitles = emptySet(),
|
||||
rating = RATING_UNKNOWN,
|
||||
tags = emptySet(),
|
||||
authors = emptySet(),
|
||||
state = null,
|
||||
source = source,
|
||||
contentRating = if (isNsfwSource) ContentRating.ADULT else null,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
override suspend fun getDetails(manga: Manga): Manga {
|
||||
val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml()
|
||||
val statusText = doc.selectFirst("p:contains(Status:)")?.text()
|
||||
val status = when {
|
||||
statusText?.contains("publishing", ignoreCase = true) == true -> MangaState.ONGOING
|
||||
else -> null
|
||||
}
|
||||
override suspend fun getDetails(manga: Manga): Manga {
|
||||
val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml()
|
||||
val statusText = doc.selectFirst("p:contains(Status:)")?.text()
|
||||
val status = when {
|
||||
statusText?.contains("publishing", ignoreCase = true) == true -> MangaState.ONGOING
|
||||
else -> null
|
||||
}
|
||||
|
||||
val chapterElements = doc.select("ul.list-group a")
|
||||
val totalChapters = chapterElements.size
|
||||
val chapterElements = doc.select("ul.list-group a")
|
||||
val totalChapters = chapterElements.size
|
||||
|
||||
val chapters = chapterElements.mapIndexed { index, a ->
|
||||
val href = a.attrAsRelativeUrl("href")
|
||||
val title = a.text()
|
||||
MangaChapter(
|
||||
id = generateUid(href),
|
||||
title = title,
|
||||
number = totalChapters - index.toFloat(),
|
||||
volume = 0,
|
||||
url = href,
|
||||
scanlator = null,
|
||||
uploadDate = parseDate(a.selectFirst("span")?.text()),
|
||||
branch = null,
|
||||
source = source,
|
||||
)
|
||||
}
|
||||
val chapters = chapterElements.mapIndexed { index, a ->
|
||||
val href = a.attrAsRelativeUrl("href")
|
||||
val title = a.text()
|
||||
MangaChapter(
|
||||
id = generateUid(href),
|
||||
title = title,
|
||||
number = totalChapters - index.toFloat(),
|
||||
volume = 0,
|
||||
url = href,
|
||||
scanlator = null,
|
||||
uploadDate = parseDate(a.selectFirst("span")?.text()),
|
||||
branch = null,
|
||||
source = source,
|
||||
)
|
||||
}
|
||||
|
||||
return manga.copy(
|
||||
state = status,
|
||||
chapters = chapters.reversed(),
|
||||
)
|
||||
}
|
||||
return manga.copy(
|
||||
state = status,
|
||||
chapters = chapters.reversed(),
|
||||
)
|
||||
}
|
||||
|
||||
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
|
||||
val fullUrl = chapter.url.toAbsoluteUrl(domain)
|
||||
val doc = webClient.httpGet(fullUrl).parseHtml()
|
||||
return doc.select("div#chapter_imgs img").map { img ->
|
||||
val url = img.attr("src")
|
||||
MangaPage(
|
||||
id = generateUid(url),
|
||||
url = url,
|
||||
preview = null,
|
||||
source = source,
|
||||
)
|
||||
}
|
||||
}
|
||||
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
|
||||
val fullUrl = chapter.url.toAbsoluteUrl(domain)
|
||||
val doc = webClient.httpGet(fullUrl).parseHtml()
|
||||
return doc.select("div#chapter_imgs img").map { img ->
|
||||
val url = img.attr("src")
|
||||
MangaPage(
|
||||
id = generateUid(url),
|
||||
url = url,
|
||||
preview = null,
|
||||
source = source,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
private fun parseDate(dateText: String?): Long {
|
||||
if (dateText == null) return 0
|
||||
private fun parseDate(dateText: String?): Long {
|
||||
if (dateText == null) return 0
|
||||
|
||||
val number = dateText.filter { it.isDigit() }.toIntOrNull() ?: return 0
|
||||
val now = System.currentTimeMillis()
|
||||
val number = dateText.filter { it.isDigit() }.toIntOrNull() ?: return 0
|
||||
val now = System.currentTimeMillis()
|
||||
|
||||
return when {
|
||||
dateText.contains("minutos") -> {
|
||||
now - (number * 60 * 1000L)
|
||||
}
|
||||
dateText.contains("horas") -> {
|
||||
now - (number * 60 * 60 * 1000L)
|
||||
}
|
||||
dateText.contains("días") -> {
|
||||
now - (number * 24 * 60 * 60 * 1000L)
|
||||
}
|
||||
dateText.contains("día") -> {
|
||||
now - (number * 24 * 60 * 60 * 1000L)
|
||||
}
|
||||
dateText.contains("semanas") -> {
|
||||
now - (number * 7 * 24 * 60 * 60 * 1000L)
|
||||
}
|
||||
dateText.contains("meses") -> {
|
||||
now - (number * 30 * 24 * 60 * 60 * 1000L)
|
||||
}
|
||||
dateText.contains("años") -> {
|
||||
now - (number * 365 * 24 * 60 * 60 * 1000L)
|
||||
}
|
||||
else -> 0L
|
||||
}
|
||||
}
|
||||
return when {
|
||||
dateText.contains("minutos") -> {
|
||||
now - (number * 60 * 1000L)
|
||||
}
|
||||
|
||||
dateText.contains("horas") -> {
|
||||
now - (number * 60 * 60 * 1000L)
|
||||
}
|
||||
|
||||
dateText.contains("días") -> {
|
||||
now - (number * 24 * 60 * 60 * 1000L)
|
||||
}
|
||||
|
||||
dateText.contains("día") -> {
|
||||
now - (number * 24 * 60 * 60 * 1000L)
|
||||
}
|
||||
|
||||
dateText.contains("semanas") -> {
|
||||
now - (number * 7 * 24 * 60 * 60 * 1000L)
|
||||
}
|
||||
|
||||
dateText.contains("meses") -> {
|
||||
now - (number * 30 * 24 * 60 * 60 * 1000L)
|
||||
}
|
||||
|
||||
dateText.contains("años") -> {
|
||||
now - (number * 365 * 24 * 60 * 60 * 1000L)
|
||||
}
|
||||
|
||||
else -> 0L
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,143 +20,143 @@ import java.util.zip.ZipInputStream
|
||||
@Broken // Not dead but changed template
|
||||
@MangaSourceParser("RANDOMSCANS", "LuratoonScan", "pt")
|
||||
internal class LuratoonScansParser(context: MangaLoaderContext) :
|
||||
SinglePageMangaParser(context, MangaParserSource.RANDOMSCANS),
|
||||
Interceptor {
|
||||
SinglePageMangaParser(context, MangaParserSource.RANDOMSCANS),
|
||||
Interceptor {
|
||||
|
||||
override val configKeyDomain = ConfigKey.Domain("luratoons.net")
|
||||
override val configKeyDomain = ConfigKey.Domain("luratoons.net")
|
||||
|
||||
override fun getRequestHeaders(): Headers = Headers.Builder().add("User-Agent", config[userAgentKey]).build()
|
||||
override fun getRequestHeaders(): Headers = Headers.Builder().add("User-Agent", config[userAgentKey]).build()
|
||||
|
||||
override val availableSortOrders = setOf(SortOrder.ALPHABETICAL)
|
||||
override val availableSortOrders = setOf(SortOrder.ALPHABETICAL)
|
||||
|
||||
override val filterCapabilities: MangaListFilterCapabilities
|
||||
get() = MangaListFilterCapabilities()
|
||||
override val filterCapabilities: MangaListFilterCapabilities
|
||||
get() = MangaListFilterCapabilities()
|
||||
|
||||
override suspend fun getFilterOptions() = MangaListFilterOptions()
|
||||
override suspend fun getFilterOptions() = MangaListFilterOptions()
|
||||
|
||||
override suspend fun getList(order: SortOrder, filter: MangaListFilter): List<Manga> {
|
||||
require(filter.query.isNullOrEmpty()) { ErrorMessages.SEARCH_NOT_SUPPORTED }
|
||||
val url = urlBuilder()
|
||||
val tag = filter.tags.oneOrThrowIfMany()
|
||||
if (tag == null) {
|
||||
url.addPathSegment("todas-as-obras")
|
||||
} else {
|
||||
url.addPathSegment("pesquisar").addQueryParameter("category", tag.key)
|
||||
}
|
||||
val doc = webClient.httpGet(url.build()).parseHtml()
|
||||
return doc.selectFirstOrThrow(".todas__as__obras").select(".comics__all__box").map { div ->
|
||||
val a = div.selectFirstOrThrow("a")
|
||||
val href = a.attrAsRelativeUrl("href")
|
||||
Manga(
|
||||
id = generateUid(href),
|
||||
url = href,
|
||||
publicUrl = href.toAbsoluteUrl(div.host ?: domain),
|
||||
coverUrl = div.selectFirst("img")?.src().orEmpty(),
|
||||
title = div.text(),
|
||||
altTitles = emptySet(),
|
||||
rating = RATING_UNKNOWN,
|
||||
tags = emptySet(),
|
||||
authors = emptySet(),
|
||||
state = null,
|
||||
source = source,
|
||||
contentRating = null,
|
||||
)
|
||||
}
|
||||
}
|
||||
override suspend fun getList(order: SortOrder, filter: MangaListFilter): List<Manga> {
|
||||
require(filter.query.isNullOrEmpty()) { ErrorMessages.SEARCH_NOT_SUPPORTED }
|
||||
val url = urlBuilder()
|
||||
val tag = filter.tags.oneOrThrowIfMany()
|
||||
if (tag == null) {
|
||||
url.addPathSegment("todas-as-obras")
|
||||
} else {
|
||||
url.addPathSegment("pesquisar").addQueryParameter("category", tag.key)
|
||||
}
|
||||
val doc = webClient.httpGet(url.build()).parseHtml()
|
||||
return doc.selectFirstOrThrow(".todas__as__obras").select(".comics__all__box").map { div ->
|
||||
val a = div.selectFirstOrThrow("a")
|
||||
val href = a.attrAsRelativeUrl("href")
|
||||
Manga(
|
||||
id = generateUid(href),
|
||||
url = href,
|
||||
publicUrl = href.toAbsoluteUrl(div.host ?: domain),
|
||||
coverUrl = div.selectFirst("img")?.src().orEmpty(),
|
||||
title = div.text(),
|
||||
altTitles = emptySet(),
|
||||
rating = RATING_UNKNOWN,
|
||||
tags = emptySet(),
|
||||
authors = emptySet(),
|
||||
state = null,
|
||||
source = source,
|
||||
contentRating = null,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
override suspend fun getDetails(manga: Manga): Manga {
|
||||
val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml().body()
|
||||
val summaryContainer = doc.selectFirstOrThrow(".sumario__container")
|
||||
// 1 de Maio de 2024 às 20:15
|
||||
val dateFormat = SimpleDateFormat("dd 'de' MMM 'de' YYYY 'às' HH:mm", sourceLocale)
|
||||
val author = summaryContainer.getElementsContainingOwnText("Autor(es)").firstOrNull()
|
||||
?.nextElementSibling()?.textOrNull()
|
||||
return manga.copy(
|
||||
title = doc.selectFirst("h1.desc__titulo__comic")?.textOrNull() ?: manga.title,
|
||||
altTitles = setOfNotNull(
|
||||
summaryContainer.getElementsContainingOwnText("Alternativo").firstOrNull()
|
||||
?.nextElementSibling()?.textOrNull(),
|
||||
),
|
||||
tags = summaryContainer.getElementsByAttributeValueContaining("href", "?category=").mapToSet {
|
||||
MangaTag(
|
||||
title = it.text().toTitleCase(sourceLocale),
|
||||
key = it.attr("href").substringAfterLast('='),
|
||||
source = source,
|
||||
)
|
||||
},
|
||||
state = when (summaryContainer.getElementsContainingOwnText("Status").firstOrNull()
|
||||
?.nextElementSibling()?.text()?.lowercase()) {
|
||||
"em lançamento" -> MangaState.ONGOING
|
||||
"hiato" -> MangaState.PAUSED
|
||||
"finalizado" -> MangaState.FINISHED
|
||||
else -> null
|
||||
},
|
||||
authors = setOfNotNull(author),
|
||||
largeCoverUrl = doc.selectFirst("img.sumario__img")?.attrAsAbsoluteUrlOrNull("src"),
|
||||
description = summaryContainer.selectFirst(".sumario__sinopse__texto")?.html(),
|
||||
chapters = doc.selectFirstOrThrow("ul.capitulos__lista")
|
||||
.select("li")
|
||||
.mapChapters(reversed = true) { _, li ->
|
||||
val href = li.parent()?.attrAsRelativeUrlOrNull("href") ?: return@mapChapters null
|
||||
val span = li.selectFirstOrThrow(".numero__capitulo")
|
||||
MangaChapter(
|
||||
id = generateUid(href),
|
||||
title = span.text(),
|
||||
number = 0.0f,
|
||||
volume = 0,
|
||||
url = href,
|
||||
scanlator = null,
|
||||
uploadDate = dateFormat.parseSafe(span.nextElementSibling()?.text()),
|
||||
branch = null,
|
||||
source = source,
|
||||
)
|
||||
},
|
||||
)
|
||||
}
|
||||
override suspend fun getDetails(manga: Manga): Manga {
|
||||
val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml().body()
|
||||
val summaryContainer = doc.selectFirstOrThrow(".sumario__container")
|
||||
// 1 de Maio de 2024 às 20:15
|
||||
val dateFormat = SimpleDateFormat("dd 'de' MMM 'de' YYYY 'às' HH:mm", sourceLocale)
|
||||
val author = summaryContainer.getElementsContainingOwnText("Autor(es)").firstOrNull()
|
||||
?.nextElementSibling()?.textOrNull()
|
||||
return manga.copy(
|
||||
title = doc.selectFirst("h1.desc__titulo__comic")?.textOrNull() ?: manga.title,
|
||||
altTitles = setOfNotNull(
|
||||
summaryContainer.getElementsContainingOwnText("Alternativo").firstOrNull()
|
||||
?.nextElementSibling()?.textOrNull(),
|
||||
),
|
||||
tags = summaryContainer.getElementsByAttributeValueContaining("href", "?category=").mapToSet {
|
||||
MangaTag(
|
||||
title = it.text().toTitleCase(sourceLocale),
|
||||
key = it.attr("href").substringAfterLast('='),
|
||||
source = source,
|
||||
)
|
||||
},
|
||||
state = when (summaryContainer.getElementsContainingOwnText("Status").firstOrNull()
|
||||
?.nextElementSibling()?.text()?.lowercase()) {
|
||||
"em lançamento" -> MangaState.ONGOING
|
||||
"hiato" -> MangaState.PAUSED
|
||||
"finalizado" -> MangaState.FINISHED
|
||||
else -> null
|
||||
},
|
||||
authors = setOfNotNull(author),
|
||||
largeCoverUrl = doc.selectFirst("img.sumario__img")?.attrAsAbsoluteUrlOrNull("src"),
|
||||
description = summaryContainer.selectFirst(".sumario__sinopse__texto")?.html(),
|
||||
chapters = doc.selectFirstOrThrow("ul.capitulos__lista")
|
||||
.select("li")
|
||||
.mapChapters(reversed = true) { _, li ->
|
||||
val href = li.parent()?.attrAsRelativeUrlOrNull("href") ?: return@mapChapters null
|
||||
val span = li.selectFirstOrThrow(".numero__capitulo")
|
||||
MangaChapter(
|
||||
id = generateUid(href),
|
||||
title = span.text(),
|
||||
number = 0.0f,
|
||||
volume = 0,
|
||||
url = href,
|
||||
scanlator = null,
|
||||
uploadDate = dateFormat.parseSafe(span.nextElementSibling()?.text()),
|
||||
branch = null,
|
||||
source = source,
|
||||
)
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
|
||||
val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml()
|
||||
val regex = Regex("const\\s+urls\\s*=\\s*(\\[.*])")
|
||||
val urls = doc.select("script").firstNotNullOf {
|
||||
regex.find(it.data())?.groupValues?.getOrNull(1)
|
||||
}
|
||||
val ja = JSONArray(urls)
|
||||
return (0 until ja.length()).map { i ->
|
||||
val url = ja.getString(i)
|
||||
MangaPage(
|
||||
id = generateUid(url),
|
||||
url = url,
|
||||
preview = null,
|
||||
source = source,
|
||||
)
|
||||
}
|
||||
}
|
||||
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
|
||||
val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml()
|
||||
val regex = Regex("const\\s+urls\\s*=\\s*(\\[.*])")
|
||||
val urls = doc.select("script").firstNotNullOf {
|
||||
regex.find(it.data())?.groupValues?.getOrNull(1)
|
||||
}
|
||||
val ja = JSONArray(urls)
|
||||
return (0 until ja.length()).map { i ->
|
||||
val url = ja.getString(i)
|
||||
MangaPage(
|
||||
id = generateUid(url),
|
||||
url = url,
|
||||
preview = null,
|
||||
source = source,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
override fun intercept(chain: Interceptor.Chain): Response {
|
||||
val response = chain.proceed(chain.request())
|
||||
if (response.mimeType == "application/octet-stream") {
|
||||
val (bytes, name) = response.use { resp ->
|
||||
ZipInputStream(resp.requireBody().byteStream()).use {
|
||||
val entry = it.nextEntry
|
||||
it.readBytes() to entry?.name
|
||||
}
|
||||
}
|
||||
val type = if (name?.endsWith(".avif", ignoreCase = true) == true) {
|
||||
"image/avif"
|
||||
} else {
|
||||
"image/*"
|
||||
}.toMediaTypeOrNull()
|
||||
return response.newBuilder()
|
||||
.setHeader("Content-Type", type?.toString())
|
||||
.body(bytes.toResponseBody(type))
|
||||
.build()
|
||||
} else {
|
||||
return response
|
||||
}
|
||||
}
|
||||
override fun intercept(chain: Interceptor.Chain): Response {
|
||||
val response = chain.proceed(chain.request())
|
||||
if (response.mimeType == "application/octet-stream") {
|
||||
val (bytes, name) = response.use { resp ->
|
||||
ZipInputStream(resp.body.byteStream()).use {
|
||||
val entry = it.nextEntry
|
||||
it.readBytes() to entry?.name
|
||||
}
|
||||
}
|
||||
val type = if (name?.endsWith(".avif", ignoreCase = true) == true) {
|
||||
"image/avif"
|
||||
} else {
|
||||
"image/*"
|
||||
}.toMediaTypeOrNull()
|
||||
return response.newBuilder()
|
||||
.setHeader("Content-Type", type?.toString())
|
||||
.body(bytes.toResponseBody(type))
|
||||
.build()
|
||||
} else {
|
||||
return response
|
||||
}
|
||||
}
|
||||
|
||||
override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) {
|
||||
super.onCreateConfig(keys)
|
||||
keys.add(userAgentKey)
|
||||
}
|
||||
override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) {
|
||||
super.onCreateConfig(keys)
|
||||
keys.add(userAgentKey)
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -4,67 +4,52 @@ package org.koitharu.kotatsu.parsers.util
|
||||
|
||||
import kotlinx.coroutines.suspendCancellableCoroutine
|
||||
import okhttp3.*
|
||||
import okhttp3.internal.toLongOrDefault
|
||||
import kotlin.contracts.InvocationKind
|
||||
import kotlin.contracts.contract
|
||||
|
||||
public suspend fun Call.await(): Response = suspendCancellableCoroutine { continuation ->
|
||||
val callback = ContinuationCallCallback(this, continuation)
|
||||
enqueue(callback)
|
||||
continuation.invokeOnCancellation(callback)
|
||||
val callback = ContinuationCallCallback(this, continuation)
|
||||
enqueue(callback)
|
||||
continuation.invokeOnCancellation(callback)
|
||||
}
|
||||
|
||||
public val Response.mimeType: String?
|
||||
get() = header("content-type")?.substringBefore(';')?.trim()?.nullIfEmpty()?.lowercase()
|
||||
get() = header("content-type")?.substringBefore(';')?.trim()?.nullIfEmpty()?.lowercase()
|
||||
|
||||
public val HttpUrl.isHttpOrHttps: Boolean
|
||||
get() = scheme.equals("https", ignoreCase = true) || scheme.equals("http", ignoreCase = true)
|
||||
get() = scheme.equals("https", ignoreCase = true) || scheme.equals("http", ignoreCase = true)
|
||||
|
||||
public fun Headers.Builder.mergeWith(other: Headers, replaceExisting: Boolean): Headers.Builder {
|
||||
for ((name, value) in other) {
|
||||
if (replaceExisting || this[name] == null) {
|
||||
this[name] = value
|
||||
}
|
||||
}
|
||||
return this
|
||||
for ((name, value) in other) {
|
||||
if (replaceExisting || this[name] == null) {
|
||||
this[name] = value
|
||||
}
|
||||
}
|
||||
return this
|
||||
}
|
||||
|
||||
public fun Response.copy(): Response = newBuilder()
|
||||
.body(peekBody(Long.MAX_VALUE))
|
||||
.build()
|
||||
.body(peekBody(Long.MAX_VALUE))
|
||||
.build()
|
||||
|
||||
public fun Response.Builder.setHeader(name: String, value: String?): Response.Builder = if (value == null) {
|
||||
removeHeader(name)
|
||||
removeHeader(name)
|
||||
} else {
|
||||
header(name, value)
|
||||
header(name, value)
|
||||
}
|
||||
|
||||
public inline fun Response.map(mapper: (ResponseBody) -> ResponseBody): Response {
|
||||
contract {
|
||||
callsInPlace(mapper, InvocationKind.AT_MOST_ONCE)
|
||||
}
|
||||
return body?.use { responseBody ->
|
||||
newBuilder()
|
||||
.body(mapper(responseBody))
|
||||
.build()
|
||||
} ?: this
|
||||
contract {
|
||||
callsInPlace(mapper, InvocationKind.AT_MOST_ONCE)
|
||||
}
|
||||
return body.use { responseBody ->
|
||||
newBuilder()
|
||||
.body(mapper(responseBody))
|
||||
.build()
|
||||
}
|
||||
}
|
||||
|
||||
public fun Cookie.newBuilder(): Cookie.Builder = Cookie.Builder().also { c ->
|
||||
c.name(name)
|
||||
c.value(value)
|
||||
if (persistent) {
|
||||
c.expiresAt(expiresAt)
|
||||
}
|
||||
if (hostOnly) {
|
||||
c.hostOnlyDomain(domain)
|
||||
} else {
|
||||
c.domain(domain)
|
||||
}
|
||||
c.path(path)
|
||||
if (secure) {
|
||||
c.secure()
|
||||
}
|
||||
if (httpOnly) {
|
||||
c.httpOnly()
|
||||
}
|
||||
}
|
||||
public fun Response.headersContentLength(
|
||||
defaultValue: Long = -1,
|
||||
): Long = headers["Content-Length"]?.toLongOrDefault(defaultValue) ?: defaultValue
|
||||
|
||||
@@ -8,7 +8,6 @@ import org.json.JSONArray
|
||||
import org.json.JSONObject
|
||||
import org.jsoup.Jsoup
|
||||
import org.jsoup.nodes.Document
|
||||
import org.koitharu.kotatsu.parsers.ErrorMessages
|
||||
import org.koitharu.kotatsu.parsers.InternalParsersApi
|
||||
import java.text.DateFormat
|
||||
|
||||
@@ -22,9 +21,9 @@ internal const val SCHEME_HTTPS = "https"
|
||||
*/
|
||||
// TODO suspend
|
||||
public fun Response.parseHtml(): Document = use { response ->
|
||||
val body = response.requireBody()
|
||||
val charset = body.contentType()?.charset()?.name()
|
||||
Jsoup.parse(body.byteStream(), charset, response.request.url.toString())
|
||||
val body = response.body
|
||||
val charset = body.contentType()?.charset()?.name()
|
||||
Jsoup.parse(body.byteStream(), charset, response.request.url.toString())
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -33,7 +32,7 @@ public fun Response.parseHtml(): Document = use { response ->
|
||||
* @see [parseHtml]
|
||||
*/
|
||||
public fun Response.parseJson(): JSONObject = use { response ->
|
||||
JSONObject(response.requireBody().string())
|
||||
JSONObject(response.body.string())
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -42,15 +41,15 @@ public fun Response.parseJson(): JSONObject = use { response ->
|
||||
* @see [parseHtml]
|
||||
*/
|
||||
public fun Response.parseJsonArray(): JSONArray = use { response ->
|
||||
JSONArray(response.requireBody().string())
|
||||
JSONArray(response.body.string())
|
||||
}
|
||||
|
||||
public fun Response.parseRaw(): String = use { response ->
|
||||
response.requireBody().string()
|
||||
response.body.string()
|
||||
}
|
||||
|
||||
public fun Response.parseBytes(): ByteArray = use { response ->
|
||||
response.requireBody().bytes()
|
||||
response.body.bytes()
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -58,10 +57,10 @@ public fun Response.parseBytes(): ByteArray = use { response ->
|
||||
* @return an url relative to the [domain] or absolute, if domain is mismatching
|
||||
*/
|
||||
public fun String.toRelativeUrl(domain: String): String {
|
||||
if (isEmpty() || startsWith("/")) {
|
||||
return this
|
||||
}
|
||||
return replace(Regex("^[^/]{2,6}://${Regex.escape(domain)}+/", RegexOption.IGNORE_CASE), "/")
|
||||
if (isEmpty() || startsWith("/")) {
|
||||
return this
|
||||
}
|
||||
return replace(Regex("^[^/]{2,6}://${Regex.escape(domain)}+/", RegexOption.IGNORE_CASE), "/")
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -69,36 +68,35 @@ public fun String.toRelativeUrl(domain: String): String {
|
||||
* @return an absolute url with [domain] if this is relative
|
||||
*/
|
||||
public fun String.toAbsoluteUrl(domain: String): String = when {
|
||||
startsWith("//") -> "$SCHEME_HTTPS:$this"
|
||||
startsWith('/') -> "$SCHEME_HTTPS://$domain$this"
|
||||
REGEX_SCHEME_PREFIX.containsMatchIn(this) -> this
|
||||
else -> "$SCHEME_HTTPS://$domain/$this"
|
||||
startsWith("//") -> "$SCHEME_HTTPS:$this"
|
||||
startsWith('/') -> "$SCHEME_HTTPS://$domain$this"
|
||||
REGEX_SCHEME_PREFIX.containsMatchIn(this) -> this
|
||||
else -> "$SCHEME_HTTPS://$domain/$this"
|
||||
}
|
||||
|
||||
public fun concatUrl(host: String, path: String): String {
|
||||
val hostWithSlash = host.endsWith('/')
|
||||
val pathWithSlash = path.startsWith('/')
|
||||
val hostWithScheme = if (host.startsWith("//")) "https:$host" else host
|
||||
return when {
|
||||
hostWithSlash && pathWithSlash -> hostWithScheme + path.drop(1)
|
||||
!hostWithSlash && !pathWithSlash -> "$hostWithScheme/$path"
|
||||
else -> hostWithScheme + path
|
||||
}
|
||||
val hostWithSlash = host.endsWith('/')
|
||||
val pathWithSlash = path.startsWith('/')
|
||||
val hostWithScheme = if (host.startsWith("//")) "https:$host" else host
|
||||
return when {
|
||||
hostWithSlash && pathWithSlash -> hostWithScheme + path.drop(1)
|
||||
!hostWithSlash && !pathWithSlash -> "$hostWithScheme/$path"
|
||||
else -> hostWithScheme + path
|
||||
}
|
||||
}
|
||||
|
||||
@InternalParsersApi
|
||||
public fun DateFormat.parseSafe(str: String?): Long = if (str.isNullOrEmpty()) {
|
||||
0L
|
||||
0L
|
||||
} else {
|
||||
runCatching {
|
||||
parse(str)?.time ?: 0L
|
||||
}.onFailure {
|
||||
if (javaClass.desiredAssertionStatus()) {
|
||||
throw AssertionError("Cannot parse date $str", it)
|
||||
}
|
||||
}.getOrDefault(0L)
|
||||
runCatching {
|
||||
parse(str)?.time ?: 0L
|
||||
}.onFailure {
|
||||
if (javaClass.desiredAssertionStatus()) {
|
||||
throw AssertionError("Cannot parse date $str", it)
|
||||
}
|
||||
}.getOrDefault(0L)
|
||||
}
|
||||
|
||||
public fun Response.requireBody(): ResponseBody = requireNotNull(body) {
|
||||
ErrorMessages.RESPONSE_NULL_BODY
|
||||
}
|
||||
@Deprecated("Useless since OkHttp 5.0", replaceWith = ReplaceWith("body"))
|
||||
public fun Response.requireBody(): ResponseBody = body
|
||||
|
||||
@@ -3,51 +3,55 @@ package org.koitharu.kotatsu.parsers.util
|
||||
import kotlinx.coroutines.*
|
||||
import org.koitharu.kotatsu.parsers.MangaParser
|
||||
import org.koitharu.kotatsu.parsers.model.Manga
|
||||
import org.koitharu.kotatsu.parsers.model.MangaListFilter
|
||||
import org.koitharu.kotatsu.parsers.model.SortOrder
|
||||
import org.koitharu.kotatsu.parsers.model.search.MangaSearchQuery
|
||||
import org.koitharu.kotatsu.parsers.model.search.QueryCriteria
|
||||
import org.koitharu.kotatsu.parsers.model.search.SearchableField
|
||||
|
||||
public class RelatedMangaFinder(
|
||||
private val parsers: Collection<MangaParser>,
|
||||
private val parsers: Collection<MangaParser>,
|
||||
) {
|
||||
|
||||
public suspend operator fun invoke(seed: Manga): List<Manga> = withContext(Dispatchers.Default) {
|
||||
coroutineScope {
|
||||
parsers.singleOrNull()?.let { parser ->
|
||||
findRelatedImpl(this, parser, seed)
|
||||
} ?: parsers.map { parser ->
|
||||
async {
|
||||
findRelatedImpl(this, parser, seed)
|
||||
}
|
||||
}.awaitAll().flatten()
|
||||
}
|
||||
}
|
||||
public suspend operator fun invoke(seed: Manga): List<Manga> = withContext(Dispatchers.Default) {
|
||||
coroutineScope {
|
||||
parsers.singleOrNull()?.let { parser ->
|
||||
findRelatedImpl(this, parser, seed)
|
||||
} ?: parsers.map { parser ->
|
||||
async {
|
||||
findRelatedImpl(this, parser, seed)
|
||||
}
|
||||
}.awaitAll().flatten()
|
||||
}
|
||||
}
|
||||
|
||||
private suspend fun findRelatedImpl(scope: CoroutineScope, parser: MangaParser, seed: Manga): List<Manga> {
|
||||
val words = HashSet<String>()
|
||||
words += seed.title.splitByWhitespace()
|
||||
seed.altTitles.forEach {
|
||||
words += it.splitByWhitespace()
|
||||
}
|
||||
if (words.isEmpty()) {
|
||||
return emptyList()
|
||||
}
|
||||
val results = words.map { keyword ->
|
||||
scope.async {
|
||||
val result = parser.getList(
|
||||
MangaSearchQuery.Builder()
|
||||
.order(SortOrder.RELEVANCE)
|
||||
.criterion(QueryCriteria.Match(SearchableField.TITLE_NAME, keyword))
|
||||
.build(),
|
||||
)
|
||||
result.filter { it.id != seed.id && it.containKeyword(keyword) }
|
||||
}
|
||||
}.awaitAll()
|
||||
return results.minBy { if (it.isEmpty()) Int.MAX_VALUE else it.size }
|
||||
}
|
||||
private suspend fun findRelatedImpl(scope: CoroutineScope, parser: MangaParser, seed: Manga): List<Manga> {
|
||||
val words = HashSet<String>()
|
||||
words += seed.title.splitByWhitespace()
|
||||
seed.altTitles.forEach {
|
||||
words += it.splitByWhitespace()
|
||||
}
|
||||
if (words.isEmpty()) {
|
||||
return emptyList()
|
||||
}
|
||||
val results = words.map { keyword ->
|
||||
scope.async {
|
||||
val result = parser.getList(
|
||||
0,
|
||||
if (SortOrder.RELEVANCE in parser.availableSortOrders) {
|
||||
SortOrder.RELEVANCE
|
||||
} else {
|
||||
parser.availableSortOrders.first()
|
||||
},
|
||||
MangaListFilter(
|
||||
query = keyword,
|
||||
),
|
||||
)
|
||||
result.filter { it.id != seed.id && it.containKeyword(keyword) }
|
||||
}
|
||||
}.awaitAll()
|
||||
return results.minBy { if (it.isEmpty()) Int.MAX_VALUE else it.size }
|
||||
}
|
||||
|
||||
private fun Manga.containKeyword(keyword: String): Boolean {
|
||||
return title.contains(keyword, ignoreCase = true) || altTitle?.contains(keyword, ignoreCase = true) == true
|
||||
}
|
||||
private fun Manga.containKeyword(keyword: String): Boolean {
|
||||
return title.contains(keyword, ignoreCase = true)
|
||||
|| altTitles.any { it.contains(keyword, ignoreCase = true) }
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user