From 1e1756aa1799109117b5e16bae1583243aae27c2 Mon Sep 17 00:00:00 2001 From: Koitharu Date: Wed, 8 Feb 2023 10:13:22 +0200 Subject: [PATCH 1/2] Refactor web client --- .editorconfig | 4 +- .github/workflows/test-parsers.yml | 4 +- .../kotatsu/parsers/MangaLoaderContext.kt | 167 +-- .../koitharu/kotatsu/parsers/MangaParser.kt | 313 ++--- .../kotatsu/parsers/PagedMangaParser.kt | 81 +- .../kotatsu/parsers/model/MangaPage.kt | 83 +- .../parsers/network/OkHttpWebClient.kt | 107 ++ .../kotatsu/parsers/network/WebClient.kt | 68 ++ .../kotatsu/parsers/site/AnibelParser.kt | 358 +++--- .../kotatsu/parsers/site/BatoToParser.kt | 543 ++++----- .../kotatsu/parsers/site/BlogTruyenParser.kt | 469 ++++---- .../kotatsu/parsers/site/CloneMangaParser.kt | 179 +-- .../kotatsu/parsers/site/ComickFunParser.kt | 362 +++--- .../kotatsu/parsers/site/DesuMeParser.kt | 278 ++--- .../kotatsu/parsers/site/ExHentaiParser.kt | 534 ++++----- .../kotatsu/parsers/site/MangaDexParser.kt | 470 ++++---- .../kotatsu/parsers/site/MangaInUaParser.kt | 261 ++--- .../kotatsu/parsers/site/MangaOwlParser.kt | 187 --- .../kotatsu/parsers/site/MangaTownParser.kt | 384 +++--- .../kotatsu/parsers/site/Manhwa18Parser.kt | 318 ++--- .../kotatsu/parsers/site/NHentaiParser.kt | 322 +++--- .../kotatsu/parsers/site/NetTruyenParser.kt | 412 +++---- .../parsers/site/NicovideoSeigaParser.kt | 320 ++--- .../kotatsu/parsers/site/NineMangaParser.kt | 437 ++++--- .../kotatsu/parsers/site/NudeMoonParser.kt | 428 +++---- .../kotatsu/parsers/site/RemangaParser.kt | 480 ++++---- .../kotatsu/parsers/site/UnionMangasParser.kt | 306 ++--- .../parsers/site/grouple/GroupleParser.kt | 558 +++++---- .../parsers/site/grouple/MintMangaParser.kt | 20 +- .../parsers/site/grouple/ReadmangaParser.kt | 20 +- .../parsers/site/grouple/SelfMangaParser.kt | 14 +- .../parsers/site/madara/IsekaiScanEu.kt | 62 +- .../parsers/site/madara/Madara5Parser.kt | 342 +++--- .../parsers/site/madara/Madara6Parser.kt | 100 +- .../parsers/site/madara/MadaraParser.kt | 948 +++++++-------- .../parsers/site/madara/MangalinkParser.kt | 69 +- .../site/madara/MangasOriginesParser.kt | 71 +- .../parsers/site/madara/PrismaScansParser.kt | 75 +- .../parsers/site/madara/ReaperScansParser.kt | 140 +-- .../parsers/site/madara/TatakaeScansParser.kt | 82 +- .../site/mangareader/MangaReaderParser.kt | 1030 +++++++++-------- .../parsers/site/multichan/ChanParser.kt | 316 ++--- .../parsers/site/multichan/HenChanParser.kt | 96 +- .../parsers/site/multichan/MangaChanParser.kt | 6 +- .../parsers/site/multichan/YaoiChanParser.kt | 52 +- .../parsers/site/rulib/MangaLibParser.kt | 530 ++++----- .../kotatsu/parsers/util/FaviconParser.kt | 152 ++- .../kotatsu/parsers/util/MangaParserEnv.kt | 78 ++ .../koitharu/kotatsu/parsers/util/OkHttp.kt | 20 +- .../kotatsu/parsers/AuthCheckExtension.kt | 44 +- .../kotatsu/parsers/CloudFlareInterceptor.kt | 35 +- .../parsers/CommonHeadersInterceptor.kt | 47 + .../kotatsu/parsers/MangaLoaderContextMock.kt | 96 +- .../kotatsu/parsers/MangaParserTest.kt | 361 +++--- .../koitharu/kotatsu/parsers/MangaSources.kt | 2 +- .../kotatsu/parsers/UserAgentInterceptor.kt | 20 - 56 files changed, 6600 insertions(+), 6661 deletions(-) create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/network/OkHttpWebClient.kt create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/network/WebClient.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaOwlParser.kt create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/util/MangaParserEnv.kt create mode 100644 src/test/kotlin/org/koitharu/kotatsu/parsers/CommonHeadersInterceptor.kt delete mode 100644 src/test/kotlin/org/koitharu/kotatsu/parsers/UserAgentInterceptor.kt diff --git a/.editorconfig b/.editorconfig index 7f2b16d0c..a352878ed 100644 --- a/.editorconfig +++ b/.editorconfig @@ -6,8 +6,8 @@ end_of_line = lf indent_style = tab max_line_length = 120 tab_width = 4 -insert_final_newline = false -disabled_rules=no-wildcard-imports,no-unused-imports +insert_final_newline = true +disabled_rules = no-wildcard-imports, no-unused-imports [{*.kt,*.kts}] ij_kotlin_allow_trailing_comma = true diff --git a/.github/workflows/test-parsers.yml b/.github/workflows/test-parsers.yml index 4479e7587..817a39411 100644 --- a/.github/workflows/test-parsers.yml +++ b/.github/workflows/test-parsers.yml @@ -4,7 +4,7 @@ on: workflow_dispatch: pull_request: paths: - - 'src/main/kotlin/org/koitharu/kotatsu/parsers/site/*' + - 'src/main/kotlin/org/koitharu/kotatsu/parsers/site/**' permissions: contents: read @@ -24,4 +24,4 @@ jobs: - uses: actions/upload-artifact@v3 with: name: Report - path: build/test-results-html/TEST-org.koitharu.kotatsu.parsers.MangaParserTest.htm \ No newline at end of file + path: build/test-results-html/TEST-org.koitharu.kotatsu.parsers.MangaParserTest.htm diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaLoaderContext.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaLoaderContext.kt index 047537838..1268be961 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaLoaderContext.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaLoaderContext.kt @@ -1,166 +1,29 @@ package org.koitharu.kotatsu.parsers -import okhttp3.* -import okhttp3.HttpUrl.Companion.toHttpUrl -import okhttp3.MediaType.Companion.toMediaType -import okhttp3.RequestBody.Companion.toRequestBody -import org.json.JSONObject -import org.jsoup.HttpStatusException +import okhttp3.CookieJar +import okhttp3.OkHttpClient import org.koitharu.kotatsu.parsers.config.MangaSourceConfig -import org.koitharu.kotatsu.parsers.exception.GraphQLException -import org.koitharu.kotatsu.parsers.exception.NotFoundException import org.koitharu.kotatsu.parsers.model.MangaSource -import org.koitharu.kotatsu.parsers.util.await -import org.koitharu.kotatsu.parsers.util.parseJson import java.util.* abstract class MangaLoaderContext { - protected abstract val httpClient: OkHttpClient + abstract val httpClient: OkHttpClient - abstract val cookieJar: CookieJar + abstract val cookieJar: CookieJar - /** - * Do a GET http request to specific url - * @param url - * @param headers an additional headers for request, may be null - */ - suspend fun httpGet(url: HttpUrl, headers: Headers? = null): Response { - val request = Request.Builder() - .get() - .url(url) - if (headers != null) { - request.headers(headers) - } - return httpClient.newCall(request.build()).await().ensureSuccess() - } + open fun encodeBase64(data: ByteArray): String = Base64.getEncoder().encodeToString(data) - suspend fun httpGet(url: String, headers: Headers? = null): Response { - return httpGet(url.toHttpUrl(), headers) - } + open fun decodeBase64(data: String): ByteArray = Base64.getDecoder().decode(data) + open fun getPreferredLocales(): List = listOf(Locale.getDefault()) - /** - * Do a HEAD http request to specific url - * @param url - * @param headers an additional headers for request, may be null - */ - suspend fun httpHead(url: String, headers: Headers? = null): Response { - val request = Request.Builder() - .head() - .url(url) - if (headers != null) { - request.headers(headers) - } - return httpClient.newCall(request.build()).await().ensureSuccess() - } + /** + * Execute JavaScript code and return result + * @param script JavaScript source code + * @return execution result as string, may be null + */ + abstract suspend fun evaluateJs(script: String): String? - /** - * Do a POST http request to specific url with `multipart/form-data` payload - * @param url - * @param form payload as key=>value map - * @param headers an additional headers for request, may be null - */ - suspend fun httpPost( - url: String, - form: Map, - headers: Headers? = null, - ): Response { - val body = FormBody.Builder() - form.forEach { (k, v) -> - body.addEncoded(k, v) - } - val request = Request.Builder() - .post(body.build()) - .url(url) - if (headers != null) { - request.headers(headers) - } - return httpClient.newCall(request.build()).await().ensureSuccess() - } - - /** - * Do a POST http request to specific url with `multipart/form-data` payload - * @param url - * @param payload payload as `key=value` string with `&` separator - * @param headers an additional headers for request, may be null - */ - suspend fun httpPost( - url: String, - payload: String, - headers: Headers?, - ): Response { - val body = FormBody.Builder() - payload.split('&').forEach { - val pos = it.indexOf('=') - if (pos != -1) { - val k = it.substring(0, pos) - val v = it.substring(pos + 1) - body.addEncoded(k, v) - } - } - val request = Request.Builder() - .post(body.build()) - .url(url) - if (headers != null) { - request.headers(headers) - } - return httpClient.newCall(request.build()).await().ensureSuccess() - } - - /** - * Do a GraphQL request to specific url - * @param endpoint an url - * @param query GraphQL request payload - */ - suspend fun graphQLQuery(endpoint: String, query: String): JSONObject { - val body = JSONObject() - body.put("operationName", null as Any?) - body.put("variables", JSONObject()) - body.put("query", "{$query}") - val mediaType = "application/json; charset=utf-8".toMediaType() - val requestBody = body.toString().toRequestBody(mediaType) - val request = Request.Builder() - .post(requestBody) - .url(endpoint) - val json = httpClient.newCall(request.build()).await().ensureSuccess().parseJson() - json.optJSONArray("errors")?.let { - if (it.length() != 0) { - throw GraphQLException(it) - } - } - return json - } - - open fun encodeBase64(data: ByteArray): String = Base64.getEncoder().encodeToString(data) - - open fun decodeBase64(data: String): ByteArray = Base64.getDecoder().decode(data) - - open fun getPreferredLocales(): List = listOf(Locale.getDefault()) - - /** - * Execute JavaScript code and return result - * @param script JavaScript source code - * @return execution result as string, may be null - */ - abstract suspend fun evaluateJs(script: String): String? - - abstract fun getConfig(source: MangaSource): MangaSourceConfig - - private fun Response.ensureSuccess(): Response { - val exception: Exception? = when (code) { // Catch some error codes, not all - 404 -> NotFoundException(message, request.url.toString()) - in 500..599 -> HttpStatusException(message, code, request.url.toString()) - else -> null - } - if (exception != null) { - runCatching { - close() - }.onFailure { - exception.addSuppressed(it) - } - throw exception - } - return this - } -} \ No newline at end of file + abstract fun getConfig(source: MangaSource): MangaSourceConfig +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaParser.kt index 3d4047cda..435976f16 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaParser.kt @@ -2,197 +2,134 @@ package org.koitharu.kotatsu.parsers import androidx.annotation.CallSuper import okhttp3.Headers -import okhttp3.HttpUrl -import org.jsoup.nodes.Element import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.exception.ParseException import org.koitharu.kotatsu.parsers.model.* +import org.koitharu.kotatsu.parsers.network.OkHttpWebClient +import org.koitharu.kotatsu.parsers.network.WebClient import org.koitharu.kotatsu.parsers.util.FaviconParser +import org.koitharu.kotatsu.parsers.util.domain import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl import java.util.* -abstract class MangaParser @InternalParsersApi constructor(val source: MangaSource) { - - protected abstract val context: MangaLoaderContext - - /** - * Supported [SortOrder] variants. Must not be empty. - * - * For better performance use [EnumSet] for more than one item. - */ - abstract val sortOrders: Set - - val config by lazy { context.getConfig(source) } - - open val sourceLocale: Locale? - get() = source.locale?.let { Locale(it) } - - /** - * Provide default domain and available alternatives, if any. - * - * Never hardcode domain in requests, use [getDomain] instead. - */ - protected abstract val configKeyDomain: ConfigKey.Domain - - open val headers: Headers? = null - - /** - * Used as fallback if value of `sortOrder` passed to [getList] is null - */ - protected open val defaultSortOrder: SortOrder - get() { - val supported = sortOrders - return SortOrder.values().first { it in supported } - } - - /** - * Parse list of manga by specified criteria - * - * @param offset starting from 0 and used for pagination. - * Note than passed value may not be divisible by internal page size, so you should adjust it manually. - * @param query search query, may be null or empty if no search needed - * @param tags genres for filtering, values from [getTags] and [Manga.tags]. May be null or empty - * @param sortOrder one of [sortOrders] or null for default value - */ - @JvmSynthetic - @InternalParsersApi - abstract suspend fun getList( - offset: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List - - /** - * Parse list of manga with search by text query - * - * @param offset starting from 0 and used for pagination. - * @param query search query - */ - open suspend fun getList(offset: Int, query: String): List { - return getList(offset, query, null, defaultSortOrder) - } - - /** - * Parse list of manga by specified criteria - * - * @param offset starting from 0 and used for pagination. - * Note than passed value may not be divisible by internal page size, so you should adjust it manually. - * @param tags genres for filtering, values from [getTags] and [Manga.tags]. May be null or empty - * @param sortOrder one of [sortOrders] or null for default value - */ - open suspend fun getList(offset: Int, tags: Set?, sortOrder: SortOrder?): List { - return getList(offset, null, tags, sortOrder ?: defaultSortOrder) - } - - /** - * Parse details for [Manga]: chapters list, description, large cover, etc. - * Must return the same manga, may change any fields excepts id, url and source - * @see Manga.copy - */ - abstract suspend fun getDetails(manga: Manga): Manga - - /** - * Parse pages list for specified chapter. - * @see MangaPage for details - */ - abstract suspend fun getPages(chapter: MangaChapter): List - - /** - * Fetch direct link to the page image. - */ - open suspend fun getPageUrl(page: MangaPage): String = page.url.toAbsoluteUrl(getDomain()) - - /** - * Fetch available tags (genres) for source - */ - abstract suspend fun getTags(): Set - - /** - * Returns direct link to the website favicon - */ - @Deprecated( - message = "Use parseFavicons() to get multiple favicons with different size", - replaceWith = ReplaceWith("parseFavicons()"), - ) - open fun getFaviconUrl() = "https://${getDomain()}/favicon.ico" - - /** - * Parse favicons from the main page of the source`s website - */ - open suspend fun getFavicons(): Favicons { - return FaviconParser(context, getDomain(), headers).parseFavicons() - } - - @CallSuper - open fun onCreateConfig(keys: MutableCollection>) { - keys.add(configKeyDomain) - } - - /* Utils */ - - fun getDomain(): String { - return config[configKeyDomain] - } - - fun getDomain(subdomain: String): String { - val domain = getDomain() - return subdomain + "." + domain.removePrefix("www.") - } - - fun urlBuilder(): HttpUrl.Builder { - return HttpUrl.Builder() - .scheme("https") - .host(getDomain()) - } - - /** - * Create a unique id for [Manga]/[MangaChapter]/[MangaPage]. - * @param url must be relative url, without a domain - * @see [Manga.id] - * @see [MangaChapter.id] - * @see [MangaPage.id] - */ - @InternalParsersApi - protected fun generateUid(url: String): Long { - var h = 1125899906842597L - source.name.forEach { c -> - h = 31 * h + c.code - } - url.forEach { c -> - h = 31 * h + c.code - } - return h - } - - /** - * Create a unique id for [Manga]/[MangaChapter]/[MangaPage]. - * @param id an internal identifier - * @see [Manga.id] - * @see [MangaChapter.id] - * @see [MangaPage.id] - */ - @InternalParsersApi - protected fun generateUid(id: Long): Long { - var h = 1125899906842597L - source.name.forEach { c -> - h = 31 * h + c.code - } - h = 31 * h + id - return h - } - - @InternalParsersApi - protected fun Element.parseFailed(message: String? = null): Nothing { - throw ParseException(message, ownerDocument()?.location() ?: baseUri(), null) - } - - @InternalParsersApi - protected fun Set?.oneOrThrowIfMany(): MangaTag? { - return when { - isNullOrEmpty() -> null - size == 1 -> first() - else -> throw IllegalArgumentException("Multiple genres are not supported by this source") - } - } -} \ No newline at end of file +abstract class MangaParser @InternalParsersApi constructor( + @property:InternalParsersApi val context: MangaLoaderContext, + val source: MangaSource, +) { + + /** + * Supported [SortOrder] variants. Must not be empty. + * + * For better performance use [EnumSet] for more than one item. + */ + abstract val sortOrders: Set + + val config by lazy { context.getConfig(source) } + + open val sourceLocale: Locale? + get() = source.locale?.let { Locale(it) } + + /** + * Provide default domain and available alternatives, if any. + * + * Never hardcode domain in requests, use [getDomain] instead. + */ + @InternalParsersApi + abstract val configKeyDomain: ConfigKey.Domain + + open val headers: Headers? = null + + /** + * Used as fallback if value of `sortOrder` passed to [getList] is null + */ + protected open val defaultSortOrder: SortOrder + get() { + val supported = sortOrders + return SortOrder.values().first { it in supported } + } + + protected val webClient: WebClient = OkHttpWebClient(context.httpClient, source) + + /** + * Parse list of manga by specified criteria + * + * @param offset starting from 0 and used for pagination. + * Note than passed value may not be divisible by internal page size, so you should adjust it manually. + * @param query search query, may be null or empty if no search needed + * @param tags genres for filtering, values from [getTags] and [Manga.tags]. May be null or empty + * @param sortOrder one of [sortOrders] or null for default value + */ + @JvmSynthetic + @InternalParsersApi + abstract suspend fun getList( + offset: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List + + /** + * Parse list of manga with search by text query + * + * @param offset starting from 0 and used for pagination. + * @param query search query + */ + open suspend fun getList(offset: Int, query: String): List { + return getList(offset, query, null, defaultSortOrder) + } + + /** + * Parse list of manga by specified criteria + * + * @param offset starting from 0 and used for pagination. + * Note than passed value may not be divisible by internal page size, so you should adjust it manually. + * @param tags genres for filtering, values from [getTags] and [Manga.tags]. May be null or empty + * @param sortOrder one of [sortOrders] or null for default value + */ + open suspend fun getList(offset: Int, tags: Set?, sortOrder: SortOrder?): List { + return getList(offset, null, tags, sortOrder ?: defaultSortOrder) + } + + /** + * Parse details for [Manga]: chapters list, description, large cover, etc. + * Must return the same manga, may change any fields excepts id, url and source + * @see Manga.copy + */ + abstract suspend fun getDetails(manga: Manga): Manga + + /** + * Parse pages list for specified chapter. + * @see MangaPage for details + */ + abstract suspend fun getPages(chapter: MangaChapter): List + + /** + * Fetch direct link to the page image. + */ + open suspend fun getPageUrl(page: MangaPage): String = page.url.toAbsoluteUrl(domain) + + /** + * Fetch available tags (genres) for source + */ + abstract suspend fun getTags(): Set + + /** + * Returns direct link to the website favicon + */ + @Deprecated( + message = "Use parseFavicons() to get multiple favicons with different size", + replaceWith = ReplaceWith("parseFavicons()"), + ) + open fun getFaviconUrl() = "https://${domain}/favicon.ico" + + /** + * Parse favicons from the main page of the source`s website + */ + open suspend fun getFavicons(): Favicons { + return FaviconParser(webClient, domain).parseFavicons() + } + + @CallSuper + open fun onCreateConfig(keys: MutableCollection>) { + keys.add(configKeyDomain) + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/PagedMangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/PagedMangaParser.kt index 7040bce47..ea18f848f 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/PagedMangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/PagedMangaParser.kt @@ -8,43 +8,44 @@ import org.koitharu.kotatsu.parsers.util.Paginator @InternalParsersApi abstract class PagedMangaParser( - source: MangaSource, - pageSize: Int, - searchPageSize: Int = pageSize, -) : MangaParser(source) { - - protected val paginator = Paginator(pageSize) - protected val searchPaginator = Paginator(searchPageSize) - - override suspend fun getList(offset: Int, query: String): List { - return getList(searchPaginator, offset, query, null, defaultSortOrder) - } - - override suspend fun getList(offset: Int, tags: Set?, sortOrder: SortOrder?): List { - return getList(paginator, offset, null, tags, sortOrder ?: defaultSortOrder) - } - - @InternalParsersApi - @Deprecated("You should use getListPage for PagedMangaParser", level = DeprecationLevel.HIDDEN) - final override suspend fun getList( - offset: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List = throw UnsupportedOperationException("You should use getListPage for PagedMangaParser") - - abstract suspend fun getListPage(page: Int, query: String?, tags: Set?, sortOrder: SortOrder): List - - private suspend fun getList( - paginator: Paginator, - offset: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - val page = paginator.getPage(offset) - val list = getListPage(page, query, tags, sortOrder) - paginator.onListReceived(offset, page, list.size) - return list - } -} \ No newline at end of file + context: MangaLoaderContext, + source: MangaSource, + pageSize: Int, + searchPageSize: Int = pageSize, +) : MangaParser(context, source) { + + protected val paginator = Paginator(pageSize) + protected val searchPaginator = Paginator(searchPageSize) + + override suspend fun getList(offset: Int, query: String): List { + return getList(searchPaginator, offset, query, null, defaultSortOrder) + } + + override suspend fun getList(offset: Int, tags: Set?, sortOrder: SortOrder?): List { + return getList(paginator, offset, null, tags, sortOrder ?: defaultSortOrder) + } + + @InternalParsersApi + @Deprecated("You should use getListPage for PagedMangaParser", level = DeprecationLevel.HIDDEN) + final override suspend fun getList( + offset: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List = throw UnsupportedOperationException("You should use getListPage for PagedMangaParser") + + abstract suspend fun getListPage(page: Int, query: String?, tags: Set?, sortOrder: SortOrder): List + + private suspend fun getList( + paginator: Paginator, + offset: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + val page = paginator.getPage(offset) + val list = getListPage(page, query, tags, sortOrder) + paginator.onListReceived(offset, page, list.size) + return list + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/model/MangaPage.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/model/MangaPage.kt index 4cc66d298..0c5102a09 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/model/MangaPage.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/model/MangaPage.kt @@ -3,50 +3,49 @@ package org.koitharu.kotatsu.parsers.model import org.koitharu.kotatsu.parsers.MangaParser class MangaPage( - /** - * Unique identifier for manga - */ - val id: Long, - /** - * Relative url to page (**without** a domain) or any other uri. - * Used principally in parsers. - * May contain link to image or html page. - * @see MangaParser.getPageUrl - */ - val url: String, - /** - * Absolute link to the chapter or website home page. - * Used in Referer header - */ - val referer: String, - /** - * Absolute url of the small page image if exists, null otherwise - */ - val preview: String?, - val source: MangaSource, + /** + * Unique identifier for manga + */ + val id: Long, + /** + * Relative url to page (**without** a domain) or any other uri. + * Used principally in parsers. + * May contain link to image or html page. + * @see MangaParser.getPageUrl + */ + val url: String, + /** + * Absolute link to the chapter or website home page. + * Used in Referer header + */ + @Deprecated("Referer header should be added based on source domain") + val referer: String, + /** + * Absolute url of the small page image if exists, null otherwise + */ + val preview: String?, + val source: MangaSource, ) { - override fun equals(other: Any?): Boolean { - if (this === other) return true - if (javaClass != other?.javaClass) return false + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (javaClass != other?.javaClass) return false - other as MangaPage + other as MangaPage - if (id != other.id) return false - if (url != other.url) return false - if (referer != other.referer) return false - if (preview != other.preview) return false - if (source != other.source) return false + if (id != other.id) return false + if (url != other.url) return false + if (referer != other.referer) return false + if (preview != other.preview) return false + return source == other.source + } - return true - } - - override fun hashCode(): Int { - var result = id.hashCode() - result = 31 * result + url.hashCode() - result = 31 * result + referer.hashCode() - result = 31 * result + (preview?.hashCode() ?: 0) - result = 31 * result + source.hashCode() - return result - } -} \ No newline at end of file + override fun hashCode(): Int { + var result = id.hashCode() + result = 31 * result + url.hashCode() + result = 31 * result + referer.hashCode() + result = 31 * result + (preview?.hashCode() ?: 0) + result = 31 * result + source.hashCode() + return result + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/network/OkHttpWebClient.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/network/OkHttpWebClient.kt new file mode 100644 index 000000000..7d61ba1e7 --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/network/OkHttpWebClient.kt @@ -0,0 +1,107 @@ +package org.koitharu.kotatsu.parsers.network + +import okhttp3.* +import okhttp3.MediaType.Companion.toMediaType +import okhttp3.RequestBody.Companion.toRequestBody +import org.json.JSONObject +import org.jsoup.HttpStatusException +import org.koitharu.kotatsu.parsers.exception.GraphQLException +import org.koitharu.kotatsu.parsers.exception.NotFoundException +import org.koitharu.kotatsu.parsers.model.MangaSource +import org.koitharu.kotatsu.parsers.util.await +import org.koitharu.kotatsu.parsers.util.parseJson + +class OkHttpWebClient( + private val httpClient: OkHttpClient, + private val mangaSource: MangaSource, +) : WebClient { + + override suspend fun httpGet(url: HttpUrl): Response { + val request = Request.Builder() + .get() + .url(url) + .addTags() + return httpClient.newCall(request.build()).await().ensureSuccess() + } + + override suspend fun httpHead(url: HttpUrl): Response { + val request = Request.Builder() + .head() + .url(url) + .addTags() + return httpClient.newCall(request.build()).await().ensureSuccess() + } + + override suspend fun httpPost(url: HttpUrl, form: Map): Response { + val body = FormBody.Builder() + form.forEach { (k, v) -> + body.addEncoded(k, v) + } + val request = Request.Builder() + .post(body.build()) + .url(url) + .addTags() + return httpClient.newCall(request.build()).await().ensureSuccess() + } + + override suspend fun httpPost(url: HttpUrl, payload: String): Response { + val body = FormBody.Builder() + payload.split('&').forEach { + val pos = it.indexOf('=') + if (pos != -1) { + val k = it.substring(0, pos) + val v = it.substring(pos + 1) + body.addEncoded(k, v) + } + } + val request = Request.Builder() + .post(body.build()) + .url(url) + .addTags() + return httpClient.newCall(request.build()).await().ensureSuccess() + } + + override suspend fun graphQLQuery(endpoint: String, query: String): JSONObject { + val body = JSONObject() + body.put("operationName", null as Any?) + body.put("variables", JSONObject()) + body.put("query", "{$query}") + val mediaType = "application/json; charset=utf-8".toMediaType() + val requestBody = body.toString().toRequestBody(mediaType) + val request = Request.Builder() + .post(requestBody) + .url(endpoint) + .addTags() + val json = httpClient.newCall(request.build()).await().ensureSuccess().parseJson() + json.optJSONArray("errors")?.let { + if (it.length() != 0) { + throw GraphQLException(it) + } + } + return json + } + + private fun Request.Builder.addTags(): Request.Builder { + if (mangaSource != null) { + tag(MangaSource::class.java, mangaSource) + } + return this + } + + private fun Response.ensureSuccess(): Response { + val exception: Exception? = when (code) { // Catch some error codes, not all + 404 -> NotFoundException(message, request.url.toString()) + in 500..599 -> HttpStatusException(message, code, request.url.toString()) + else -> null + } + if (exception != null) { + runCatching { + close() + }.onFailure { + exception.addSuppressed(it) + } + throw exception + } + return this + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/network/WebClient.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/network/WebClient.kt new file mode 100644 index 000000000..93770c4ff --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/network/WebClient.kt @@ -0,0 +1,68 @@ +package org.koitharu.kotatsu.parsers.network + +import okhttp3.HttpUrl +import okhttp3.HttpUrl.Companion.toHttpUrl +import okhttp3.Response +import org.json.JSONObject + +interface WebClient { + + /** + * Do a GET http request to specific url + * @param url + */ + suspend fun httpGet(url: String): Response = httpGet(url.toHttpUrl()) + + /** + * Do a GET http request to specific url + * @param url + */ + suspend fun httpGet(url: HttpUrl): Response + + /** + * Do a HEAD http request to specific url + * @param url + */ + suspend fun httpHead(url: String): Response = httpHead(url.toHttpUrl()) + + /** + * Do a HEAD http request to specific url + * @param url + */ + suspend fun httpHead(url: HttpUrl): Response + + /** + * Do a POST http request to specific url with `multipart/form-data` payload + * @param url + * @param form payload as key=>value map + */ + suspend fun httpPost(url: String, form: Map): Response = httpPost(url.toHttpUrl(), form) + + /** + * Do a POST http request to specific url with `multipart/form-data` payload + * @param url + * @param form payload as key=>value map + */ + suspend fun httpPost(url: HttpUrl, form: Map): Response + + /** + * Do a POST http request to specific url with `multipart/form-data` payload + * @param url + * @param payload payload as `key=value` string with `&` separator + */ + suspend fun httpPost(url: String, payload: String): Response = httpPost(url.toHttpUrl(), payload) + + /** + * Do a POST http request to specific url with `multipart/form-data` payload + * @param url + * @param payload payload as `key=value` string with `&` separator + */ + suspend fun httpPost(url: HttpUrl, payload: String): Response + + /** + * Do a GraphQL request to specific url + * @param endpoint an url + * @param query GraphQL request payload + */ + suspend fun graphQLQuery(endpoint: String, query: String): JSONObject +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/AnibelParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/AnibelParser.kt index 8436ee1d6..a01f96bee 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/AnibelParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/AnibelParser.kt @@ -8,6 +8,9 @@ import org.koitharu.kotatsu.parsers.MangaParser import org.koitharu.kotatsu.parsers.MangaSourceParser import org.koitharu.kotatsu.parsers.config.ConfigKey import org.koitharu.kotatsu.parsers.model.* +import org.koitharu.kotatsu.parsers.util.domain +import org.koitharu.kotatsu.parsers.util.generateUid +import org.koitharu.kotatsu.parsers.util.getDomain import org.koitharu.kotatsu.parsers.util.json.mapJSON import org.koitharu.kotatsu.parsers.util.json.mapJSONIndexed import org.koitharu.kotatsu.parsers.util.json.stringIterator @@ -15,38 +18,38 @@ import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl import java.util.* @MangaSourceParser("ANIBEL", "Anibel", "be") -internal class AnibelParser(override val context: MangaLoaderContext) : MangaParser(MangaSource.ANIBEL) { +internal class AnibelParser(context: MangaLoaderContext) : MangaParser(context, MangaSource.ANIBEL) { - override val configKeyDomain = ConfigKey.Domain("anibel.net", null) + override val configKeyDomain = ConfigKey.Domain("anibel.net", null) - override val sortOrders: Set = EnumSet.of( - SortOrder.NEWEST, - ) + override val sortOrders: Set = EnumSet.of( + SortOrder.NEWEST, + ) - override fun getFaviconUrl(): String { - return "https://cdn.${getDomain()}/favicons/favicon.png" - } + override fun getFaviconUrl(): String { + return "https://cdn.${domain}/favicons/favicon.png" + } - override suspend fun getList( - offset: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - if (!query.isNullOrEmpty()) { - return if (offset == 0) { - search(query) - } else { - emptyList() - } - } - val filters = tags?.takeUnless { it.isEmpty() }?.joinToString( - separator = ",", - prefix = "genres: [", - postfix = "]", - ) { "\"${it.key}\"" }.orEmpty() - val array = apiCall( - """ + override suspend fun getList( + offset: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + if (!query.isNullOrEmpty()) { + return if (offset == 0) { + search(query) + } else { + emptyList() + } + } + val filters = tags?.takeUnless { it.isEmpty() }?.joinToString( + separator = ",", + prefix = "genres: [", + postfix = "]", + ) { "\"${it.key}\"" }.orEmpty() + val array = apiCall( + """ getMediaList(offset: $offset, limit: 20, mediaType: manga, filters: {$filters}) { docs { mediaId @@ -63,37 +66,37 @@ internal class AnibelParser(override val context: MangaLoaderContext) : MangaPar } } """.trimIndent(), - ).getJSONObject("getMediaList").getJSONArray("docs") - return array.mapJSON { jo -> - val mediaId = jo.getString("mediaId") - val title = jo.getJSONObject("title") - val href = "${jo.getString("mediaType")}/${jo.getString("slug")}" - Manga( - id = generateUid(mediaId), - title = title.getString("be"), - coverUrl = jo.getString("poster").removePrefix("/cdn") - .toAbsoluteUrl(getDomain("cdn")) + "?width=200&height=280", - altTitle = title.getString("alt").takeUnless(String::isEmpty), - author = null, - isNsfw = false, - rating = jo.getDouble("rating").toFloat() / 10f, - url = href, - publicUrl = "https://${getDomain()}/$href", - tags = jo.getJSONArray("genres").mapToTags(), - state = when (jo.getString("status")) { - "ongoing" -> MangaState.ONGOING - "finished" -> MangaState.FINISHED - else -> null - }, - source = source, - ) - } - } + ).getJSONObject("getMediaList").getJSONArray("docs") + return array.mapJSON { jo -> + val mediaId = jo.getString("mediaId") + val title = jo.getJSONObject("title") + val href = "${jo.getString("mediaType")}/${jo.getString("slug")}" + Manga( + id = generateUid(mediaId), + title = title.getString("be"), + coverUrl = jo.getString("poster").removePrefix("/cdn") + .toAbsoluteUrl(getDomain("cdn")) + "?width=200&height=280", + altTitle = title.getString("alt").takeUnless(String::isEmpty), + author = null, + isNsfw = false, + rating = jo.getDouble("rating").toFloat() / 10f, + url = href, + publicUrl = "https://${domain}/$href", + tags = jo.getJSONArray("genres").mapToTags(), + state = when (jo.getString("status")) { + "ongoing" -> MangaState.ONGOING + "finished" -> MangaState.FINISHED + else -> null + }, + source = source, + ) + } + } - override suspend fun getDetails(manga: Manga): Manga { - val (type, slug) = manga.url.split('/') - val details = apiCall( - """ + override suspend fun getDetails(manga: Manga): Manga { + val (type, slug) = manga.url.split('/') + val details = apiCall( + """ media(mediaType: $type, slug: "$slug") { mediaId title { @@ -109,52 +112,52 @@ internal class AnibelParser(override val context: MangaLoaderContext) : MangaPar genres } """.trimIndent(), - ).getJSONObject("media") - val title = details.getJSONObject("title") - val poster = details.getString("poster").removePrefix("/cdn") - .toAbsoluteUrl(getDomain("cdn")) - val chapters = apiCall( - """ + ).getJSONObject("media") + val title = details.getJSONObject("title") + val poster = details.getString("poster").removePrefix("/cdn") + .toAbsoluteUrl(getDomain("cdn")) + val chapters = apiCall( + """ chapters(mediaId: "${details.getString("mediaId")}") { id chapter released } """.trimIndent(), - ).getJSONArray("chapters") - return manga.copy( - title = title.getString("be"), - altTitle = title.getString("alt"), - coverUrl = "$poster?width=200&height=280", - largeCoverUrl = poster, - description = details.getJSONObject("description").getString("be"), - rating = details.getDouble("rating").toFloat() / 10f, - tags = details.getJSONArray("genres").mapToTags(), - state = when (details.getString("status")) { - "ongoing" -> MangaState.ONGOING - "finished" -> MangaState.FINISHED - else -> null - }, - chapters = chapters.mapJSON { jo -> - val number = jo.getInt("chapter") - MangaChapter( - id = generateUid(jo.getString("id")), - name = "Глава $number", - number = number, - url = "${manga.url}/read/$number", - scanlator = null, - uploadDate = jo.getLong("released"), - branch = null, - source = source, - ) - }, - ) - } + ).getJSONArray("chapters") + return manga.copy( + title = title.getString("be"), + altTitle = title.getString("alt"), + coverUrl = "$poster?width=200&height=280", + largeCoverUrl = poster, + description = details.getJSONObject("description").getString("be"), + rating = details.getDouble("rating").toFloat() / 10f, + tags = details.getJSONArray("genres").mapToTags(), + state = when (details.getString("status")) { + "ongoing" -> MangaState.ONGOING + "finished" -> MangaState.FINISHED + else -> null + }, + chapters = chapters.mapJSON { jo -> + val number = jo.getInt("chapter") + MangaChapter( + id = generateUid(jo.getString("id")), + name = "Глава $number", + number = number, + url = "${manga.url}/read/$number", + scanlator = null, + uploadDate = jo.getLong("released"), + branch = null, + source = source, + ) + }, + ) + } - override suspend fun getPages(chapter: MangaChapter): List { - val (_, slug, _, number) = chapter.url.split('/') - val chapterJson = apiCall( - """ + override suspend fun getPages(chapter: MangaChapter): List { + val (_, slug, _, number) = chapter.url.split('/') + val chapterJson = apiCall( + """ chapter(slug: "$slug", chapter: $number) { id images { @@ -163,35 +166,35 @@ internal class AnibelParser(override val context: MangaLoaderContext) : MangaPar } } """.trimIndent(), - ).getJSONObject("chapter") - val pages = chapterJson.getJSONArray("images") - val chapterUrl = "https://${getDomain()}/${chapter.url}" - return pages.mapJSONIndexed { i, jo -> - MangaPage( - id = generateUid("${chapter.url}/$i"), - url = jo.getString("large"), - referer = chapterUrl, - preview = jo.getString("thumbnail"), - source = source, - ) - } - } + ).getJSONObject("chapter") + val pages = chapterJson.getJSONArray("images") + val chapterUrl = "https://${domain}/${chapter.url}" + return pages.mapJSONIndexed { i, jo -> + MangaPage( + id = generateUid("${chapter.url}/$i"), + url = jo.getString("large"), + referer = chapterUrl, + preview = jo.getString("thumbnail"), + source = source, + ) + } + } - override suspend fun getTags(): Set { - val json = apiCall( - """ + override suspend fun getTags(): Set { + val json = apiCall( + """ getFilters(mediaType: manga) { genres } """.trimIndent(), - ) - val array = json.getJSONObject("getFilters").getJSONArray("genres") - return array.mapToTags() - } + ) + val array = json.getJSONObject("getFilters").getJSONArray("genres") + return array.mapToTags() + } - private suspend fun search(query: String): List { - val json = apiCall( - """ + private suspend fun search(query: String): List { + val json = apiCall( + """ search(query: "$query", limit: 40) { id title { @@ -203,64 +206,65 @@ internal class AnibelParser(override val context: MangaLoaderContext) : MangaPar type } """.trimIndent(), - ) - val array = json.getJSONArray("search") - return array.mapJSON { jo -> - val mediaId = jo.getString("id") - val title = jo.getJSONObject("title") - val href = "${jo.getString("type").lowercase()}/${jo.getString("url")}" - Manga( - id = generateUid(mediaId), - title = title.getString("be"), - coverUrl = jo.getString("poster").removePrefix("/cdn") - .toAbsoluteUrl(getDomain("cdn")) + "?width=200&height=280", - altTitle = title.getString("en").takeUnless(String::isEmpty), - author = null, - isNsfw = false, - rating = RATING_UNKNOWN, - url = href, - publicUrl = "https://${getDomain()}/$href", - tags = emptySet(), - state = null, - source = source, - ) - } - } + ) + val array = json.getJSONArray("search") + return array.mapJSON { jo -> + val mediaId = jo.getString("id") + val title = jo.getJSONObject("title") + val href = "${jo.getString("type").lowercase()}/${jo.getString("url")}" + Manga( + id = generateUid(mediaId), + title = title.getString("be"), + coverUrl = jo.getString("poster").removePrefix("/cdn") + .toAbsoluteUrl(getDomain("cdn")) + "?width=200&height=280", + altTitle = title.getString("en").takeUnless(String::isEmpty), + author = null, + isNsfw = false, + rating = RATING_UNKNOWN, + url = href, + publicUrl = "https://${domain}/$href", + tags = emptySet(), + state = null, + source = source, + ) + } + } - private suspend fun apiCall(request: String): JSONObject { - return context.graphQLQuery("https://api.${getDomain()}/graphql", request) - .getJSONObject("data") - } + private suspend fun apiCall(request: String): JSONObject { + return webClient.graphQLQuery("https://api.${domain}/graphql", request) + .getJSONObject("data") + } - private fun JSONArray.mapToTags(): Set { + private fun JSONArray.mapToTags(): Set { - fun toTitle(slug: String): String { - val builder = StringBuilder(slug) - var capitalize = true - for ((i, c) in builder.withIndex()) { - when { - c == '-' -> { - builder.setCharAt(i, ' ') - } - capitalize -> { - builder.setCharAt(i, c.uppercaseChar()) - capitalize = false - } - } - } - return builder.toString() - } + fun toTitle(slug: String): String { + val builder = StringBuilder(slug) + var capitalize = true + for ((i, c) in builder.withIndex()) { + when { + c == '-' -> { + builder.setCharAt(i, ' ') + } + + capitalize -> { + builder.setCharAt(i, c.uppercaseChar()) + capitalize = false + } + } + } + return builder.toString() + } - val result = ArraySet(length()) - stringIterator().forEach { - result.add( - MangaTag( - title = toTitle(it), - key = it, - source = source, - ), - ) - } - return result - } -} \ No newline at end of file + val result = ArraySet(length()) + stringIterator().forEach { + result.add( + MangaTag( + title = toTitle(it), + key = it, + source = source, + ), + ) + } + return result + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BatoToParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BatoToParser.kt index a366c1d1b..f162a6804 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BatoToParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BatoToParser.kt @@ -19,292 +19,293 @@ import javax.crypto.spec.IvParameterSpec import javax.crypto.spec.SecretKeySpec @MangaSourceParser("BATOTO", "Bato.To") -internal class BatoToParser(override val context: MangaLoaderContext) : PagedMangaParser( - source = MangaSource.BATOTO, - pageSize = 60, - searchPageSize = 20, +internal class BatoToParser(context: MangaLoaderContext) : PagedMangaParser( + context = context, + source = MangaSource.BATOTO, + pageSize = 60, + searchPageSize = 20, ) { - override val sortOrders: Set = EnumSet.of( - SortOrder.NEWEST, - SortOrder.UPDATED, - SortOrder.POPULARITY, - SortOrder.ALPHABETICAL, - ) + override val sortOrders: Set = EnumSet.of( + SortOrder.NEWEST, + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.ALPHABETICAL, + ) - override val configKeyDomain = ConfigKey.Domain( - "bato.to", - arrayOf("bato.to", "mto.to", "mangatoto.com", "battwo.com", "batotwo.com", "comiko.net", "batotoo.com"), - ) + override val configKeyDomain = ConfigKey.Domain( + "bato.to", + arrayOf("bato.to", "mto.to", "mangatoto.com", "battwo.com", "batotwo.com", "comiko.net", "batotoo.com"), + ) - override suspend fun getListPage( - page: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - if (!query.isNullOrEmpty()) { - return search(page, query) - } - @Suppress("NON_EXHAUSTIVE_WHEN_STATEMENT") - val url = buildString { - append("https://") - append(getDomain()) - append("/browse?sort=") - when (sortOrder) { - SortOrder.UPDATED, - -> append("update.za") + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + if (!query.isNullOrEmpty()) { + return search(page, query) + } + @Suppress("NON_EXHAUSTIVE_WHEN_STATEMENT") + val url = buildString { + append("https://") + append(domain) + append("/browse?sort=") + when (sortOrder) { + SortOrder.UPDATED, + -> append("update.za") - SortOrder.POPULARITY -> append("views_a.za") - SortOrder.NEWEST -> append("create.za") - SortOrder.ALPHABETICAL -> append("title.az") - } - if (!tags.isNullOrEmpty()) { - append("&genres=") - appendAll(tags, ",") { it.key } - } - append("&page=") - append(page) - } - return parseList(url, page) - } + SortOrder.POPULARITY -> append("views_a.za") + SortOrder.NEWEST -> append("create.za") + SortOrder.ALPHABETICAL -> append("title.az") + } + if (!tags.isNullOrEmpty()) { + append("&genres=") + appendAll(tags, ",") { it.key } + } + append("&page=") + append(page) + } + return parseList(url, page) + } - override suspend fun getDetails(manga: Manga): Manga { - val root = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml() - .requireElementById("mainer") - val details = root.selectFirstOrThrow(".detail-set") - val attrs = details.selectFirst(".attr-main")?.select(".attr-item")?.associate { - it.child(0).text().trim() to it.child(1) - }.orEmpty() - return manga.copy( - title = root.selectFirst("h3.item-title")?.text() ?: manga.title, - isNsfw = !root.selectFirst("alert")?.getElementsContainingOwnText("NSFW").isNullOrEmpty(), - largeCoverUrl = details.selectFirst("img[src]")?.absUrl("src"), - description = details.getElementById("limit-height-body-summary") - ?.selectFirst(".limit-html") - ?.html(), - tags = manga.tags + attrs["Genres:"]?.parseTags().orEmpty(), - state = when (attrs["Release status:"]?.text()) { - "Ongoing" -> MangaState.ONGOING - "Completed" -> MangaState.FINISHED - else -> manga.state - }, - author = attrs["Authors:"]?.text()?.trim() ?: manga.author, - chapters = root.selectFirst(".episode-list") - ?.selectFirst(".main") - ?.children() - ?.reversed() - ?.mapChapters { i, div -> - div.parseChapter(i) - }.orEmpty(), - ) - } + override suspend fun getDetails(manga: Manga): Manga { + val root = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + .requireElementById("mainer") + val details = root.selectFirstOrThrow(".detail-set") + val attrs = details.selectFirst(".attr-main")?.select(".attr-item")?.associate { + it.child(0).text().trim() to it.child(1) + }.orEmpty() + return manga.copy( + title = root.selectFirst("h3.item-title")?.text() ?: manga.title, + isNsfw = !root.selectFirst("alert")?.getElementsContainingOwnText("NSFW").isNullOrEmpty(), + largeCoverUrl = details.selectFirst("img[src]")?.absUrl("src"), + description = details.getElementById("limit-height-body-summary") + ?.selectFirst(".limit-html") + ?.html(), + tags = manga.tags + attrs["Genres:"]?.parseTags().orEmpty(), + state = when (attrs["Release status:"]?.text()) { + "Ongoing" -> MangaState.ONGOING + "Completed" -> MangaState.FINISHED + else -> manga.state + }, + author = attrs["Authors:"]?.text()?.trim() ?: manga.author, + chapters = root.selectFirst(".episode-list") + ?.selectFirst(".main") + ?.children() + ?.reversed() + ?.mapChapters { i, div -> + div.parseChapter(i) + }.orEmpty(), + ) + } - override suspend fun getPages(chapter: MangaChapter): List { - val fullUrl = chapter.url.toAbsoluteUrl(getDomain()) - val scripts = context.httpGet(fullUrl).parseHtml().select("script") - for (script in scripts) { - val scriptSrc = script.html() - val p = scriptSrc.indexOf("const imgHttpLis =") - if (p == -1) continue - val start = scriptSrc.indexOf('[', p) - val end = scriptSrc.indexOf(';', start) - if (start == -1 || end == -1) { - continue - } - val images = JSONArray(scriptSrc.substring(start, end)) - val batoPass = scriptSrc.substringBetweenFirst("batoPass =", ";")?.trim(' ', '"', '\n') - ?: script.parseFailed("Cannot find batoPass") - val batoWord = scriptSrc.substringBetweenFirst("batoWord =", ";")?.trim(' ', '"', '\n') - ?: script.parseFailed("Cannot find batoWord") - val password = context.evaluateJs(batoPass)?.removeSurrounding('"') - ?: script.parseFailed("Cannot evaluate batoPass") - val args = JSONArray(decryptAES(batoWord, password)) - val result = ArrayList(images.length()) - repeat(images.length()) { i -> - val url = images.getString(i) - result += MangaPage( - id = generateUid(url), - url = url + "?" + args.getString(i), - referer = fullUrl, - preview = null, - source = source, - ) - } - return result - } - throw ParseException("Cannot find images list", fullUrl) - } + override suspend fun getPages(chapter: MangaChapter): List { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val scripts = webClient.httpGet(fullUrl).parseHtml().select("script") + for (script in scripts) { + val scriptSrc = script.html() + val p = scriptSrc.indexOf("const imgHttpLis =") + if (p == -1) continue + val start = scriptSrc.indexOf('[', p) + val end = scriptSrc.indexOf(';', start) + if (start == -1 || end == -1) { + continue + } + val images = JSONArray(scriptSrc.substring(start, end)) + val batoPass = scriptSrc.substringBetweenFirst("batoPass =", ";")?.trim(' ', '"', '\n') + ?: script.parseFailed("Cannot find batoPass") + val batoWord = scriptSrc.substringBetweenFirst("batoWord =", ";")?.trim(' ', '"', '\n') + ?: script.parseFailed("Cannot find batoWord") + val password = context.evaluateJs(batoPass)?.removeSurrounding('"') + ?: script.parseFailed("Cannot evaluate batoPass") + val args = JSONArray(decryptAES(batoWord, password)) + val result = ArrayList(images.length()) + repeat(images.length()) { i -> + val url = images.getString(i) + result += MangaPage( + id = generateUid(url), + url = url + "?" + args.getString(i), + referer = fullUrl, + preview = null, + source = source, + ) + } + return result + } + throw ParseException("Cannot find images list", fullUrl) + } - override suspend fun getTags(): Set { - val scripts = context.httpGet( - "https://${getDomain()}/browse", - ).parseHtml().selectOrThrow("script") - for (script in scripts) { - val genres = script.html().substringBetweenFirst("const _genres =", ";") ?: continue - val jo = JSONObject(genres) - val result = ArraySet(jo.length()) - jo.keys().forEach { key -> - val item = jo.getJSONObject(key) - result += MangaTag( - title = item.getString("text").toTitleCase(), - key = item.getString("file"), - source = source, - ) - } - return result - } - throw ParseException("Cannot find gernes list", scripts[0].baseUri()) - } + override suspend fun getTags(): Set { + val scripts = webClient.httpGet( + "https://${domain}/browse", + ).parseHtml().selectOrThrow("script") + for (script in scripts) { + val genres = script.html().substringBetweenFirst("const _genres =", ";") ?: continue + val jo = JSONObject(genres) + val result = ArraySet(jo.length()) + jo.keys().forEach { key -> + val item = jo.getJSONObject(key) + result += MangaTag( + title = item.getString("text").toTitleCase(), + key = item.getString("file"), + source = source, + ) + } + return result + } + throw ParseException("Cannot find gernes list", scripts[0].baseUri()) + } - override fun getFaviconUrl(): String = "https://styles.amarkcdn.com/img/batoto/favicon.ico?v0" + override fun getFaviconUrl(): String = "https://styles.amarkcdn.com/img/batoto/favicon.ico?v0" - private suspend fun search(page: Int, query: String): List { - val url = buildString { - append("https://") - append(getDomain()) - append("/search?word=") - append(query.replace(' ', '+')) - append("&page=") - append(page) - } - return parseList(url, page) - } + private suspend fun search(page: Int, query: String): List { + val url = buildString { + append("https://") + append(domain) + append("/search?word=") + append(query.replace(' ', '+')) + append("&page=") + append(page) + } + return parseList(url, page) + } - private fun getActivePage(body: Element): Int = body.select("nav ul.pagination > li.page-item.active") - .lastOrNull() - ?.text() - ?.toIntOrNull() ?: body.parseFailed("Cannot determine current page") + private fun getActivePage(body: Element): Int = body.select("nav ul.pagination > li.page-item.active") + .lastOrNull() + ?.text() + ?.toIntOrNull() ?: body.parseFailed("Cannot determine current page") - private suspend fun parseList(url: String, page: Int): List { - val body = context.httpGet(url).parseHtml().body() - if (body.selectFirst(".browse-no-matches") != null) { - return emptyList() - } - val activePage = getActivePage(body) - if (activePage != page) { - return emptyList() - } - val root = body.requireElementById("series-list") - return root.children().map { div -> - val a = div.selectFirstOrThrow("a") - val href = a.attrAsRelativeUrl("href") - val title = div.selectFirstOrThrow(".item-title").text() - Manga( - id = generateUid(href), - title = title, - altTitle = div.selectFirst(".item-alias")?.text()?.takeUnless { it == title }, - url = href, - publicUrl = a.absUrl("href"), - rating = RATING_UNKNOWN, - isNsfw = false, - coverUrl = div.selectFirst("img[src]")?.absUrl("src").orEmpty(), - largeCoverUrl = null, - description = null, - tags = div.selectFirst(".item-genre")?.parseTags().orEmpty(), - state = null, - author = null, - source = source, - ) - } - } + private suspend fun parseList(url: String, page: Int): List { + val body = webClient.httpGet(url).parseHtml().body() + if (body.selectFirst(".browse-no-matches") != null) { + return emptyList() + } + val activePage = getActivePage(body) + if (activePage != page) { + return emptyList() + } + val root = body.requireElementById("series-list") + return root.children().map { div -> + val a = div.selectFirstOrThrow("a") + val href = a.attrAsRelativeUrl("href") + val title = div.selectFirstOrThrow(".item-title").text() + Manga( + id = generateUid(href), + title = title, + altTitle = div.selectFirst(".item-alias")?.text()?.takeUnless { it == title }, + url = href, + publicUrl = a.absUrl("href"), + rating = RATING_UNKNOWN, + isNsfw = false, + coverUrl = div.selectFirst("img[src]")?.absUrl("src").orEmpty(), + largeCoverUrl = null, + description = null, + tags = div.selectFirst(".item-genre")?.parseTags().orEmpty(), + state = null, + author = null, + source = source, + ) + } + } - private fun Element.parseTags() = children().mapToSet { span -> - val text = span.ownText() - MangaTag( - title = text.toTitleCase(), - key = text.lowercase(Locale.ENGLISH).replace(' ', '_'), - source = source, - ) - } + private fun Element.parseTags() = children().mapToSet { span -> + val text = span.ownText() + MangaTag( + title = text.toTitleCase(), + key = text.lowercase(Locale.ENGLISH).replace(' ', '_'), + source = source, + ) + } - private fun Element.parseChapter(index: Int): MangaChapter? { - val a = selectFirst("a.chapt") ?: return null - val extra = selectFirst(".extra") - val href = a.attrAsRelativeUrl("href") - return MangaChapter( - id = generateUid(href), - name = a.text(), - number = index + 1, - url = href, - scanlator = extra?.getElementsByAttributeValueContaining("href", "/group/")?.text(), - uploadDate = runCatching { - parseChapterDate(extra?.select("i")?.lastOrNull()?.ownText()) - }.getOrDefault(0), - branch = null, - source = source, - ) - } + private fun Element.parseChapter(index: Int): MangaChapter? { + val a = selectFirst("a.chapt") ?: return null + val extra = selectFirst(".extra") + val href = a.attrAsRelativeUrl("href") + return MangaChapter( + id = generateUid(href), + name = a.text(), + number = index + 1, + url = href, + scanlator = extra?.getElementsByAttributeValueContaining("href", "/group/")?.text(), + uploadDate = runCatching { + parseChapterDate(extra?.select("i")?.lastOrNull()?.ownText()) + }.getOrDefault(0), + branch = null, + source = source, + ) + } - private fun parseChapterDate(date: String?): Long { - if (date.isNullOrEmpty()) { - return 0 - } - val value = date.substringBefore(' ').toInt() - val field = when { - "sec" in date -> Calendar.SECOND - "min" in date -> Calendar.MINUTE - "hour" in date -> Calendar.HOUR - "day" in date -> Calendar.DAY_OF_MONTH - "week" in date -> Calendar.WEEK_OF_YEAR - "month" in date -> Calendar.MONTH - "year" in date -> Calendar.YEAR - else -> return 0 - } - val calendar = Calendar.getInstance() - calendar.add(field, -value) - return calendar.timeInMillis - } + private fun parseChapterDate(date: String?): Long { + if (date.isNullOrEmpty()) { + return 0 + } + val value = date.substringBefore(' ').toInt() + val field = when { + "sec" in date -> Calendar.SECOND + "min" in date -> Calendar.MINUTE + "hour" in date -> Calendar.HOUR + "day" in date -> Calendar.DAY_OF_MONTH + "week" in date -> Calendar.WEEK_OF_YEAR + "month" in date -> Calendar.MONTH + "year" in date -> Calendar.YEAR + else -> return 0 + } + val calendar = Calendar.getInstance() + calendar.add(field, -value) + return calendar.timeInMillis + } - private fun decryptAES(encrypted: String, password: String): String { - val cipherData = context.decodeBase64(encrypted) - val saltData = cipherData.copyOfRange(8, 16) - val (key, iv) = generateKeyAndIV( - keyLength = 32, - ivLength = 16, - iterations = 1, - salt = saltData, - password = password.toByteArray(StandardCharsets.UTF_8), - md = MessageDigest.getInstance("MD5"), - ) - val encryptedData = cipherData.copyOfRange(16, cipherData.size) - val cipher = Cipher.getInstance("AES/CBC/PKCS5Padding") - cipher.init(Cipher.DECRYPT_MODE, key, iv) - return cipher.doFinal(encryptedData).toString(Charsets.UTF_8) - } + private fun decryptAES(encrypted: String, password: String): String { + val cipherData = context.decodeBase64(encrypted) + val saltData = cipherData.copyOfRange(8, 16) + val (key, iv) = generateKeyAndIV( + keyLength = 32, + ivLength = 16, + iterations = 1, + salt = saltData, + password = password.toByteArray(StandardCharsets.UTF_8), + md = MessageDigest.getInstance("MD5"), + ) + val encryptedData = cipherData.copyOfRange(16, cipherData.size) + val cipher = Cipher.getInstance("AES/CBC/PKCS5Padding") + cipher.init(Cipher.DECRYPT_MODE, key, iv) + return cipher.doFinal(encryptedData).toString(Charsets.UTF_8) + } - @Suppress("SameParameterValue") - private fun generateKeyAndIV( - keyLength: Int, - ivLength: Int, - iterations: Int, - salt: ByteArray, - password: ByteArray, - md: MessageDigest, - ): Pair { - val digestLength = md.digestLength - val requiredLength = (keyLength + ivLength + digestLength - 1) / digestLength * digestLength - val generatedData = ByteArray(requiredLength) - var generatedLength = 0 - md.reset() - while (generatedLength < keyLength + ivLength) { - if (generatedLength > 0) { - md.update(generatedData, generatedLength - digestLength, digestLength) - } - md.update(password) - md.update(salt, 0, 8) - md.digest(generatedData, generatedLength, digestLength) - repeat(iterations - 1) { - md.update(generatedData, generatedLength, digestLength) - md.digest(generatedData, generatedLength, digestLength) - } - generatedLength += digestLength - } + @Suppress("SameParameterValue") + private fun generateKeyAndIV( + keyLength: Int, + ivLength: Int, + iterations: Int, + salt: ByteArray, + password: ByteArray, + md: MessageDigest, + ): Pair { + val digestLength = md.digestLength + val requiredLength = (keyLength + ivLength + digestLength - 1) / digestLength * digestLength + val generatedData = ByteArray(requiredLength) + var generatedLength = 0 + md.reset() + while (generatedLength < keyLength + ivLength) { + if (generatedLength > 0) { + md.update(generatedData, generatedLength - digestLength, digestLength) + } + md.update(password) + md.update(salt, 0, 8) + md.digest(generatedData, generatedLength, digestLength) + repeat(iterations - 1) { + md.update(generatedData, generatedLength, digestLength) + md.digest(generatedData, generatedLength, digestLength) + } + generatedLength += digestLength + } - return SecretKeySpec(generatedData.copyOfRange(0, keyLength), "AES") to IvParameterSpec( - if (ivLength > 0) { - generatedData.copyOfRange(keyLength, keyLength + ivLength) - } else byteArrayOf(), - ) - } -} \ No newline at end of file + return SecretKeySpec(generatedData.copyOfRange(0, keyLength), "AES") to IvParameterSpec( + if (ivLength > 0) { + generatedData.copyOfRange(keyLength, keyLength + ivLength) + } else byteArrayOf(), + ) + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BlogTruyenParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BlogTruyenParser.kt index 42cff91d5..05b7b22d1 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BlogTruyenParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BlogTruyenParser.kt @@ -14,240 +14,241 @@ import org.koitharu.kotatsu.parsers.model.* import org.koitharu.kotatsu.parsers.util.* import java.text.SimpleDateFormat import java.util.* -import kotlin.collections.HashSet @MangaSourceParser("BLOGTRUYEN", "BlogTruyen", "vi") -class BlogTruyenParser(override val context: MangaLoaderContext) : - PagedMangaParser(MangaSource.BLOGTRUYEN, pageSize = 20) { - - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("blogtruyen.vn", null) - - override val sortOrders: Set - get() = EnumSet.of(SortOrder.UPDATED) - - private val mutex = Mutex() - private val dateFormat = SimpleDateFormat("dd/MM/yyyy HH:mm", Locale.US) - private var cacheTags: ArrayMap? = null - - override suspend fun getDetails(manga: Manga): Manga { - val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml() - val descriptionElement = doc.selectFirstOrThrow("div.description") - val statusText = descriptionElement - .selectFirst("p:contains(Trạng thái) > span.color-red") - ?.text() - - val state = when (statusText) { - "Đang tiến hành" -> MangaState.ONGOING - "Đã hoàn thành" -> MangaState.FINISHED - else -> null - } - - val rating = doc.selectFirst("span.total-vote")?.attr("ng-init")?.let { text -> - val like = text.substringAfter("TotalLike=") - .substringBefore(';') - .toIntOrNull() ?: return@let RATING_UNKNOWN - val dislike = text.substringAfter("TotalDisLike=") - .toIntOrNull() ?: return@let RATING_UNKNOWN - - when { - like == 0 && dislike == 0 -> RATING_UNKNOWN - else -> like.toFloat() / (like + dislike) - } - } - - val tagMap = getOrCreateTagMap() - val tags = descriptionElement.select("p > span.category").mapNotNullToSet { - val tagName = it.selectFirst("a")?.text()?.trim() ?: return@mapNotNullToSet null - tagMap[tagName] - } - - return manga.copy( - tags = tags, - author = descriptionElement.selectFirst("p:contains(Tác giả) > a")?.text(), - description = doc.selectFirst(".detail .content")?.html(), - chapters = parseChapterList(doc), - largeCoverUrl = doc.selectLast("div.thumbnail > img")?.imageUrl(), - state = state, - rating = rating ?: RATING_UNKNOWN, - isNsfw = doc.getElementById("warningCategory") != null - ) - } - - private fun parseChapterList(doc: Document): List { - val chapterList = doc.select("#list-chapters > p") - return chapterList.asReversed().mapChapters { index, element -> - val titleElement = element.selectFirst("span.title > a") ?: return@mapChapters null - val name = titleElement.text() - val relativeUrl = titleElement.attrAsRelativeUrl("href") - val id = relativeUrl.substringAfter('/').substringBefore('/') - val uploadDate = dateFormat.tryParse(element.select("span.publishedDate").text()) - MangaChapter( - id = generateUid(id), - name = name, - number = index + 1, - url = relativeUrl, - scanlator = null, - uploadDate = uploadDate, - branch = null, - source = source - ) - } - } - - override suspend fun getListPage( - page: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - return when { - !query.isNullOrEmpty() -> { - val searchUrl = "https://${getDomain()}/timkiem/nangcao/1/0/-1/-1?txt=${query.urlEncoded()}&p=$page" - val searchContent = context.httpGet(searchUrl).parseHtml() - .selectFirst("section.list-manga-bycate > div.list") - parseMangaList(searchContent) - } - - !tags.isNullOrEmpty() -> { - val tag = tags.oneOrThrowIfMany()!! - val categoryAjax = "https://${getDomain()}/ajax/Category/AjaxLoadMangaByCategory?id=${tag.key}&orderBy=5&p=$page" - val listContent = context.httpGet(categoryAjax).parseHtml().selectFirst("div.list") - parseMangaList(listContent) - } - else -> getNormalList(page) - } - } - - private suspend fun getNormalList(page: Int): List { - val pageLink = "https://${getDomain()}/page-$page" - val doc = context.httpGet(pageLink).parseHtml() - val listElements = doc.selectFirstOrThrow("section.list-mainpage.listview") - .select("div.bg-white.storyitem") - - return listElements.mapNotNull { - val linkTag = it.selectFirst("div.fl-l > a") ?: return@mapNotNull null - val relativeUrl = linkTag.attrAsRelativeUrl("href") - val tagMap = getOrCreateTagMap() - val tags = it.select("footer > div.category > a").mapNotNullToSet { a -> - tagMap[a.text()] - } - - Manga( - id = generateUid(relativeUrl), - title = linkTag.attr("title"), - altTitle = null, - description = it.selectFirst("p.al-j.break.line-height-15")?.text(), - url = relativeUrl, - publicUrl = relativeUrl.toAbsoluteUrl(getDomain()), - coverUrl = linkTag.selectLast("img")?.imageUrl().orEmpty(), - source = source, - tags = tags, - isNsfw = false, - rating = RATING_UNKNOWN, - author = null, - state = null, - ) - } - } - - private fun parseMangaList(listElement: Element?): List { - listElement ?: return emptyList() - - return listElement.select("span.tiptip[data-tiptip]").mapNotNull { - val mangaInfo = listElement.getElementById(it.attr("data-tiptip")) ?: return@mapNotNull null - val a = it.selectFirst("a") ?: return@mapNotNull null - val relativeUrl = a.attrAsRelativeUrl("href") - Manga( - id = generateUid(relativeUrl), - title = a.text(), - altTitle = null, - description = mangaInfo.select("div.al-j.fs-12").text(), - url = relativeUrl, - publicUrl = relativeUrl.toAbsoluteUrl(getDomain()), - coverUrl = mangaInfo.selectFirst("div > img.img")?.imageUrl().orEmpty(), - isNsfw = false, - rating = RATING_UNKNOWN, - tags = emptySet(), - author = null, - state = null, - source = source, - ) - } - } - - override suspend fun getPages(chapter: MangaChapter): List { - fun generateImageId(index: Int) = generateUid("${chapter.url}/$index") - - val doc = context.httpGet(chapter.url.toAbsoluteUrl(getDomain())).parseHtml() - val pages = ArrayList() - val referer = chapter.url.toAbsoluteUrl(getDomain()) - doc.select("#content > img").forEach { img -> - pages.add( - MangaPage( - id = generateImageId(pages.size), - url = img.imageUrl(), - referer = referer, - preview = null, - source = source, - ) - ) - } - - // Some chapters use js script to render images - val script = doc.selectLast("#content > script") - if (script != null && script.data().contains("listImageCaption")) { - val imagesStr = script.data().substringBefore(';').substringAfterLast('=').trim() - val imageArr = JSONArray(imagesStr) - for (i in 0 until imageArr.length()) { - val imageUrl = imageArr.getJSONObject(i).getString("url") - pages.add( - MangaPage( - id = generateImageId(pages.size), - url = imageUrl, - referer = referer, - preview = null, - source = source - ) - ) - } - } - - return pages - } - - override suspend fun getTags(): Set { - val map = getOrCreateTagMap() - val tags = HashSet(map.size) - for (entry in map) { - tags.add(entry.value) - } - - return tags - } - - - private suspend fun getOrCreateTagMap(): ArrayMap = mutex.withLock { - cacheTags?.let { return@withLock it } - val doc = context.httpGet("/timkiem/nangcao".toAbsoluteUrl(getDomain())).parseHtml() - val tagItems = doc.select("li[data-id]") - val tagMap = ArrayMap(tagItems.size) - for (tag in tagItems) { - val title = tag.text().trim() - tagMap[tag.text().trim()] = MangaTag( - title = title, - key = tag.attr("data-id"), - source = source - ) - } - - cacheTags = tagMap - tagMap - } - - private fun Element.imageUrl(): String { - return attrAsAbsoluteUrlOrNull("src") - ?: attrAsAbsoluteUrlOrNull("data-cfsrc") - ?: "" - } +class BlogTruyenParser(context: MangaLoaderContext) : + PagedMangaParser(context, MangaSource.BLOGTRUYEN, pageSize = 20) { + + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("blogtruyen.vn", null) + + override val sortOrders: Set + get() = EnumSet.of(SortOrder.UPDATED) + + private val mutex = Mutex() + private val dateFormat = SimpleDateFormat("dd/MM/yyyy HH:mm", Locale.US) + private var cacheTags: ArrayMap? = null + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val descriptionElement = doc.selectFirstOrThrow("div.description") + val statusText = descriptionElement + .selectFirst("p:contains(Trạng thái) > span.color-red") + ?.text() + + val state = when (statusText) { + "Đang tiến hành" -> MangaState.ONGOING + "Đã hoàn thành" -> MangaState.FINISHED + else -> null + } + + val rating = doc.selectFirst("span.total-vote")?.attr("ng-init")?.let { text -> + val like = text.substringAfter("TotalLike=") + .substringBefore(';') + .toIntOrNull() ?: return@let RATING_UNKNOWN + val dislike = text.substringAfter("TotalDisLike=") + .toIntOrNull() ?: return@let RATING_UNKNOWN + + when { + like == 0 && dislike == 0 -> RATING_UNKNOWN + else -> like.toFloat() / (like + dislike) + } + } + + val tagMap = getOrCreateTagMap() + val tags = descriptionElement.select("p > span.category").mapNotNullToSet { + val tagName = it.selectFirst("a")?.text()?.trim() ?: return@mapNotNullToSet null + tagMap[tagName] + } + + return manga.copy( + tags = tags, + author = descriptionElement.selectFirst("p:contains(Tác giả) > a")?.text(), + description = doc.selectFirst(".detail .content")?.html(), + chapters = parseChapterList(doc), + largeCoverUrl = doc.selectLast("div.thumbnail > img")?.imageUrl(), + state = state, + rating = rating ?: RATING_UNKNOWN, + isNsfw = doc.getElementById("warningCategory") != null, + ) + } + + private fun parseChapterList(doc: Document): List { + val chapterList = doc.select("#list-chapters > p") + return chapterList.asReversed().mapChapters { index, element -> + val titleElement = element.selectFirst("span.title > a") ?: return@mapChapters null + val name = titleElement.text() + val relativeUrl = titleElement.attrAsRelativeUrl("href") + val id = relativeUrl.substringAfter('/').substringBefore('/') + val uploadDate = dateFormat.tryParse(element.select("span.publishedDate").text()) + MangaChapter( + id = generateUid(id), + name = name, + number = index + 1, + url = relativeUrl, + scanlator = null, + uploadDate = uploadDate, + branch = null, + source = source, + ) + } + } + + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + return when { + !query.isNullOrEmpty() -> { + val searchUrl = "https://${domain}/timkiem/nangcao/1/0/-1/-1?txt=${query.urlEncoded()}&p=$page" + val searchContent = webClient.httpGet(searchUrl).parseHtml() + .selectFirst("section.list-manga-bycate > div.list") + parseMangaList(searchContent) + } + + !tags.isNullOrEmpty() -> { + val tag = tags.oneOrThrowIfMany()!! + val categoryAjax = + "https://${domain}/ajax/Category/AjaxLoadMangaByCategory?id=${tag.key}&orderBy=5&p=$page" + val listContent = webClient.httpGet(categoryAjax).parseHtml().selectFirst("div.list") + parseMangaList(listContent) + } + + else -> getNormalList(page) + } + } + + private suspend fun getNormalList(page: Int): List { + val pageLink = "https://${domain}/page-$page" + val doc = webClient.httpGet(pageLink).parseHtml() + val listElements = doc.selectFirstOrThrow("section.list-mainpage.listview") + .select("div.bg-white.storyitem") + + return listElements.mapNotNull { + val linkTag = it.selectFirst("div.fl-l > a") ?: return@mapNotNull null + val relativeUrl = linkTag.attrAsRelativeUrl("href") + val tagMap = getOrCreateTagMap() + val tags = it.select("footer > div.category > a").mapNotNullToSet { a -> + tagMap[a.text()] + } + + Manga( + id = generateUid(relativeUrl), + title = linkTag.attr("title"), + altTitle = null, + description = it.selectFirst("p.al-j.break.line-height-15")?.text(), + url = relativeUrl, + publicUrl = relativeUrl.toAbsoluteUrl(domain), + coverUrl = linkTag.selectLast("img")?.imageUrl().orEmpty(), + source = source, + tags = tags, + isNsfw = false, + rating = RATING_UNKNOWN, + author = null, + state = null, + ) + } + } + + private fun parseMangaList(listElement: Element?): List { + listElement ?: return emptyList() + + return listElement.select("span.tiptip[data-tiptip]").mapNotNull { + val mangaInfo = listElement.getElementById(it.attr("data-tiptip")) ?: return@mapNotNull null + val a = it.selectFirst("a") ?: return@mapNotNull null + val relativeUrl = a.attrAsRelativeUrl("href") + Manga( + id = generateUid(relativeUrl), + title = a.text(), + altTitle = null, + description = mangaInfo.select("div.al-j.fs-12").text(), + url = relativeUrl, + publicUrl = relativeUrl.toAbsoluteUrl(domain), + coverUrl = mangaInfo.selectFirst("div > img.img")?.imageUrl().orEmpty(), + isNsfw = false, + rating = RATING_UNKNOWN, + tags = emptySet(), + author = null, + state = null, + source = source, + ) + } + } + + override suspend fun getPages(chapter: MangaChapter): List { + fun generateImageId(index: Int) = generateUid("${chapter.url}/$index") + + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + val pages = ArrayList() + val referer = chapter.url.toAbsoluteUrl(domain) + doc.select("#content > img").forEach { img -> + pages.add( + MangaPage( + id = generateImageId(pages.size), + url = img.imageUrl(), + referer = referer, + preview = null, + source = source, + ), + ) + } + + // Some chapters use js script to render images + val script = doc.selectLast("#content > script") + if (script != null && script.data().contains("listImageCaption")) { + val imagesStr = script.data().substringBefore(';').substringAfterLast('=').trim() + val imageArr = JSONArray(imagesStr) + for (i in 0 until imageArr.length()) { + val imageUrl = imageArr.getJSONObject(i).getString("url") + pages.add( + MangaPage( + id = generateImageId(pages.size), + url = imageUrl, + referer = referer, + preview = null, + source = source, + ), + ) + } + } + + return pages + } + + override suspend fun getTags(): Set { + val map = getOrCreateTagMap() + val tags = HashSet(map.size) + for (entry in map) { + tags.add(entry.value) + } + + return tags + } + + + private suspend fun getOrCreateTagMap(): ArrayMap = mutex.withLock { + cacheTags?.let { return@withLock it } + val doc = webClient.httpGet("/timkiem/nangcao".toAbsoluteUrl(domain)).parseHtml() + val tagItems = doc.select("li[data-id]") + val tagMap = ArrayMap(tagItems.size) + for (tag in tagItems) { + val title = tag.text().trim() + tagMap[tag.text().trim()] = MangaTag( + title = title, + key = tag.attr("data-id"), + source = source, + ) + } + + cacheTags = tagMap + tagMap + } + + private fun Element.imageUrl(): String { + return attrAsAbsoluteUrlOrNull("src") + ?: attrAsAbsoluteUrlOrNull("data-cfsrc") + ?: "" + } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/CloneMangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/CloneMangaParser.kt index a4d0f7978..9bdea84f4 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/CloneMangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/CloneMangaParser.kt @@ -1,104 +1,107 @@ package org.koitharu.kotatsu.parsers.site -import org.koitharu.kotatsu.parsers.* +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.PagedMangaParser import org.koitharu.kotatsu.parsers.config.ConfigKey import org.koitharu.kotatsu.parsers.model.* import org.koitharu.kotatsu.parsers.util.* import java.util.* @MangaSourceParser("CLONEMANGA", "CloneManga", "en") -internal class CloneMangaParser(override val context: MangaLoaderContext) : PagedMangaParser( - MangaSource.CLONEMANGA, - pageSize = 1, +internal class CloneMangaParser(context: MangaLoaderContext) : PagedMangaParser( + context, + MangaSource.CLONEMANGA, + pageSize = 1, ) { - override val sortOrders: Set = Collections.singleton( - SortOrder.POPULARITY, - ) + override val sortOrders: Set = Collections.singleton( + SortOrder.POPULARITY, + ) - override val configKeyDomain = ConfigKey.Domain("manga.clone-army.org", null) + override val configKeyDomain = ConfigKey.Domain("manga.clone-army.org", null) - override fun getFaviconUrl(): String { - return "https://pbs.twimg.com/profile_images/458758466346029056/Ys93EANp_400x400.png" - } + override fun getFaviconUrl(): String { + return "https://pbs.twimg.com/profile_images/458758466346029056/Ys93EANp_400x400.png" + } - override suspend fun getListPage( - page: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - if (query != null || page > 1) { - return emptyList() - } - val link = "https://${getDomain()}/viewer_landing.php" - val doc = context.httpGet(link).parseHtml() - val mangas = doc.getElementsByClass("comicPreviewContainer") - return mangas.mapNotNull { item -> - val attr = item.getElementsByClass("comicPreview").attr("style") - val href = item.selectFirst("a")?.attrAsAbsoluteUrl("href") ?: return@mapNotNull null - val cover = attr.substring(attr.indexOf("site/themes"), attr.indexOf(")")) - Manga( - id = generateUid(href), - title = item.selectFirst("h3")?.text() ?: return@mapNotNull null, - coverUrl = "https://${getDomain()}/$cover", - altTitle = null, - author = "Dan Kim", - rating = RATING_UNKNOWN, - url = href, - isNsfw = false, - tags = emptySet(), - state = null, - publicUrl = href.toAbsoluteUrl(getDomain()), - source = source, - ) - } - } + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + if (query != null || page > 1) { + return emptyList() + } + val link = "https://${domain}/viewer_landing.php" + val doc = webClient.httpGet(link).parseHtml() + val mangas = doc.getElementsByClass("comicPreviewContainer") + return mangas.mapNotNull { item -> + val attr = item.getElementsByClass("comicPreview").attr("style") + val href = item.selectFirst("a")?.attrAsAbsoluteUrl("href") ?: return@mapNotNull null + val cover = attr.substring(attr.indexOf("site/themes"), attr.indexOf(")")) + Manga( + id = generateUid(href), + title = item.selectFirst("h3")?.text() ?: return@mapNotNull null, + coverUrl = "https://${domain}/$cover", + altTitle = null, + author = "Dan Kim", + rating = RATING_UNKNOWN, + url = href, + isNsfw = false, + tags = emptySet(), + state = null, + publicUrl = href.toAbsoluteUrl(domain), + source = source, + ) + } + } - override suspend fun getDetails(manga: Manga): Manga { - val doc = context.httpGet(manga.publicUrl).parseHtml() - val series = doc.location() - val numChapters = Regex( - pattern = "&page=(.*)&lang=", - ).findAll( - input = doc.getElementsByTag("script")[3].toString(), - ) - .elementAt(3).destructured.component1() - .toInt() - val chapters = ArrayList() - for (i in 0..numChapters) { - val chapter = MangaChapter( - id = generateUid("$series&page=$i"), - name = "Chapter ${i + 1}", - number = i + 1, - url = "$series&page=$i", - scanlator = null, - branch = null, - uploadDate = 0L, - source = MangaSource.DUMMY, - ) - chapters.add(chapter) - } - return manga.copy(chapters = chapters) - } + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.publicUrl).parseHtml() + val series = doc.location() + val numChapters = Regex( + pattern = "&page=(.*)&lang=", + ).findAll( + input = doc.getElementsByTag("script")[3].toString(), + ) + .elementAt(3).destructured.component1() + .toInt() + val chapters = ArrayList() + for (i in 0..numChapters) { + val chapter = MangaChapter( + id = generateUid("$series&page=$i"), + name = "Chapter ${i + 1}", + number = i + 1, + url = "$series&page=$i", + scanlator = null, + branch = null, + uploadDate = 0L, + source = MangaSource.DUMMY, + ) + chapters.add(chapter) + } + return manga.copy(chapters = chapters) + } - override suspend fun getPages(chapter: MangaChapter): List { - val doc = context.httpGet(chapter.url.toAbsoluteUrl(getDomain())).parseHtml() - val imgUrl = doc.getElementsByClass("subsectionContainer")[0] - .selectFirst("img") - ?.attrAsAbsoluteUrlOrNull("src") ?: doc.parseFailed("Something broken") - return listOf( - MangaPage( - id = generateUid(imgUrl), - url = imgUrl, - referer = imgUrl, - preview = null, - source = MangaSource.DUMMY, - ), - ) - } + override suspend fun getPages(chapter: MangaChapter): List { + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + val imgUrl = doc.getElementsByClass("subsectionContainer")[0] + .selectFirst("img") + ?.attrAsAbsoluteUrlOrNull("src") ?: doc.parseFailed("Something broken") + return listOf( + MangaPage( + id = generateUid(imgUrl), + url = imgUrl, + referer = imgUrl, + preview = null, + source = MangaSource.DUMMY, + ), + ) + } - override suspend fun getTags(): Set { - return emptySet() - } -} \ No newline at end of file + override suspend fun getTags(): Set { + return emptySet() + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ComickFunParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ComickFunParser.kt index ab7d471db..fdb94512e 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ComickFunParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ComickFunParser.kt @@ -22,195 +22,195 @@ private const val PAGE_SIZE = 20 private const val CHAPTERS_LIMIT = 99999 @MangaSourceParser("COMICK_FUN", "ComicK") -internal class ComickFunParser(override val context: MangaLoaderContext) : MangaParser(MangaSource.COMICK_FUN) { +internal class ComickFunParser(context: MangaLoaderContext) : MangaParser(context, MangaSource.COMICK_FUN) { - override val configKeyDomain = ConfigKey.Domain("comick.app", null) + override val configKeyDomain = ConfigKey.Domain("comick.app", null) - override val sortOrders: Set = EnumSet.of( - SortOrder.POPULARITY, - SortOrder.UPDATED, - SortOrder.RATING, - ) + override val sortOrders: Set = EnumSet.of( + SortOrder.POPULARITY, + SortOrder.UPDATED, + SortOrder.RATING, + ) - @Volatile - private var cachedTags: SparseArrayCompat? = null + @Volatile + private var cachedTags: SparseArrayCompat? = null - override suspend fun getList( - offset: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - val domain = getDomain() - val url = buildString { - append("https://api.") - append(domain) - append("/search?tachiyomi=true") - if (!query.isNullOrEmpty()) { - if (offset > 0) { - return emptyList() - } - append("&q=") - append(query.urlEncoded()) - } else { - append("&limit=") - append(PAGE_SIZE) - append("&page=") - append((offset / PAGE_SIZE) + 1) - if (!tags.isNullOrEmpty()) { - append("&genres=") - appendAll(tags, "&genres=", MangaTag::key) - } - append("&sort=") // view, uploaded, rating, follow, user_follow_count - append( - when (sortOrder) { - SortOrder.POPULARITY -> "view" - SortOrder.RATING -> "rating" - else -> "uploaded" - }, - ) - } - } - val ja = context.httpGet(url).parseJsonArray() - val tagsMap = cachedTags ?: loadTags() - return ja.mapJSON { jo -> - val slug = jo.getString("slug") - Manga( - id = generateUid(slug), - title = jo.getString("title"), - altTitle = null, - url = slug, - publicUrl = "https://$domain/comic/$slug", - rating = jo.getDoubleOrDefault("rating", -10.0).toFloat() / 10f, - isNsfw = false, - coverUrl = jo.getString("cover_url"), - largeCoverUrl = null, - description = jo.getStringOrNull("desc"), - tags = jo.selectGenres("genres", tagsMap), - state = runCatching { - if (jo.getBoolean("translation_completed")) { - MangaState.FINISHED - } else { - MangaState.ONGOING - } - }.getOrNull(), - author = null, - source = source, - ) - } - } + override suspend fun getList( + offset: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + val domain = domain + val url = buildString { + append("https://api.") + append(domain) + append("/search?tachiyomi=true") + if (!query.isNullOrEmpty()) { + if (offset > 0) { + return emptyList() + } + append("&q=") + append(query.urlEncoded()) + } else { + append("&limit=") + append(PAGE_SIZE) + append("&page=") + append((offset / PAGE_SIZE) + 1) + if (!tags.isNullOrEmpty()) { + append("&genres=") + appendAll(tags, "&genres=", MangaTag::key) + } + append("&sort=") // view, uploaded, rating, follow, user_follow_count + append( + when (sortOrder) { + SortOrder.POPULARITY -> "view" + SortOrder.RATING -> "rating" + else -> "uploaded" + }, + ) + } + } + val ja = webClient.httpGet(url).parseJsonArray() + val tagsMap = cachedTags ?: loadTags() + return ja.mapJSON { jo -> + val slug = jo.getString("slug") + Manga( + id = generateUid(slug), + title = jo.getString("title"), + altTitle = null, + url = slug, + publicUrl = "https://$domain/comic/$slug", + rating = jo.getDoubleOrDefault("rating", -10.0).toFloat() / 10f, + isNsfw = false, + coverUrl = jo.getString("cover_url"), + largeCoverUrl = null, + description = jo.getStringOrNull("desc"), + tags = jo.selectGenres("genres", tagsMap), + state = runCatching { + if (jo.getBoolean("translation_completed")) { + MangaState.FINISHED + } else { + MangaState.ONGOING + } + }.getOrNull(), + author = null, + source = source, + ) + } + } - override suspend fun getDetails(manga: Manga): Manga { - val domain = getDomain() - val url = "https://api.$domain/comic/${manga.url}?tachiyomi=true" - val jo = context.httpGet(url).parseJson() - val comic = jo.getJSONObject("comic") - return manga.copy( - title = comic.getString("title"), - altTitle = null, // TODO - isNsfw = jo.getBoolean("matureContent") || comic.getBoolean("hentai"), - description = comic.getStringOrNull("parsed") ?: comic.getString("desc"), - tags = manga.tags + jo.getJSONArray("genres").mapJSONToSet { - MangaTag( - title = it.getString("name"), - key = it.getString("slug"), - source = source, - ) - }, - author = jo.getJSONArray("artists").optJSONObject(0)?.getString("name"), - chapters = getChapters(comic.getLong("id")), - ) - } + override suspend fun getDetails(manga: Manga): Manga { + val domain = domain + val url = "https://api.$domain/comic/${manga.url}?tachiyomi=true" + val jo = webClient.httpGet(url).parseJson() + val comic = jo.getJSONObject("comic") + return manga.copy( + title = comic.getString("title"), + altTitle = null, // TODO + isNsfw = jo.getBoolean("matureContent") || comic.getBoolean("hentai"), + description = comic.getStringOrNull("parsed") ?: comic.getString("desc"), + tags = manga.tags + jo.getJSONArray("genres").mapJSONToSet { + MangaTag( + title = it.getString("name"), + key = it.getString("slug"), + source = source, + ) + }, + author = jo.getJSONArray("artists").optJSONObject(0)?.getString("name"), + chapters = getChapters(comic.getLong("id")), + ) + } - override suspend fun getPages(chapter: MangaChapter): List { - val jo = context.httpGet( - "https://api.${getDomain()}/chapter/${chapter.url}?tachiyomi=true", - ).parseJson().getJSONObject("chapter") - val referer = "https://${getDomain()}/" - return jo.getJSONArray("images").mapJSON { - val url = it.getString("url") - MangaPage( - id = generateUid(url), - url = url, - referer = referer, - preview = null, - source = source, - ) - } - } + override suspend fun getPages(chapter: MangaChapter): List { + val jo = webClient.httpGet( + "https://api.${domain}/chapter/${chapter.url}?tachiyomi=true", + ).parseJson().getJSONObject("chapter") + val referer = "https://${domain}/" + return jo.getJSONArray("images").mapJSON { + val url = it.getString("url") + MangaPage( + id = generateUid(url), + url = url, + referer = referer, + preview = null, + source = source, + ) + } + } - override suspend fun getTags(): Set { - val sparseArray = cachedTags ?: loadTags() - val set = ArraySet(sparseArray.size()) - for (i in 0 until sparseArray.size()) { - set.add(sparseArray.valueAt(i)) - } - return set - } + override suspend fun getTags(): Set { + val sparseArray = cachedTags ?: loadTags() + val set = ArraySet(sparseArray.size()) + for (i in 0 until sparseArray.size()) { + set.add(sparseArray.valueAt(i)) + } + return set + } - private suspend fun loadTags(): SparseArrayCompat { - val ja = context.httpGet("https://api.${getDomain()}/genre").parseJsonArray() - val tags = SparseArrayCompat(ja.length()) - for (jo in ja.JSONIterator()) { - tags.append( - jo.getInt("id"), - MangaTag( - title = jo.getString("name"), - key = jo.getString("slug"), - source = source, - ), - ) - } - cachedTags = tags - return tags - } + private suspend fun loadTags(): SparseArrayCompat { + val ja = webClient.httpGet("https://api.${domain}/genre").parseJsonArray() + val tags = SparseArrayCompat(ja.length()) + for (jo in ja.JSONIterator()) { + tags.append( + jo.getInt("id"), + MangaTag( + title = jo.getString("name"), + key = jo.getString("slug"), + source = source, + ), + ) + } + cachedTags = tags + return tags + } - private suspend fun getChapters(id: Long): List { - val ja = context.httpGet( - url = "https://api.${getDomain()}/comic/$id/chapter?tachiyomi=true&limit=$CHAPTERS_LIMIT", - ).parseJson().getJSONArray("chapters") - val dateFormat = SimpleDateFormat("yyyy-MM-dd") - val counters = HashMap() - return ja.mapReversed { jo -> - val locale = Locale.forLanguageTag(jo.getString("lang")) - var number = counters[locale] ?: 0 - number++ - counters[locale] = number - MangaChapter( - id = generateUid(jo.getLong("id")), - name = buildString { - jo.getStringOrNull("vol")?.let { append("Vol ").append(it).append(' ') } - jo.getStringOrNull("chap")?.let { append("Chap ").append(it) } - jo.getStringOrNull("title")?.let { append(": ").append(it) } - }, - number = number, - url = jo.getString("hid"), - scanlator = jo.optJSONArray("group_name")?.optString(0), - uploadDate = dateFormat.tryParse(jo.getString("created_at").substringBefore('T')), - branch = locale.getDisplayName(locale).toTitleCase(locale), - source = source, - ) - } - } + private suspend fun getChapters(id: Long): List { + val ja = webClient.httpGet( + url = "https://api.${domain}/comic/$id/chapter?tachiyomi=true&limit=$CHAPTERS_LIMIT", + ).parseJson().getJSONArray("chapters") + val dateFormat = SimpleDateFormat("yyyy-MM-dd") + val counters = HashMap() + return ja.mapReversed { jo -> + val locale = Locale.forLanguageTag(jo.getString("lang")) + var number = counters[locale] ?: 0 + number++ + counters[locale] = number + MangaChapter( + id = generateUid(jo.getLong("id")), + name = buildString { + jo.getStringOrNull("vol")?.let { append("Vol ").append(it).append(' ') } + jo.getStringOrNull("chap")?.let { append("Chap ").append(it) } + jo.getStringOrNull("title")?.let { append(": ").append(it) } + }, + number = number, + url = jo.getString("hid"), + scanlator = jo.optJSONArray("group_name")?.optString(0), + uploadDate = dateFormat.tryParse(jo.getString("created_at").substringBefore('T')), + branch = locale.getDisplayName(locale).toTitleCase(locale), + source = source, + ) + } + } - private inline fun JSONArray.mapReversed(block: (JSONObject) -> R): List { - val len = length() - val destination = ArrayList(len) - for (i in (0 until len).reversed()) { - val jo = getJSONObject(i) - destination.add(block(jo)) - } - return destination - } + private inline fun JSONArray.mapReversed(block: (JSONObject) -> R): List { + val len = length() + val destination = ArrayList(len) + for (i in (0 until len).reversed()) { + val jo = getJSONObject(i) + destination.add(block(jo)) + } + return destination + } - private fun JSONObject.selectGenres(name: String, tags: SparseArrayCompat): Set { - val array = optJSONArray(name) ?: return emptySet() - val res = ArraySet(array.length()) - for (i in 0 until array.length()) { - val id = array.getInt(i) - val tag = tags.get(id) ?: continue - res.add(tag) - } - return res - } -} \ No newline at end of file + private fun JSONObject.selectGenres(name: String, tags: SparseArrayCompat): Set { + val array = optJSONArray(name) ?: return emptySet() + val res = ArraySet(array.length()) + for (i in 0 until array.length()) { + val id = array.getInt(i) + val tag = tags.get(id) ?: continue + res.add(tag) + } + return res + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/DesuMeParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/DesuMeParser.kt index 3efe87163..3b76baed8 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/DesuMeParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/DesuMeParser.kt @@ -13,149 +13,149 @@ import org.koitharu.kotatsu.parsers.util.json.mapJSONToSet import java.util.* @MangaSourceParser("DESUME", "Desu.me", "ru") -internal class DesuMeParser(override val context: MangaLoaderContext) : PagedMangaParser(MangaSource.DESUME, 20) { +internal class DesuMeParser(context: MangaLoaderContext) : PagedMangaParser(context, MangaSource.DESUME, 20) { - override val configKeyDomain = ConfigKey.Domain("desu.me", null) + override val configKeyDomain = ConfigKey.Domain("desu.me", null) - override val sortOrders: Set = EnumSet.of( - SortOrder.UPDATED, - SortOrder.POPULARITY, - SortOrder.NEWEST, - SortOrder.ALPHABETICAL, - ) + override val sortOrders: Set = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.NEWEST, + SortOrder.ALPHABETICAL, + ) - override suspend fun getListPage( - page: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - if (query != null && page != searchPaginator.firstPage) { - return emptyList() - } - val domain = getDomain() - val url = buildString { - append("https://") - append(domain) - append("/manga/api/?limit=20&order=") - append(getSortKey(sortOrder)) - append("&page=") - append(page) - if (!tags.isNullOrEmpty()) { - append("&genres=") - appendAll(tags, ",") { it.key } - } - if (query != null) { - append("&search=") - append(query) - } - } - val json = context.httpGet(url).parseJson().getJSONArray("response") - ?: throw ParseException("Invalid response", url) - val total = json.length() - val list = ArrayList(total) - for (i in 0 until total) { - val jo = json.getJSONObject(i) - val cover = jo.getJSONObject("image") - val id = jo.getLong("id") - list += Manga( - url = "/manga/api/$id", - publicUrl = jo.getString("url"), - source = MangaSource.DESUME, - title = jo.getString("russian"), - altTitle = jo.getString("name"), - coverUrl = cover.getString("preview"), - largeCoverUrl = cover.getString("original"), - state = when { - jo.getInt("ongoing") == 1 -> MangaState.ONGOING - else -> null - }, - rating = jo.getDouble("score").toFloat().coerceIn(0f, 1f), - id = generateUid(id), - isNsfw = false, - tags = emptySet(), - author = null, - description = jo.getString("description"), - ) - } - return list - } + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + if (query != null && page != searchPaginator.firstPage) { + return emptyList() + } + val domain = domain + val url = buildString { + append("https://") + append(domain) + append("/manga/api/?limit=20&order=") + append(getSortKey(sortOrder)) + append("&page=") + append(page) + if (!tags.isNullOrEmpty()) { + append("&genres=") + appendAll(tags, ",") { it.key } + } + if (query != null) { + append("&search=") + append(query) + } + } + val json = webClient.httpGet(url).parseJson().getJSONArray("response") + ?: throw ParseException("Invalid response", url) + val total = json.length() + val list = ArrayList(total) + for (i in 0 until total) { + val jo = json.getJSONObject(i) + val cover = jo.getJSONObject("image") + val id = jo.getLong("id") + list += Manga( + url = "/manga/api/$id", + publicUrl = jo.getString("url"), + source = MangaSource.DESUME, + title = jo.getString("russian"), + altTitle = jo.getString("name"), + coverUrl = cover.getString("preview"), + largeCoverUrl = cover.getString("original"), + state = when { + jo.getInt("ongoing") == 1 -> MangaState.ONGOING + else -> null + }, + rating = jo.getDouble("score").toFloat().coerceIn(0f, 1f), + id = generateUid(id), + isNsfw = false, + tags = emptySet(), + author = null, + description = jo.getString("description"), + ) + } + return list + } - override suspend fun getDetails(manga: Manga): Manga { - val url = manga.url.toAbsoluteUrl(getDomain()) - val json = context.httpGet(url).parseJson().getJSONObject("response") - ?: throw ParseException("Invalid response", url) - val baseChapterUrl = manga.url + "/chapter/" - val chaptersList = json.getJSONObject("chapters").getJSONArray("list") - val totalChapters = chaptersList.length() - return manga.copy( - tags = json.getJSONArray("genres").mapJSONToSet { - MangaTag( - key = it.getString("text"), - title = it.getString("russian").toTitleCase(), - source = manga.source, - ) - }, - publicUrl = json.getString("url"), - description = json.getString("description"), - chapters = chaptersList.mapJSONIndexed { i, it -> - val chid = it.getLong("id") - val volChap = "Том " + it.optString("vol", "0") + ". " + "Глава " + it.optString("ch", "0") - val title = it.optString("title", "null").takeUnless { it == "null" } - MangaChapter( - id = generateUid(chid), - source = manga.source, - url = "$baseChapterUrl$chid", - uploadDate = it.getLong("date") * 1000, - name = if (title.isNullOrEmpty()) volChap else "$volChap: $title", - number = totalChapters - i, - scanlator = null, - branch = null, - ) - }.reversed(), - ) - } + override suspend fun getDetails(manga: Manga): Manga { + val url = manga.url.toAbsoluteUrl(domain) + val json = webClient.httpGet(url).parseJson().getJSONObject("response") + ?: throw ParseException("Invalid response", url) + val baseChapterUrl = manga.url + "/chapter/" + val chaptersList = json.getJSONObject("chapters").getJSONArray("list") + val totalChapters = chaptersList.length() + return manga.copy( + tags = json.getJSONArray("genres").mapJSONToSet { + MangaTag( + key = it.getString("text"), + title = it.getString("russian").toTitleCase(), + source = manga.source, + ) + }, + publicUrl = json.getString("url"), + description = json.getString("description"), + chapters = chaptersList.mapJSONIndexed { i, it -> + val chid = it.getLong("id") + val volChap = "Том " + it.optString("vol", "0") + ". " + "Глава " + it.optString("ch", "0") + val title = it.optString("title", "null").takeUnless { it == "null" } + MangaChapter( + id = generateUid(chid), + source = manga.source, + url = "$baseChapterUrl$chid", + uploadDate = it.getLong("date") * 1000, + name = if (title.isNullOrEmpty()) volChap else "$volChap: $title", + number = totalChapters - i, + scanlator = null, + branch = null, + ) + }.reversed(), + ) + } - override suspend fun getPages(chapter: MangaChapter): List { - val fullUrl = chapter.url.toAbsoluteUrl(getDomain()) - val json = context.httpGet(fullUrl) - .parseJson() - .getJSONObject("response") ?: throw ParseException("Invalid response", fullUrl) - return json.getJSONObject("pages").getJSONArray("list").mapJSON { jo -> - MangaPage( - id = generateUid(jo.getLong("id")), - referer = fullUrl, - preview = null, - source = chapter.source, - url = jo.getString("img"), - ) - } - } + override suspend fun getPages(chapter: MangaChapter): List { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val json = webClient.httpGet(fullUrl) + .parseJson() + .getJSONObject("response") ?: throw ParseException("Invalid response", fullUrl) + return json.getJSONObject("pages").getJSONArray("list").mapJSON { jo -> + MangaPage( + id = generateUid(jo.getLong("id")), + referer = fullUrl, + preview = null, + source = chapter.source, + url = jo.getString("img"), + ) + } + } - override suspend fun getTags(): Set { - val doc = context.httpGet("https://${getDomain()}/manga/").parseHtml() - val root = doc.body().requireElementById("animeFilter") - .selectFirstOrThrow(".catalog-genres") - return root.select("li").mapToSet { - val input = it.selectFirstOrThrow("input") - MangaTag( - source = source, - key = input.attr("data-genre-slug").ifEmpty { - it.parseFailed("data-genre-slug is empty") - }, - title = input.attr("data-genre-name").toTitleCase().ifEmpty { - it.parseFailed("data-genre-name is empty") - }, - ) - } - } + override suspend fun getTags(): Set { + val doc = webClient.httpGet("https://${domain}/manga/").parseHtml() + val root = doc.body().requireElementById("animeFilter") + .selectFirstOrThrow(".catalog-genres") + return root.select("li").mapToSet { + val input = it.selectFirstOrThrow("input") + MangaTag( + source = source, + key = input.attr("data-genre-slug").ifEmpty { + it.parseFailed("data-genre-slug is empty") + }, + title = input.attr("data-genre-name").toTitleCase().ifEmpty { + it.parseFailed("data-genre-name is empty") + }, + ) + } + } - private fun getSortKey(sortOrder: SortOrder) = - when (sortOrder) { - SortOrder.ALPHABETICAL -> "name" - SortOrder.POPULARITY -> "popular" - SortOrder.UPDATED -> "updated" - SortOrder.NEWEST -> "id" - else -> "updated" - } -} \ No newline at end of file + private fun getSortKey(sortOrder: SortOrder) = + when (sortOrder) { + SortOrder.ALPHABETICAL -> "name" + SortOrder.POPULARITY -> "popular" + SortOrder.UPDATED -> "updated" + SortOrder.NEWEST -> "id" + else -> "updated" + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ExHentaiParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ExHentaiParser.kt index 6432667fa..86be79389 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ExHentaiParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ExHentaiParser.kt @@ -20,289 +20,289 @@ private const val DOMAIN_AUTHORIZED = "exhentai.org" @MangaSourceParser("EXHENTAI", "ExHentai") internal class ExHentaiParser( - override val context: MangaLoaderContext, -) : PagedMangaParser(MangaSource.EXHENTAI, pageSize = 25), MangaParserAuthProvider { + context: MangaLoaderContext, +) : PagedMangaParser(context, MangaSource.EXHENTAI, pageSize = 25), MangaParserAuthProvider { - override val sortOrders: Set = Collections.singleton( - SortOrder.NEWEST, - ) + override val sortOrders: Set = Collections.singleton( + SortOrder.NEWEST, + ) - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain(if (isAuthorized) DOMAIN_AUTHORIZED else DOMAIN_UNAUTHORIZED, null) + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain(if (isAuthorized) DOMAIN_AUTHORIZED else DOMAIN_UNAUTHORIZED, null) - override val authUrl: String - get() = "https://${getDomain()}/bounce_login.php" + override val authUrl: String + get() = "https://${domain}/bounce_login.php" - private val ratingPattern = Regex("-?[0-9]+px") - private val authCookies = arrayOf("ipb_member_id", "ipb_pass_hash") - private var updateDm = false - private val nextPages = SparseArrayCompat() - private val suspiciousContentKey = ConfigKey.ShowSuspiciousContent(true) + private val ratingPattern = Regex("-?[0-9]+px") + private val authCookies = arrayOf("ipb_member_id", "ipb_pass_hash") + private var updateDm = false + private val nextPages = SparseArrayCompat() + private val suspiciousContentKey = ConfigKey.ShowSuspiciousContent(true) - override val isAuthorized: Boolean - get() { - val authorized = isAuthorized(DOMAIN_UNAUTHORIZED) - if (authorized) { - if (!isAuthorized(DOMAIN_AUTHORIZED)) { - context.cookieJar.copyCookies( - DOMAIN_UNAUTHORIZED, - DOMAIN_AUTHORIZED, - authCookies, - ) - context.cookieJar.insertCookies(DOMAIN_AUTHORIZED, "yay=louder") - } - return true - } - return false - } + override val isAuthorized: Boolean + get() { + val authorized = isAuthorized(DOMAIN_UNAUTHORIZED) + if (authorized) { + if (!isAuthorized(DOMAIN_AUTHORIZED)) { + context.cookieJar.copyCookies( + DOMAIN_UNAUTHORIZED, + DOMAIN_AUTHORIZED, + authCookies, + ) + context.cookieJar.insertCookies(DOMAIN_AUTHORIZED, "yay=louder") + } + return true + } + return false + } - init { - context.cookieJar.insertCookies(DOMAIN_AUTHORIZED, "nw=1", "sl=dm_2") - context.cookieJar.insertCookies(DOMAIN_UNAUTHORIZED, "nw=1", "sl=dm_2") - paginator.firstPage = 0 - } + init { + context.cookieJar.insertCookies(DOMAIN_AUTHORIZED, "nw=1", "sl=dm_2") + context.cookieJar.insertCookies(DOMAIN_UNAUTHORIZED, "nw=1", "sl=dm_2") + paginator.firstPage = 0 + } - override suspend fun getListPage( - page: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - var search = query?.urlEncoded().orEmpty() - val next = nextPages.get(page, 0L) - if (page > 0 && next == 0L) { - assert(false) { "Page timestamp not found" } - return emptyList() - } - val url = buildString { - append("https://") - append(getDomain()) - append("/?next=") - append(next) - if (!tags.isNullOrEmpty()) { - var fCats = 0 - for (tag in tags) { - tag.key.toIntOrNull()?.let { fCats = fCats or it } ?: run { - search += tag.key + " " - } - } - if (fCats != 0) { - append("&f_cats=") - append(1023 - fCats) - } - } - if (search.isNotEmpty()) { - append("&f_search=") - append(search.trim().replace(' ', '+')) - } - // by unknown reason cookie "sl=dm_2" is ignored, so, we should request it again - if (updateDm) { - append("&inline_set=dm_e") - } - append("&advsearch=1") - if (config[suspiciousContentKey]) { - append("&f_sh=on") - } - } - val body = context.httpGet(url).parseHtml().body() - val root = body.selectFirst("table.itg") - ?.selectFirst("tbody") - ?: if (updateDm) { - body.parseFailed("Cannot find root") - } else { - updateDm = true - return getListPage(page, query, tags, sortOrder) - } - updateDm = false - nextPages[page + 1] = getNextTimestamp(body) - return root.children().mapNotNull { tr -> - if (tr.childrenSize() != 2) return@mapNotNull null - val (td1, td2) = tr.children() - val glink = td2.selectFirstOrThrow("div.glink") - val a = glink.parents().select("a").first() ?: glink.parseFailed("link not found") - val href = a.attrAsRelativeUrl("href") - val tagsDiv = glink.nextElementSibling() ?: glink.parseFailed("tags div not found") - val mainTag = td2.selectFirst("div.cn")?.let { div -> - MangaTag( - title = div.text().toTitleCase(), - key = tagIdByClass(div.classNames()) ?: return@let null, - source = source, - ) - } - Manga( - id = generateUid(href), - title = glink.text().cleanupTitle(), - altTitle = null, - url = href, - publicUrl = a.absUrl("href"), - rating = td2.selectFirst("div.ir")?.parseRating() ?: RATING_UNKNOWN, - isNsfw = true, - coverUrl = td1.selectFirst("img")?.absUrl("src").orEmpty(), - tags = setOfNotNull(mainTag), - state = null, - author = tagsDiv.getElementsContainingOwnText("artist:").first() - ?.nextElementSibling()?.text(), - source = source, - ) - } - } + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + var search = query?.urlEncoded().orEmpty() + val next = nextPages.get(page, 0L) + if (page > 0 && next == 0L) { + assert(false) { "Page timestamp not found" } + return emptyList() + } + val url = buildString { + append("https://") + append(domain) + append("/?next=") + append(next) + if (!tags.isNullOrEmpty()) { + var fCats = 0 + for (tag in tags) { + tag.key.toIntOrNull()?.let { fCats = fCats or it } ?: run { + search += tag.key + " " + } + } + if (fCats != 0) { + append("&f_cats=") + append(1023 - fCats) + } + } + if (search.isNotEmpty()) { + append("&f_search=") + append(search.trim().replace(' ', '+')) + } + // by unknown reason cookie "sl=dm_2" is ignored, so, we should request it again + if (updateDm) { + append("&inline_set=dm_e") + } + append("&advsearch=1") + if (config[suspiciousContentKey]) { + append("&f_sh=on") + } + } + val body = webClient.httpGet(url).parseHtml().body() + val root = body.selectFirst("table.itg") + ?.selectFirst("tbody") + ?: if (updateDm) { + body.parseFailed("Cannot find root") + } else { + updateDm = true + return getListPage(page, query, tags, sortOrder) + } + updateDm = false + nextPages[page + 1] = getNextTimestamp(body) + return root.children().mapNotNull { tr -> + if (tr.childrenSize() != 2) return@mapNotNull null + val (td1, td2) = tr.children() + val glink = td2.selectFirstOrThrow("div.glink") + val a = glink.parents().select("a").first() ?: glink.parseFailed("link not found") + val href = a.attrAsRelativeUrl("href") + val tagsDiv = glink.nextElementSibling() ?: glink.parseFailed("tags div not found") + val mainTag = td2.selectFirst("div.cn")?.let { div -> + MangaTag( + title = div.text().toTitleCase(), + key = tagIdByClass(div.classNames()) ?: return@let null, + source = source, + ) + } + Manga( + id = generateUid(href), + title = glink.text().cleanupTitle(), + altTitle = null, + url = href, + publicUrl = a.absUrl("href"), + rating = td2.selectFirst("div.ir")?.parseRating() ?: RATING_UNKNOWN, + isNsfw = true, + coverUrl = td1.selectFirst("img")?.absUrl("src").orEmpty(), + tags = setOfNotNull(mainTag), + state = null, + author = tagsDiv.getElementsContainingOwnText("artist:").first() + ?.nextElementSibling()?.text(), + source = source, + ) + } + } - override suspend fun getDetails(manga: Manga): Manga { - val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml() - val root = doc.body().selectFirstOrThrow("div.gm") - val cover = root.getElementById("gd1")?.children()?.first() - val title = root.getElementById("gd2") - val taglist = root.getElementById("taglist") - val tabs = doc.body().selectFirst("table.ptt")?.selectFirst("tr") - return manga.copy( - title = title?.getElementById("gn")?.text()?.cleanupTitle() ?: manga.title, - altTitle = title?.getElementById("gj")?.text()?.cleanupTitle() ?: manga.altTitle, - publicUrl = doc.baseUri().ifEmpty { manga.publicUrl }, - rating = root.getElementById("rating_label")?.text() - ?.substringAfterLast(' ') - ?.toFloatOrNull() - ?.div(5f) ?: manga.rating, - largeCoverUrl = cover?.styleValueOrNull("background")?.cssUrl(), - description = taglist?.select("tr")?.joinToString("
") { tr -> - val (tc, td) = tr.children() - val subtags = td.select("a").joinToString { it.html() } - "${tc.html()} $subtags" - }, - chapters = tabs?.select("a")?.findLast { a -> - a.text().toIntOrNull() != null - }?.let { a -> - val count = a.text().toInt() - val chapters = ChaptersListBuilder(count) - for (i in 1..count) { - val url = "${manga.url}?p=${i - 1}" - chapters += MangaChapter( - id = generateUid(url), - name = "${manga.title} #$i", - number = i, - url = url, - uploadDate = 0L, - source = source, - scanlator = null, - branch = null, - ) - } - chapters.toList() - }, - ) - } + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val root = doc.body().selectFirstOrThrow("div.gm") + val cover = root.getElementById("gd1")?.children()?.first() + val title = root.getElementById("gd2") + val taglist = root.getElementById("taglist") + val tabs = doc.body().selectFirst("table.ptt")?.selectFirst("tr") + return manga.copy( + title = title?.getElementById("gn")?.text()?.cleanupTitle() ?: manga.title, + altTitle = title?.getElementById("gj")?.text()?.cleanupTitle() ?: manga.altTitle, + publicUrl = doc.baseUri().ifEmpty { manga.publicUrl }, + rating = root.getElementById("rating_label")?.text() + ?.substringAfterLast(' ') + ?.toFloatOrNull() + ?.div(5f) ?: manga.rating, + largeCoverUrl = cover?.styleValueOrNull("background")?.cssUrl(), + description = taglist?.select("tr")?.joinToString("
") { tr -> + val (tc, td) = tr.children() + val subtags = td.select("a").joinToString { it.html() } + "${tc.html()} $subtags" + }, + chapters = tabs?.select("a")?.findLast { a -> + a.text().toIntOrNull() != null + }?.let { a -> + val count = a.text().toInt() + val chapters = ChaptersListBuilder(count) + for (i in 1..count) { + val url = "${manga.url}?p=${i - 1}" + chapters += MangaChapter( + id = generateUid(url), + name = "${manga.title} #$i", + number = i, + url = url, + uploadDate = 0L, + source = source, + scanlator = null, + branch = null, + ) + } + chapters.toList() + }, + ) + } - override suspend fun getPages(chapter: MangaChapter): List { - val doc = context.httpGet(chapter.url.toAbsoluteUrl(getDomain())).parseHtml() - val root = doc.body().requireElementById("gdt") - return root.select("a").map { a -> - val url = a.attrAsRelativeUrl("href") - MangaPage( - id = generateUid(url), - url = url, - referer = a.absUrl("href"), - preview = null, - source = source, - ) - } - } + override suspend fun getPages(chapter: MangaChapter): List { + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + val root = doc.body().requireElementById("gdt") + return root.select("a").map { a -> + val url = a.attrAsRelativeUrl("href") + MangaPage( + id = generateUid(url), + url = url, + referer = a.absUrl("href"), + preview = null, + source = source, + ) + } + } - override suspend fun getPageUrl(page: MangaPage): String { - val doc = context.httpGet(page.url.toAbsoluteUrl(getDomain())).parseHtml() - return doc.body().requireElementById("img").attrAsAbsoluteUrl("src") - } + override suspend fun getPageUrl(page: MangaPage): String { + val doc = webClient.httpGet(page.url.toAbsoluteUrl(domain)).parseHtml() + return doc.body().requireElementById("img").attrAsAbsoluteUrl("src") + } - override suspend fun getTags(): Set { - val doc = context.httpGet("https://${getDomain()}").parseHtml() - val root = doc.body().requireElementById("searchbox").selectFirstOrThrow("table") - return root.select("div.cs").mapNotNullToSet { div -> - val id = div.id().substringAfterLast('_').toIntOrNull() - ?: return@mapNotNullToSet null - MangaTag( - title = div.text().toTitleCase(), - key = id.toString(), - source = source, - ) - } - } + override suspend fun getTags(): Set { + val doc = webClient.httpGet("https://${domain}").parseHtml() + val root = doc.body().requireElementById("searchbox").selectFirstOrThrow("table") + return root.select("div.cs").mapNotNullToSet { div -> + val id = div.id().substringAfterLast('_').toIntOrNull() + ?: return@mapNotNullToSet null + MangaTag( + title = div.text().toTitleCase(), + key = id.toString(), + source = source, + ) + } + } - override suspend fun getUsername(): String { - val doc = context.httpGet("https://forums.$DOMAIN_UNAUTHORIZED/").parseHtml().body() - val username = doc.getElementById("userlinks") - ?.getElementsByAttributeValueContaining("href", "showuser=") - ?.firstOrNull() - ?.ownText() - ?: if (doc.getElementById("userlinksguest") != null) { - throw AuthRequiredException(source) - } else { - doc.parseFailed() - } - return username - } + override suspend fun getUsername(): String { + val doc = webClient.httpGet("https://forums.$DOMAIN_UNAUTHORIZED/").parseHtml().body() + val username = doc.getElementById("userlinks") + ?.getElementsByAttributeValueContaining("href", "showuser=") + ?.firstOrNull() + ?.ownText() + ?: if (doc.getElementById("userlinksguest") != null) { + throw AuthRequiredException(source) + } else { + doc.parseFailed() + } + return username + } - override fun onCreateConfig(keys: MutableCollection>) { - super.onCreateConfig(keys) - keys.add(suspiciousContentKey) - } + override fun onCreateConfig(keys: MutableCollection>) { + super.onCreateConfig(keys) + keys.add(suspiciousContentKey) + } - private fun isAuthorized(domain: String): Boolean { - val cookies = context.cookieJar.getCookies(domain).mapToSet { x -> x.name } - return authCookies.all { it in cookies } - } + private fun isAuthorized(domain: String): Boolean { + val cookies = context.cookieJar.getCookies(domain).mapToSet { x -> x.name } + return authCookies.all { it in cookies } + } - private fun Element.parseRating(): Float { - return runCatching { - val style = requireNotNull(attr("style")) - val (v1, v2) = ratingPattern.find(style)!!.destructured - var p1 = v1.dropLast(2).toInt() - val p2 = v2.dropLast(2).toInt() - if (p2 != -1) { - p1 += 8 - } - (80 - p1) / 80f - }.getOrDefault(RATING_UNKNOWN) - } + private fun Element.parseRating(): Float { + return runCatching { + val style = requireNotNull(attr("style")) + val (v1, v2) = ratingPattern.find(style)!!.destructured + var p1 = v1.dropLast(2).toInt() + val p2 = v2.dropLast(2).toInt() + if (p2 != -1) { + p1 += 8 + } + (80 - p1) / 80f + }.getOrDefault(RATING_UNKNOWN) + } - private fun String.cleanupTitle(): String { - val result = StringBuilder(length) - var skip = false - for (c in this) { - when { - c == '[' -> skip = true - c == ']' -> skip = false - c.isWhitespace() && result.isEmpty() -> continue - !skip -> result.append(c) - } - } - while (result.lastOrNull()?.isWhitespace() == true) { - result.deleteCharAt(result.lastIndex) - } - return result.toString() - } + private fun String.cleanupTitle(): String { + val result = StringBuilder(length) + var skip = false + for (c in this) { + when { + c == '[' -> skip = true + c == ']' -> skip = false + c.isWhitespace() && result.isEmpty() -> continue + !skip -> result.append(c) + } + } + while (result.lastOrNull()?.isWhitespace() == true) { + result.deleteCharAt(result.lastIndex) + } + return result.toString() + } - private fun String.cssUrl(): String? { - val fromIndex = indexOf("url(") - if (fromIndex == -1) { - return null - } - val toIndex = indexOf(')', startIndex = fromIndex) - return if (toIndex == -1) { - null - } else { - substring(fromIndex + 4, toIndex).trim() - } - } + private fun String.cssUrl(): String? { + val fromIndex = indexOf("url(") + if (fromIndex == -1) { + return null + } + val toIndex = indexOf(')', startIndex = fromIndex) + return if (toIndex == -1) { + null + } else { + substring(fromIndex + 4, toIndex).trim() + } + } - private fun tagIdByClass(classNames: Collection): String? { - val className = classNames.find { x -> x.startsWith("ct") } ?: return null - val num = className.drop(2).toIntOrNull(16) ?: return null - return 2.0.pow(num).toInt().toString() - } + private fun tagIdByClass(classNames: Collection): String? { + val className = classNames.find { x -> x.startsWith("ct") } ?: return null + val num = className.drop(2).toIntOrNull(16) ?: return null + return 2.0.pow(num).toInt().toString() + } - private fun getNextTimestamp(root: Element): Long { - return root.getElementById("unext") - ?.attrAsAbsoluteUrlOrNull("href") - ?.toHttpUrlOrNull() - ?.queryParameter("next") - ?.toLongOrNull() ?: 1 - } -} \ No newline at end of file + private fun getNextTimestamp(root: Element): Long { + return root.getElementById("unext") + ?.attrAsAbsoluteUrlOrNull("href") + ?.toHttpUrlOrNull() + ?.queryParameter("next") + ?.toLongOrNull() ?: 1 + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaDexParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaDexParser.kt index a5632c46e..ed236b158 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaDexParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaDexParser.kt @@ -22,254 +22,254 @@ private const val CHAPTERS_MAX_PAGE_SIZE = 500 private const val CHAPTERS_PARALLELISM = 3 private const val CHAPTERS_MAX_COUNT = 10_000 // strange api behavior, looks like a bug private const val CONTENT_RATING = - "contentRating[]=safe&contentRating[]=suggestive&contentRating[]=erotica&contentRating[]=pornographic" + "contentRating[]=safe&contentRating[]=suggestive&contentRating[]=erotica&contentRating[]=pornographic" private const val LOCALE_FALLBACK = "en" @MangaSourceParser("MANGADEX", "MangaDex") -internal class MangaDexParser(override val context: MangaLoaderContext) : MangaParser(MangaSource.MANGADEX) { +internal class MangaDexParser(context: MangaLoaderContext) : MangaParser(context, MangaSource.MANGADEX) { - override val configKeyDomain = ConfigKey.Domain("mangadex.org", null) + override val configKeyDomain = ConfigKey.Domain("mangadex.org", null) - override val sortOrders: EnumSet = EnumSet.of( - SortOrder.UPDATED, - SortOrder.ALPHABETICAL, - SortOrder.NEWEST, - SortOrder.POPULARITY, - ) + override val sortOrders: EnumSet = EnumSet.of( + SortOrder.UPDATED, + SortOrder.ALPHABETICAL, + SortOrder.NEWEST, + SortOrder.POPULARITY, + ) - override suspend fun getList( - offset: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - val domain = getDomain() - val url = buildString { - append("https://api.") - append(domain) - append("/manga?limit=") - append(PAGE_SIZE) - append("&offset=") - append(offset) - append("&includes[]=cover_art&includes[]=author&includes[]=artist&") - tags?.forEach { tag -> - append("includedTags[]=") - append(tag.key) - append('&') - } - if (!query.isNullOrEmpty()) { - append("title=") - append(query.urlEncoded()) - append('&') - } - append(CONTENT_RATING) - append("&order") - append( - when (sortOrder) { - SortOrder.UPDATED, - -> "[latestUploadedChapter]=desc" + override suspend fun getList( + offset: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + val domain = domain + val url = buildString { + append("https://api.") + append(domain) + append("/manga?limit=") + append(PAGE_SIZE) + append("&offset=") + append(offset) + append("&includes[]=cover_art&includes[]=author&includes[]=artist&") + tags?.forEach { tag -> + append("includedTags[]=") + append(tag.key) + append('&') + } + if (!query.isNullOrEmpty()) { + append("title=") + append(query.urlEncoded()) + append('&') + } + append(CONTENT_RATING) + append("&order") + append( + when (sortOrder) { + SortOrder.UPDATED, + -> "[latestUploadedChapter]=desc" - SortOrder.ALPHABETICAL -> "[title]=asc" - SortOrder.NEWEST -> "[createdAt]=desc" - SortOrder.POPULARITY -> "[followedCount]=desc" - else -> "[followedCount]=desc" - }, - ) - } - val json = context.httpGet(url).parseJson().getJSONArray("data") - return json.mapJSON { jo -> - val id = jo.getString("id") - val attrs = jo.getJSONObject("attributes") - val relations = jo.getJSONArray("relationships").associateByKey("type") - val cover = relations["cover_art"] - ?.getJSONObject("attributes") - ?.getString("fileName") - ?.let { - "https://uploads.$domain/covers/$id/$it" - } - Manga( - id = generateUid(id), - title = requireNotNull(attrs.getJSONObject("title").selectByLocale()) { - "Title should not be null" - }, - altTitle = attrs.optJSONObject("altTitles")?.selectByLocale(), - url = id, - publicUrl = "https://$domain/title/$id", - rating = RATING_UNKNOWN, - isNsfw = attrs.getStringOrNull("contentRating") == "erotica", - coverUrl = cover?.plus(".256.jpg").orEmpty(), - largeCoverUrl = cover, - description = attrs.optJSONObject("description")?.selectByLocale(), - tags = attrs.getJSONArray("tags").mapJSONToSet { tag -> - MangaTag( - title = tag.getJSONObject("attributes") - .getJSONObject("name") - .firstStringValue() - .toTitleCase(), - key = tag.getString("id"), - source = source, - ) - }, - state = when (jo.getStringOrNull("status")) { - "ongoing" -> MangaState.ONGOING - "completed" -> MangaState.FINISHED - else -> null - }, - author = (relations["author"] ?: relations["artist"]) - ?.getJSONObject("attributes") - ?.getStringOrNull("name"), - source = source, - ) - } - } + SortOrder.ALPHABETICAL -> "[title]=asc" + SortOrder.NEWEST -> "[createdAt]=desc" + SortOrder.POPULARITY -> "[followedCount]=desc" + else -> "[followedCount]=desc" + }, + ) + } + val json = webClient.httpGet(url).parseJson().getJSONArray("data") + return json.mapJSON { jo -> + val id = jo.getString("id") + val attrs = jo.getJSONObject("attributes") + val relations = jo.getJSONArray("relationships").associateByKey("type") + val cover = relations["cover_art"] + ?.getJSONObject("attributes") + ?.getString("fileName") + ?.let { + "https://uploads.$domain/covers/$id/$it" + } + Manga( + id = generateUid(id), + title = requireNotNull(attrs.getJSONObject("title").selectByLocale()) { + "Title should not be null" + }, + altTitle = attrs.optJSONObject("altTitles")?.selectByLocale(), + url = id, + publicUrl = "https://$domain/title/$id", + rating = RATING_UNKNOWN, + isNsfw = attrs.getStringOrNull("contentRating") == "erotica", + coverUrl = cover?.plus(".256.jpg").orEmpty(), + largeCoverUrl = cover, + description = attrs.optJSONObject("description")?.selectByLocale(), + tags = attrs.getJSONArray("tags").mapJSONToSet { tag -> + MangaTag( + title = tag.getJSONObject("attributes") + .getJSONObject("name") + .firstStringValue() + .toTitleCase(), + key = tag.getString("id"), + source = source, + ) + }, + state = when (jo.getStringOrNull("status")) { + "ongoing" -> MangaState.ONGOING + "completed" -> MangaState.FINISHED + else -> null + }, + author = (relations["author"] ?: relations["artist"]) + ?.getJSONObject("attributes") + ?.getStringOrNull("name"), + source = source, + ) + } + } - override suspend fun getDetails(manga: Manga): Manga = coroutineScope { - val domain = getDomain() - val mangaId = manga.url.removePrefix("/") - val attrsDeferred = async { - context.httpGet( - "https://api.$domain/manga/${mangaId}?includes[]=artist&includes[]=author&includes[]=cover_art", - ).parseJson().getJSONObject("data").getJSONObject("attributes") - } - val feedDeferred = async { loadChapters(mangaId) } - val mangaAttrs = attrsDeferred.await() - val feed = feedDeferred.await() - // 2022-01-02T00:27:11+00:00 - val dateFormat = SimpleDateFormat( - "yyyy-MM-dd'T'HH:mm:ss'+00:00'", - Locale.ROOT, - ) - manga.copy( - description = mangaAttrs.optJSONObject("description")?.selectByLocale() - ?: manga.description, - chapters = feed.mapChapters { _, jo -> - val id = jo.getString("id") - val attrs = jo.getJSONObject("attributes") - if (!attrs.isNull("externalUrl")) { - return@mapChapters null - } - val locale = attrs.getStringOrNull("translatedLanguage")?.let { Locale.forLanguageTag(it) } - val relations = jo.getJSONArray("relationships").associateByKey("type") - val number = attrs.getIntOrDefault("chapter", 0) - MangaChapter( - id = generateUid(id), - name = attrs.getStringOrNull("title")?.takeUnless(String::isEmpty) - ?: "Chapter #$number", - number = number, - url = id, - scanlator = relations["scanlation_group"]?.getStringOrNull("name"), - uploadDate = dateFormat.tryParse(attrs.getString("publishAt")), - branch = locale?.getDisplayName(locale)?.toTitleCase(locale), - source = source, - ) - }, - ) - } + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { + val domain = domain + val mangaId = manga.url.removePrefix("/") + val attrsDeferred = async { + webClient.httpGet( + "https://api.$domain/manga/${mangaId}?includes[]=artist&includes[]=author&includes[]=cover_art", + ).parseJson().getJSONObject("data").getJSONObject("attributes") + } + val feedDeferred = async { loadChapters(mangaId) } + val mangaAttrs = attrsDeferred.await() + val feed = feedDeferred.await() + // 2022-01-02T00:27:11+00:00 + val dateFormat = SimpleDateFormat( + "yyyy-MM-dd'T'HH:mm:ss'+00:00'", + Locale.ROOT, + ) + manga.copy( + description = mangaAttrs.optJSONObject("description")?.selectByLocale() + ?: manga.description, + chapters = feed.mapChapters { _, jo -> + val id = jo.getString("id") + val attrs = jo.getJSONObject("attributes") + if (!attrs.isNull("externalUrl")) { + return@mapChapters null + } + val locale = attrs.getStringOrNull("translatedLanguage")?.let { Locale.forLanguageTag(it) } + val relations = jo.getJSONArray("relationships").associateByKey("type") + val number = attrs.getIntOrDefault("chapter", 0) + MangaChapter( + id = generateUid(id), + name = attrs.getStringOrNull("title")?.takeUnless(String::isEmpty) + ?: "Chapter #$number", + number = number, + url = id, + scanlator = relations["scanlation_group"]?.getStringOrNull("name"), + uploadDate = dateFormat.tryParse(attrs.getString("publishAt")), + branch = locale?.getDisplayName(locale)?.toTitleCase(locale), + source = source, + ) + }, + ) + } - override suspend fun getPages(chapter: MangaChapter): List { - val domain = getDomain() - val chapterJson = context.httpGet("https://api.$domain/at-home/server/${chapter.url}?forcePort443=false") - .parseJson() - .getJSONObject("chapter") - val pages = chapterJson.getJSONArray("data") - val prefix = "https://uploads.$domain/data/${chapterJson.getString("hash")}/" - val referer = "https://$domain/" - return List(pages.length()) { i -> - val url = prefix + pages.getString(i) - MangaPage( - id = generateUid(url), - url = url, - referer = referer, - preview = null, // TODO prefix + dataSaver.getString(i), - source = source, - ) - } - } + override suspend fun getPages(chapter: MangaChapter): List { + val domain = domain + val chapterJson = webClient.httpGet("https://api.$domain/at-home/server/${chapter.url}?forcePort443=false") + .parseJson() + .getJSONObject("chapter") + val pages = chapterJson.getJSONArray("data") + val prefix = "https://uploads.$domain/data/${chapterJson.getString("hash")}/" + val referer = "https://$domain/" + return List(pages.length()) { i -> + val url = prefix + pages.getString(i) + MangaPage( + id = generateUid(url), + url = url, + referer = referer, + preview = null, // TODO prefix + dataSaver.getString(i), + source = source, + ) + } + } - override suspend fun getTags(): Set { - val tags = context.httpGet("https://api.${getDomain()}/manga/tag").parseJson() - .getJSONArray("data") - return tags.mapJSONToSet { jo -> - MangaTag( - title = jo.getJSONObject("attributes").getJSONObject("name").firstStringValue().toTitleCase(), - key = jo.getString("id"), - source = source, - ) - } - } + override suspend fun getTags(): Set { + val tags = webClient.httpGet("https://api.${domain}/manga/tag").parseJson() + .getJSONArray("data") + return tags.mapJSONToSet { jo -> + MangaTag( + title = jo.getJSONObject("attributes").getJSONObject("name").firstStringValue().toTitleCase(), + key = jo.getString("id"), + source = source, + ) + } + } - private fun JSONObject.firstStringValue() = values().next() as String + private fun JSONObject.firstStringValue() = values().next() as String - private fun JSONObject.selectByLocale(): String? { - val preferredLocales = context.getPreferredLocales() - for (locale in preferredLocales) { - getStringOrNull(locale.language)?.let { return it } - getStringOrNull(locale.toLanguageTag())?.let { return it } - } - return getStringOrNull(LOCALE_FALLBACK) ?: values().nextOrNull() as? String - } + private fun JSONObject.selectByLocale(): String? { + val preferredLocales = context.getPreferredLocales() + for (locale in preferredLocales) { + getStringOrNull(locale.language)?.let { return it } + getStringOrNull(locale.toLanguageTag())?.let { return it } + } + return getStringOrNull(LOCALE_FALLBACK) ?: values().nextOrNull() as? String + } - private suspend fun loadChapters(mangaId: String): List { - val firstPage = loadChapters(mangaId, offset = 0, limit = CHAPTERS_FIRST_PAGE_SIZE) - if (firstPage.size >= firstPage.total) { - return firstPage.data - } - val tail = coroutineScope { - val leftCount = firstPage.total.coerceAtMost(CHAPTERS_MAX_COUNT) - firstPage.size - val pages = (leftCount / CHAPTERS_MAX_PAGE_SIZE.toFloat()).toIntUp() - val dispatcher = Dispatchers.Default.limitedParallelism(CHAPTERS_PARALLELISM) - List(pages) { page -> - val offset = page * CHAPTERS_MAX_PAGE_SIZE + firstPage.size - async(dispatcher) { - loadChapters(mangaId, offset, CHAPTERS_MAX_PAGE_SIZE) - } - }.awaitAll() - } - val result = ArrayList(firstPage.total) - result += firstPage.data - tail.flatMapTo(result) { it.data } - return result - } + private suspend fun loadChapters(mangaId: String): List { + val firstPage = loadChapters(mangaId, offset = 0, limit = CHAPTERS_FIRST_PAGE_SIZE) + if (firstPage.size >= firstPage.total) { + return firstPage.data + } + val tail = coroutineScope { + val leftCount = firstPage.total.coerceAtMost(CHAPTERS_MAX_COUNT) - firstPage.size + val pages = (leftCount / CHAPTERS_MAX_PAGE_SIZE.toFloat()).toIntUp() + val dispatcher = Dispatchers.Default.limitedParallelism(CHAPTERS_PARALLELISM) + List(pages) { page -> + val offset = page * CHAPTERS_MAX_PAGE_SIZE + firstPage.size + async(dispatcher) { + loadChapters(mangaId, offset, CHAPTERS_MAX_PAGE_SIZE) + } + }.awaitAll() + } + val result = ArrayList(firstPage.total) + result += firstPage.data + tail.flatMapTo(result) { it.data } + return result + } - private suspend fun loadChapters(mangaId: String, offset: Int, limit: Int): Chapters { - val limitedLimit = when { - offset >= CHAPTERS_MAX_COUNT -> return Chapters(emptyList(), CHAPTERS_MAX_COUNT) - offset + limit > CHAPTERS_MAX_COUNT -> CHAPTERS_MAX_COUNT - offset - else -> limit - } - val url = buildString { - append("https://api.") - append(getDomain()) - append("/manga/") - append(mangaId) - append("/feed") - append("?limit=") - append(limitedLimit) - append("&includes[]=scanlation_group&order[volume]=asc&order[chapter]=asc&offset=") - append(offset) - append('&') - append(CONTENT_RATING) - } - val json = context.httpGet(url).parseJson() - if (json.getString("result") == "ok") { - return Chapters( - data = json.optJSONArray("data")?.toJSONList().orEmpty(), - total = json.getInt("total"), - ) - } else { - val error = json.optJSONArray("errors").mapJSON { jo -> - jo.getString("detail") - }.joinToString("\n") - throw ParseException(error, url) - } - } + private suspend fun loadChapters(mangaId: String, offset: Int, limit: Int): Chapters { + val limitedLimit = when { + offset >= CHAPTERS_MAX_COUNT -> return Chapters(emptyList(), CHAPTERS_MAX_COUNT) + offset + limit > CHAPTERS_MAX_COUNT -> CHAPTERS_MAX_COUNT - offset + else -> limit + } + val url = buildString { + append("https://api.") + append(domain) + append("/manga/") + append(mangaId) + append("/feed") + append("?limit=") + append(limitedLimit) + append("&includes[]=scanlation_group&order[volume]=asc&order[chapter]=asc&offset=") + append(offset) + append('&') + append(CONTENT_RATING) + } + val json = webClient.httpGet(url).parseJson() + if (json.getString("result") == "ok") { + return Chapters( + data = json.optJSONArray("data")?.toJSONList().orEmpty(), + total = json.getInt("total"), + ) + } else { + val error = json.optJSONArray("errors").mapJSON { jo -> + jo.getString("detail") + }.joinToString("\n") + throw ParseException(error, url) + } + } - private class Chapters( - val data: List, - val total: Int, - ) { + private class Chapters( + val data: List, + val total: Int, + ) { - val size: Int - get() = data.size - } -} \ No newline at end of file + val size: Int + get() = data.size + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaInUaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaInUaParser.kt index e041a086c..68726425b 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaInUaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaInUaParser.kt @@ -12,141 +12,142 @@ import java.util.* private const val DEF_BRANCH_NAME = "Основний переклад" @MangaSourceParser("MANGAINUA", "MANGA/in/UA", "uk") -class MangaInUaParser(override val context: MangaLoaderContext) : PagedMangaParser( - source = MangaSource.MANGAINUA, - pageSize = 24, - searchPageSize = 10, +class MangaInUaParser(context: MangaLoaderContext) : PagedMangaParser( + context = context, + source = MangaSource.MANGAINUA, + pageSize = 24, + searchPageSize = 10, ) { - override val sortOrders: Set - get() = Collections.singleton(SortOrder.UPDATED) + override val sortOrders: Set + get() = Collections.singleton(SortOrder.UPDATED) - override val configKeyDomain: ConfigKey.Domain = ConfigKey.Domain("manga.in.ua", null) + override val configKeyDomain: ConfigKey.Domain = ConfigKey.Domain("manga.in.ua", null) - override suspend fun getListPage( - page: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - val url = when { - !query.isNullOrEmpty() -> ( - "/index.php?do=search" + - "&subaction=search" + - "&search_start=$page" + - "&full_search=1" + - "&story=$query" + - "&titleonly=3" - ).toAbsoluteUrl(getDomain()) + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + val url = when { + !query.isNullOrEmpty() -> ( + "/index.php?do=search" + + "&subaction=search" + + "&search_start=$page" + + "&full_search=1" + + "&story=$query" + + "&titleonly=3" + ).toAbsoluteUrl(domain) - tags.isNullOrEmpty() -> "/mangas/page/$page".toAbsoluteUrl(getDomain()) - tags.size == 1 -> "${tags.first().key}/page/$page" - tags.size > 1 -> throw IllegalArgumentException("This source supports only 1 genre") - else -> "/mangas/page/$page".toAbsoluteUrl(getDomain()) - } - val doc = context.httpGet(url).parseHtml() - val container = doc.body().requireElementById("site-content") - val items = container.select("div.col-6") - return items.mapNotNull { item -> - val href = item.selectFirst("a")?.attrAsRelativeUrl("href") ?: return@mapNotNull null - Manga( - id = generateUid(href), - title = item.selectFirst("h3.card__title")?.text() ?: return@mapNotNull null, - coverUrl = item.selectFirst("header.card__cover")?.selectFirst("img")?.run { - attrAsAbsoluteUrlOrNull("data-src") ?: attrAsAbsoluteUrlOrNull("src") - }.orEmpty(), - altTitle = null, - author = null, - rating = item.selectFirst("div.card__short-rate--num") - ?.text() - ?.toFloatOrNull() - ?.div(10F) ?: RATING_UNKNOWN, - url = href, - isNsfw = item.selectFirst("ul.card__list")?.select("li")?.lastOrNull()?.text() == "18+", - tags = runCatching { - item.selectFirst("div.card__category")?.select("a")?.mapToSet { - MangaTag( - title = it.ownText(), - key = it.attr("href").removeSuffix("/"), - source = source, - ) - } - }.getOrNull().orEmpty(), - state = null, - publicUrl = href.toAbsoluteUrl(container.host ?: getDomain()), - source = source, - ) - } - } + tags.isNullOrEmpty() -> "/mangas/page/$page".toAbsoluteUrl(domain) + tags.size == 1 -> "${tags.first().key}/page/$page" + tags.size > 1 -> throw IllegalArgumentException("This source supports only 1 genre") + else -> "/mangas/page/$page".toAbsoluteUrl(domain) + } + val doc = webClient.httpGet(url).parseHtml() + val container = doc.body().requireElementById("site-content") + val items = container.select("div.col-6") + return items.mapNotNull { item -> + val href = item.selectFirst("a")?.attrAsRelativeUrl("href") ?: return@mapNotNull null + Manga( + id = generateUid(href), + title = item.selectFirst("h3.card__title")?.text() ?: return@mapNotNull null, + coverUrl = item.selectFirst("header.card__cover")?.selectFirst("img")?.run { + attrAsAbsoluteUrlOrNull("data-src") ?: attrAsAbsoluteUrlOrNull("src") + }.orEmpty(), + altTitle = null, + author = null, + rating = item.selectFirst("div.card__short-rate--num") + ?.text() + ?.toFloatOrNull() + ?.div(10F) ?: RATING_UNKNOWN, + url = href, + isNsfw = item.selectFirst("ul.card__list")?.select("li")?.lastOrNull()?.text() == "18+", + tags = runCatching { + item.selectFirst("div.card__category")?.select("a")?.mapToSet { + MangaTag( + title = it.ownText(), + key = it.attr("href").removeSuffix("/"), + source = source, + ) + } + }.getOrNull().orEmpty(), + state = null, + publicUrl = href.toAbsoluteUrl(container.host ?: domain), + source = source, + ) + } + } - override suspend fun getDetails(manga: Manga): Manga { - val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml() - val root = doc.body().requireElementById("site-content") - val dateFormat = SimpleDateFormat("dd.MM.yyyy", Locale.US) - val chapterNodes = root.selectFirstOrThrow(".linkstocomics").select(".ltcitems") - var prevChapterName: String? = null - var i = 0 - return manga.copy( - description = root.selectFirst("div.item__full-description")?.text(), - largeCoverUrl = root.selectFirst("div.item__full-sidebar--poster")?.selectFirst("img") - ?.attrAsAbsoluteUrlOrNull("src"), - chapters = chapterNodes.mapChapters { _, item -> - val href = item?.selectFirst("a")?.attrAsRelativeUrlOrNull("href") - ?: return@mapChapters null - val isAlternative = item.styleValueOrNull("background") != null - val name = item.selectFirst("a")?.text().orEmpty() - if (!isAlternative) i++ - MangaChapter( - id = generateUid(href), - name = if (isAlternative) { - prevChapterName ?: return@mapChapters null - } else { - prevChapterName = name - name - }, - number = i, - url = href, - scanlator = null, - branch = if (isAlternative) { - name.substringAfterLast(':').trim() - } else { - DEF_BRANCH_NAME - }, - uploadDate = dateFormat.tryParse(item.selectFirst("div.ltcright")?.text()), - source = source, - ) - }, - ) - } + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val root = doc.body().requireElementById("site-content") + val dateFormat = SimpleDateFormat("dd.MM.yyyy", Locale.US) + val chapterNodes = root.selectFirstOrThrow(".linkstocomics").select(".ltcitems") + var prevChapterName: String? = null + var i = 0 + return manga.copy( + description = root.selectFirst("div.item__full-description")?.text(), + largeCoverUrl = root.selectFirst("div.item__full-sidebar--poster")?.selectFirst("img") + ?.attrAsAbsoluteUrlOrNull("src"), + chapters = chapterNodes.mapChapters { _, item -> + val href = item?.selectFirst("a")?.attrAsRelativeUrlOrNull("href") + ?: return@mapChapters null + val isAlternative = item.styleValueOrNull("background") != null + val name = item.selectFirst("a")?.text().orEmpty() + if (!isAlternative) i++ + MangaChapter( + id = generateUid(href), + name = if (isAlternative) { + prevChapterName ?: return@mapChapters null + } else { + prevChapterName = name + name + }, + number = i, + url = href, + scanlator = null, + branch = if (isAlternative) { + name.substringAfterLast(':').trim() + } else { + DEF_BRANCH_NAME + }, + uploadDate = dateFormat.tryParse(item.selectFirst("div.ltcright")?.text()), + source = source, + ) + }, + ) + } - override suspend fun getPages(chapter: MangaChapter): List { - val fullUrl = chapter.url.toAbsoluteUrl(getDomain()) - val doc = context.httpGet(fullUrl).parseHtml() - val root = doc.body().requireElementById("comics").selectFirstOrThrow("ul.xfieldimagegallery") - return root.select("li").map { ul -> - val img = ul.selectFirstOrThrow("img") - val url = img.attrAsAbsoluteUrl("data-src") - MangaPage( - id = generateUid(url), - url = url, - preview = null, - referer = fullUrl, - source = source, - ) - } - } + override suspend fun getPages(chapter: MangaChapter): List { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + val root = doc.body().requireElementById("comics").selectFirstOrThrow("ul.xfieldimagegallery") + return root.select("li").map { ul -> + val img = ul.selectFirstOrThrow("img") + val url = img.attrAsAbsoluteUrl("data-src") + MangaPage( + id = generateUid(url), + url = url, + preview = null, + referer = fullUrl, + source = source, + ) + } + } - override suspend fun getTags(): Set { - val domain = getDomain() - val doc = context.httpGet("https://$domain/mangas").parseHtml() - val root = doc.body().requireElementById("menu_1").selectFirstOrThrow("div.menu__wrapper") - return root.select("li").mapNotNullToSet { li -> - val a = li.selectFirst("a") ?: return@mapNotNullToSet null - MangaTag( - title = a.ownText(), - key = a.attr("href").removeSuffix("/"), - source = source, - ) - } - } -} \ No newline at end of file + override suspend fun getTags(): Set { + val domain = domain + val doc = webClient.httpGet("https://$domain/mangas").parseHtml() + val root = doc.body().requireElementById("menu_1").selectFirstOrThrow("div.menu__wrapper") + return root.select("li").mapNotNullToSet { li -> + val a = li.selectFirst("a") ?: return@mapNotNullToSet null + MangaTag( + title = a.ownText(), + key = a.attr("href").removeSuffix("/"), + source = source, + ) + } + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaOwlParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaOwlParser.kt deleted file mode 100644 index 06dbf9fbf..000000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaOwlParser.kt +++ /dev/null @@ -1,187 +0,0 @@ -package org.koitharu.kotatsu.parsers.site - -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaParser -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.util.* -import java.text.SimpleDateFormat -import java.util.* - -@Deprecated("Permanently shutdown") -@MangaSourceParser("MANGAOWL", "MangaOwl", "en") -internal class MangaOwlParser(override val context: MangaLoaderContext) : MangaParser(MangaSource.MANGAOWL) { - - override val configKeyDomain = ConfigKey.Domain("mangaowls.com", null) - - override val sortOrders: Set = EnumSet.of( - SortOrder.POPULARITY, - SortOrder.NEWEST, - SortOrder.UPDATED, - ) - - private val regexNsfw = Regex("(yaoi)|(yuri)|(smut)|(mature)|(adult)", RegexOption.IGNORE_CASE) - - override suspend fun getList( - offset: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - val page = (offset / 36f).toIntUp().inc() - val link = buildString { - append("https://") - append(getDomain()) - when { - !query.isNullOrEmpty() -> { - append("/search/$page?search=") - append(query.urlEncoded()) - } - - !tags.isNullOrEmpty() -> { - for (tag in tags) { - append(tag.key) - } - append("/$page?type=${getAlternativeSortKey(sortOrder)}") - } - - else -> { - append("/${getSortKey(sortOrder)}/$page") - } - } - } - val doc = context.httpGet(link).parseHtml() - val slides = doc.body().selectOrThrow("ul.slides") - val items = slides.select("div.col-md-2") - return items.mapNotNull { item -> - val href = item.selectFirst("h6 a")?.attrAsRelativeUrlOrNull("href") ?: return@mapNotNull null - Manga( - id = generateUid(href), - title = item.selectFirst("h6 a")?.text() ?: return@mapNotNull null, - coverUrl = item.select("div.img-responsive").attr("abs:data-background-image"), - altTitle = null, - author = null, - rating = runCatching { - item.selectFirst("div.block-stars") - ?.text() - ?.toFloatOrNull() - ?.div(10f) - }.getOrNull() ?: RATING_UNKNOWN, - url = href, - isNsfw = false, - tags = emptySet(), - state = null, - publicUrl = href.toAbsoluteUrl(getDomain()), - source = source, - ) - } - } - - override suspend fun getDetails(manga: Manga): Manga { - val doc = context.httpGet(manga.publicUrl).parseHtml() - val info = doc.body().selectFirstOrThrow("div.single_detail") - val table = doc.body().selectFirstOrThrow("div.single-grid-right") - val dateFormat = SimpleDateFormat("MM/dd/yyyy", Locale.US) - val trRegex = "window\\['tr'] = '([^']*)';".toRegex(RegexOption.IGNORE_CASE) - val trElement = doc.getElementsByTag("script").find { trRegex.find(it.data()) != null } - ?: doc.parseFailed("Oops, tr not found") - val tr = trRegex.find(trElement.data())!!.groups[1]!!.value - val s = context.encodeBase64(getDomain().toByteArray()) - var isNsfw = manga.isNsfw - val parsedTags = info.select("div.col-xs-12.col-md-8.single-right-grid-right > p > a[href*=genres]") - .mapNotNullToSet { - val a = it.selectFirst("a") ?: return@mapNotNullToSet null - val name = a.text() - if (!isNsfw && isNsfwGenre(name)) { - isNsfw = true - } - MangaTag( - title = name.toTitleCase(), - key = a.attr("href"), - source = source, - ) - } - return manga.copy( - description = info.selectFirst(".description")?.html(), - largeCoverUrl = info.select("img").first()?.let { img -> - if (img.hasAttr("data-src")) img.attr("abs:data-src") else img.attr("abs:src") - }, - isNsfw = isNsfw, - author = info.selectFirst("p.fexi_header_para a.author_link")?.text(), - state = parseStatus(info.select("p.fexi_header_para:contains(status)").first()?.ownText()), - tags = manga.tags + parsedTags, - chapters = table.select("div.table.table-chapter-list").select("li.list-group-item.chapter_list") - .asReversed().mapChapters { i, li -> - val a = li.select("a") - val href = a.attr("data-href").ifEmpty { - li.parseFailed("Link is missing") - } - MangaChapter( - id = generateUid(href), - name = a.select("label").text(), - number = i + 1, - url = "$href?tr=$tr&s=$s", - scanlator = null, - branch = null, - uploadDate = dateFormat.tryParse(li.selectFirst("small:last-of-type")?.text()), - source = MangaSource.MANGAOWL, - ) - }, - ) - } - - override suspend fun getPages(chapter: MangaChapter): List { - val fullUrl = chapter.url.toAbsoluteUrl(getDomain()) - val doc = context.httpGet(fullUrl).parseHtml() - val root = doc.body().selectOrThrow("div.item img.owl-lazy") - return root.map { div -> - val url = div?.attrAsRelativeUrlOrNull("data-src") ?: doc.parseFailed("Page image not found") - MangaPage( - id = generateUid(url), - url = url, - preview = null, - referer = url, - source = MangaSource.MANGAOWL, - ) - } - } - - private fun parseStatus(status: String?) = when { - status == null -> null - status.contains("Ongoing") -> MangaState.ONGOING - status.contains("Completed") -> MangaState.FINISHED - else -> null - } - - override suspend fun getTags(): Set { - val doc = context.httpGet("https://${getDomain()}/").parseHtml() - val root = doc.body().select("ul.dropdown-menu.multi-column.columns-3").select("li") - return root.mapToSet { p -> - val a = p.selectFirstOrThrow("a") - MangaTag( - title = a.text().toTitleCase(), - key = a.attr("href"), - source = source, - ) - } - } - - private fun getSortKey(sortOrder: SortOrder) = - when (sortOrder) { - SortOrder.POPULARITY -> "popular" - SortOrder.NEWEST -> "new_release" - SortOrder.UPDATED -> "lastest" - else -> "lastest" - } - - private fun getAlternativeSortKey(sortOrder: SortOrder) = - when (sortOrder) { - SortOrder.POPULARITY -> "0" - SortOrder.NEWEST -> "2" - SortOrder.UPDATED -> "3" - else -> "3" - } - - private fun isNsfwGenre(name: String): Boolean = regexNsfw.containsMatchIn(name) -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaTownParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaTownParser.kt index 956231218..dbc5ee623 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaTownParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaTownParser.kt @@ -11,207 +11,207 @@ import java.text.SimpleDateFormat import java.util.* @MangaSourceParser("MANGATOWN", "MangaTown", "en") -internal class MangaTownParser(override val context: MangaLoaderContext) : MangaParser(MangaSource.MANGATOWN) { +internal class MangaTownParser(context: MangaLoaderContext) : MangaParser(context, MangaSource.MANGATOWN) { - override val configKeyDomain = ConfigKey.Domain("www.mangatown.com", null) + override val configKeyDomain = ConfigKey.Domain("www.mangatown.com", null) - override val sortOrders: Set = EnumSet.of( - SortOrder.ALPHABETICAL, - SortOrder.RATING, - SortOrder.POPULARITY, - SortOrder.UPDATED, - ) + override val sortOrders: Set = EnumSet.of( + SortOrder.ALPHABETICAL, + SortOrder.RATING, + SortOrder.POPULARITY, + SortOrder.UPDATED, + ) - private val regexTag = Regex("[^\\-]+-[^\\-]+-[^\\-]+-[^\\-]+-[^\\-]+-[^\\-]+") + private val regexTag = Regex("[^\\-]+-[^\\-]+-[^\\-]+-[^\\-]+-[^\\-]+-[^\\-]+") - override suspend fun getList( - offset: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - val sortKey = when (sortOrder) { - SortOrder.ALPHABETICAL -> "?name.az" - SortOrder.RATING -> "?rating.za" - SortOrder.UPDATED -> "?last_chapter_time.za" - else -> "" - } - val page = (offset / 30) + 1 - val url = when { - !query.isNullOrEmpty() -> { - if (offset != 0) { - return emptyList() - } - "/search?name=${query.urlEncoded()}".toAbsoluteUrl(getDomain()) - } + override suspend fun getList( + offset: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + val sortKey = when (sortOrder) { + SortOrder.ALPHABETICAL -> "?name.az" + SortOrder.RATING -> "?rating.za" + SortOrder.UPDATED -> "?last_chapter_time.za" + else -> "" + } + val page = (offset / 30) + 1 + val url = when { + !query.isNullOrEmpty() -> { + if (offset != 0) { + return emptyList() + } + "/search?name=${query.urlEncoded()}".toAbsoluteUrl(domain) + } - tags.isNullOrEmpty() -> "/directory/$page.htm$sortKey".toAbsoluteUrl(getDomain()) - tags.size == 1 -> "/directory/${tags.first().key}/$page.htm$sortKey".toAbsoluteUrl(getDomain()) - else -> tags.joinToString( - prefix = "/search?page=$page".toAbsoluteUrl(getDomain()), - ) { tag -> - "&genres[${tag.key}]=1" - } - } - val doc = context.httpGet(url).parseHtml() - val root = doc.body().selectFirstOrThrow("ul.manga_pic_list") - return root.select("li").mapNotNull { li -> - val a = li.selectFirst("a.manga_cover") - val href = a?.attrAsRelativeUrlOrNull("href") - ?: return@mapNotNull null - val views = li.select("p.view") - val status = views.firstNotNullOfOrNull { it.ownText().takeIf { x -> x.startsWith("Status:") } } - ?.substringAfter(':')?.trim()?.lowercase(Locale.ROOT) - Manga( - id = generateUid(href), - title = a.attr("title"), - coverUrl = a.selectFirst("img")?.absUrl("src").orEmpty(), - source = MangaSource.MANGATOWN, - altTitle = null, - rating = li.selectFirst("p.score")?.selectFirst("b") - ?.ownText()?.toFloatOrNull()?.div(5f) ?: RATING_UNKNOWN, - author = views.firstNotNullOfOrNull { it.text().takeIf { x -> x.startsWith("Author:") } } - ?.substringAfter(':') - ?.trim(), - state = when (status) { - "ongoing" -> MangaState.ONGOING - "completed" -> MangaState.FINISHED - else -> null - }, - tags = li.selectFirst("p.keyWord")?.select("a")?.mapNotNullToSet tags@{ x -> - MangaTag( - title = x.attr("title").toTitleCase(), - key = x.attr("href").parseTagKey() ?: return@tags null, - source = MangaSource.MANGATOWN, - ) - }.orEmpty(), - url = href, - isNsfw = false, - publicUrl = href.toAbsoluteUrl(a.host ?: getDomain()), - ) - } - } + tags.isNullOrEmpty() -> "/directory/$page.htm$sortKey".toAbsoluteUrl(domain) + tags.size == 1 -> "/directory/${tags.first().key}/$page.htm$sortKey".toAbsoluteUrl(domain) + else -> tags.joinToString( + prefix = "/search?page=$page".toAbsoluteUrl(domain), + ) { tag -> + "&genres[${tag.key}]=1" + } + } + val doc = webClient.httpGet(url).parseHtml() + val root = doc.body().selectFirstOrThrow("ul.manga_pic_list") + return root.select("li").mapNotNull { li -> + val a = li.selectFirst("a.manga_cover") + val href = a?.attrAsRelativeUrlOrNull("href") + ?: return@mapNotNull null + val views = li.select("p.view") + val status = views.firstNotNullOfOrNull { it.ownText().takeIf { x -> x.startsWith("Status:") } } + ?.substringAfter(':')?.trim()?.lowercase(Locale.ROOT) + Manga( + id = generateUid(href), + title = a.attr("title"), + coverUrl = a.selectFirst("img")?.absUrl("src").orEmpty(), + source = MangaSource.MANGATOWN, + altTitle = null, + rating = li.selectFirst("p.score")?.selectFirst("b") + ?.ownText()?.toFloatOrNull()?.div(5f) ?: RATING_UNKNOWN, + author = views.firstNotNullOfOrNull { it.text().takeIf { x -> x.startsWith("Author:") } } + ?.substringAfter(':') + ?.trim(), + state = when (status) { + "ongoing" -> MangaState.ONGOING + "completed" -> MangaState.FINISHED + else -> null + }, + tags = li.selectFirst("p.keyWord")?.select("a")?.mapNotNullToSet tags@{ x -> + MangaTag( + title = x.attr("title").toTitleCase(), + key = x.attr("href").parseTagKey() ?: return@tags null, + source = MangaSource.MANGATOWN, + ) + }.orEmpty(), + url = href, + isNsfw = false, + publicUrl = href.toAbsoluteUrl(a.host ?: domain), + ) + } + } - override suspend fun getDetails(manga: Manga): Manga { - val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml() - val root = doc.body().selectFirstOrThrow("section.main") - .selectFirstOrThrow("div.article_content") - val info = root.selectFirst("div.detail_info")?.selectFirst("ul") - val chaptersList = root.selectFirst("div.chapter_content") - ?.selectFirst("ul.chapter_list")?.select("li")?.asReversed() - val dateFormat = SimpleDateFormat("MMM dd,yyyy", Locale.US) - return manga.copy( - tags = manga.tags + info?.select("li")?.find { x -> - x.selectFirst("b")?.ownText() == "Genre(s):" - }?.select("a")?.mapNotNull { a -> - MangaTag( - title = a.attr("title").toTitleCase(), - key = a.attr("href").parseTagKey() ?: return@mapNotNull null, - source = MangaSource.MANGATOWN, - ) - }.orEmpty(), - description = info?.getElementById("show")?.ownText(), - chapters = chaptersList?.mapChapters { i, li -> - val href = li.selectFirst("a")?.attrAsRelativeUrlOrNull("href") - ?: return@mapChapters null - val name = li.select("span") - .filter { x -> x.className().isEmpty() } - .joinToString(" - ") { it.text() }.trim() - MangaChapter( - id = generateUid(href), - url = href, - source = MangaSource.MANGATOWN, - number = i + 1, - uploadDate = parseChapterDate( - dateFormat, - li.selectFirst("span.time")?.text(), - ), - name = name.ifEmpty { "${manga.title} - ${i + 1}" }, - scanlator = null, - branch = null, - ) - } ?: bypassLicensedChapters(manga), - ) - } + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val root = doc.body().selectFirstOrThrow("section.main") + .selectFirstOrThrow("div.article_content") + val info = root.selectFirst("div.detail_info")?.selectFirst("ul") + val chaptersList = root.selectFirst("div.chapter_content") + ?.selectFirst("ul.chapter_list")?.select("li")?.asReversed() + val dateFormat = SimpleDateFormat("MMM dd,yyyy", Locale.US) + return manga.copy( + tags = manga.tags + info?.select("li")?.find { x -> + x.selectFirst("b")?.ownText() == "Genre(s):" + }?.select("a")?.mapNotNull { a -> + MangaTag( + title = a.attr("title").toTitleCase(), + key = a.attr("href").parseTagKey() ?: return@mapNotNull null, + source = MangaSource.MANGATOWN, + ) + }.orEmpty(), + description = info?.getElementById("show")?.ownText(), + chapters = chaptersList?.mapChapters { i, li -> + val href = li.selectFirst("a")?.attrAsRelativeUrlOrNull("href") + ?: return@mapChapters null + val name = li.select("span") + .filter { x -> x.className().isEmpty() } + .joinToString(" - ") { it.text() }.trim() + MangaChapter( + id = generateUid(href), + url = href, + source = MangaSource.MANGATOWN, + number = i + 1, + uploadDate = parseChapterDate( + dateFormat, + li.selectFirst("span.time")?.text(), + ), + name = name.ifEmpty { "${manga.title} - ${i + 1}" }, + scanlator = null, + branch = null, + ) + } ?: bypassLicensedChapters(manga), + ) + } - override suspend fun getPages(chapter: MangaChapter): List { - val fullUrl = chapter.url.toAbsoluteUrl(getDomain()) - val doc = context.httpGet(fullUrl).parseHtml() - val root = doc.body().selectFirstOrThrow("div.page_select") - return root.selectFirstOrThrow("select").selectOrThrow("option").mapNotNull { - val href = it.attrAsRelativeUrlOrNull("value") - if (href == null || href.endsWith("featured.html")) { - return@mapNotNull null - } - MangaPage( - id = generateUid(href), - url = href, - preview = null, - referer = fullUrl, - source = MangaSource.MANGATOWN, - ) - } - } + override suspend fun getPages(chapter: MangaChapter): List { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + val root = doc.body().selectFirstOrThrow("div.page_select") + return root.selectFirstOrThrow("select").selectOrThrow("option").mapNotNull { + val href = it.attrAsRelativeUrlOrNull("value") + if (href == null || href.endsWith("featured.html")) { + return@mapNotNull null + } + MangaPage( + id = generateUid(href), + url = href, + preview = null, + referer = fullUrl, + source = MangaSource.MANGATOWN, + ) + } + } - override suspend fun getPageUrl(page: MangaPage): String { - val doc = context.httpGet(page.url.toAbsoluteUrl(getDomain())).parseHtml() - return doc.requireElementById("image").attrAsAbsoluteUrl("src") - } + override suspend fun getPageUrl(page: MangaPage): String { + val doc = webClient.httpGet(page.url.toAbsoluteUrl(domain)).parseHtml() + return doc.requireElementById("image").attrAsAbsoluteUrl("src") + } - override suspend fun getTags(): Set { - val doc = context.httpGet("/directory/".toAbsoluteUrl(getDomain())).parseHtml() - val root = doc.body().selectFirst("aside.right") - ?.getElementsContainingOwnText("Genres") - ?.first() - ?.nextElementSibling() ?: doc.parseFailed("Root not found") - return root.select("li").mapNotNullToSet { li -> - val a = li.selectFirst("a") ?: return@mapNotNullToSet null - val key = a.attr("href").parseTagKey() - if (key.isNullOrEmpty()) { - return@mapNotNullToSet null - } - MangaTag( - source = MangaSource.MANGATOWN, - key = key, - title = a.text().toTitleCase(), - ) - } - } + override suspend fun getTags(): Set { + val doc = webClient.httpGet("/directory/".toAbsoluteUrl(domain)).parseHtml() + val root = doc.body().selectFirst("aside.right") + ?.getElementsContainingOwnText("Genres") + ?.first() + ?.nextElementSibling() ?: doc.parseFailed("Root not found") + return root.select("li").mapNotNullToSet { li -> + val a = li.selectFirst("a") ?: return@mapNotNullToSet null + val key = a.attr("href").parseTagKey() + if (key.isNullOrEmpty()) { + return@mapNotNullToSet null + } + MangaTag( + source = MangaSource.MANGATOWN, + key = key, + title = a.text().toTitleCase(), + ) + } + } - private fun parseChapterDate(dateFormat: DateFormat, date: String?): Long { - return when { - date.isNullOrEmpty() -> 0L - date.contains("Today") -> Calendar.getInstance().timeInMillis - date.contains("Yesterday") -> Calendar.getInstance().apply { add(Calendar.DAY_OF_MONTH, -1) }.timeInMillis - else -> dateFormat.tryParse(date) - } - } + private fun parseChapterDate(dateFormat: DateFormat, date: String?): Long { + return when { + date.isNullOrEmpty() -> 0L + date.contains("Today") -> Calendar.getInstance().timeInMillis + date.contains("Yesterday") -> Calendar.getInstance().apply { add(Calendar.DAY_OF_MONTH, -1) }.timeInMillis + else -> dateFormat.tryParse(date) + } + } - private suspend fun bypassLicensedChapters(manga: Manga): List { - val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain("m"))).parseHtml() - val list = doc.body().selectFirst("ul.detail-ch-list") ?: return emptyList() - val dateFormat = SimpleDateFormat("MMM dd,yyyy", Locale.US) - return list.select("li").asReversed().mapIndexedNotNull { i, li -> - val a = li.selectFirst("a") ?: return@mapIndexedNotNull null - val href = a.attrAsRelativeUrl("href") - val name = a.selectFirst("span.vol")?.text().orEmpty().ifEmpty { - a.ownText() - } - MangaChapter( - id = generateUid(href), - url = href, - source = MangaSource.MANGATOWN, - number = i + 1, - uploadDate = parseChapterDate( - dateFormat, - li.selectFirst("span.time")?.text(), - ), - name = name.ifEmpty { "${manga.title} - ${i + 1}" }, - scanlator = null, - branch = null, - ) - } - } + private suspend fun bypassLicensedChapters(manga: Manga): List { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(getDomain("m"))).parseHtml() + val list = doc.body().selectFirst("ul.detail-ch-list") ?: return emptyList() + val dateFormat = SimpleDateFormat("MMM dd,yyyy", Locale.US) + return list.select("li").asReversed().mapIndexedNotNull { i, li -> + val a = li.selectFirst("a") ?: return@mapIndexedNotNull null + val href = a.attrAsRelativeUrl("href") + val name = a.selectFirst("span.vol")?.text().orEmpty().ifEmpty { + a.ownText() + } + MangaChapter( + id = generateUid(href), + url = href, + source = MangaSource.MANGATOWN, + number = i + 1, + uploadDate = parseChapterDate( + dateFormat, + li.selectFirst("span.time")?.text(), + ), + name = name.ifEmpty { "${manga.title} - ${i + 1}" }, + scanlator = null, + branch = null, + ) + } + } - private fun String.parseTagKey() = split('/').findLast { regexTag matches it } -} \ No newline at end of file + private fun String.parseTagKey() = split('/').findLast { regexTag matches it } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/Manhwa18Parser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/Manhwa18Parser.kt index 0f8b82b1f..97a40b944 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/Manhwa18Parser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/Manhwa18Parser.kt @@ -9,173 +9,177 @@ import org.koitharu.kotatsu.parsers.util.* import java.util.* @MangaSourceParser("MANHWA18", "Manhwa18", "en") -class Manhwa18Parser(override val context: MangaLoaderContext) : PagedMangaParser(MangaSource.MANHWA18, pageSize = 20, searchPageSize = 20) { +class Manhwa18Parser(context: MangaLoaderContext) : + PagedMangaParser(context, MangaSource.MANHWA18, pageSize = 20, searchPageSize = 20) { - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("manhwa18.net", null) + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("manhwa18.net", null) - override val sortOrders: Set - get() = EnumSet.of(SortOrder.UPDATED, SortOrder.POPULARITY, SortOrder.ALPHABETICAL) + override val sortOrders: Set + get() = EnumSet.of(SortOrder.UPDATED, SortOrder.POPULARITY, SortOrder.ALPHABETICAL) - override suspend fun getFavicons(): Favicons { - return Favicons(listOf( - Favicon("https://${getDomain()}/uploads/logos/logo-mini.png", 92, null) - ), getDomain()) - } + override suspend fun getFavicons(): Favicons { + return Favicons( + listOf( + Favicon("https://${domain}/uploads/logos/logo-mini.png", 92, null), + ), + domain, + ) + } - override suspend fun getDetails(manga: Manga): Manga { - val docs = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml() - val cardInfoElement = docs.selectFirst(".card .manga-info") - val author = cardInfoElement?.selectFirst("b:contains(Author(s))")?.parent() - ?.select("a.btn") - ?.joinToString(", ") { it.text() } - val tags = cardInfoElement?.selectFirst("b:contains(Genre(s))")?.parent() - ?.select("a.btn") - ?.mapToSet { MangaTag(it.text(), it.text().lowercase(), MangaSource.MANHWA18) } - val state = cardInfoElement?.selectFirst("b:contains(Status)")?.parent() - ?.selectFirst("a.btn") - ?.let { - when (it.text()) { - "On going" -> MangaState.ONGOING - "Completed" -> MangaState.FINISHED - else -> null - } - } + override suspend fun getDetails(manga: Manga): Manga { + val docs = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val cardInfoElement = docs.selectFirst(".card .manga-info") + val author = cardInfoElement?.selectFirst("b:contains(Author(s))")?.parent() + ?.select("a.btn") + ?.joinToString(", ") { it.text() } + val tags = cardInfoElement?.selectFirst("b:contains(Genre(s))")?.parent() + ?.select("a.btn") + ?.mapToSet { MangaTag(it.text(), it.text().lowercase(), MangaSource.MANHWA18) } + val state = cardInfoElement?.selectFirst("b:contains(Status)")?.parent() + ?.selectFirst("a.btn") + ?.let { + when (it.text()) { + "On going" -> MangaState.ONGOING + "Completed" -> MangaState.FINISHED + else -> null + } + } - return manga.copy( - altTitle = cardInfoElement?.selectFirst("b:contains(Other names)")?.parent()?.ownText()?.removePrefix(": "), - author = author, - description = docs.selectFirst(".series-summary .summary-content")?.html(), - tags = tags.orEmpty(), - state = state, - chapters = docs.select(".card-body > .list-chapters > a").asReversed().mapChapters { index, element -> - // attrAsRelativeUrl only return page url without the '/' - val chapterUrl = element.attrAsAbsoluteUrlOrNull("href")?.toRelativeUrl(getDomain()) - ?: return@mapChapters null - val uploadDate = parseUploadDate(element.selectFirst(".chapter-time")?.text()) - MangaChapter( - id = generateUid(chapterUrl), - name = element.selectFirst(".chapter-name")?.text().orEmpty(), - number = index + 1, - url = chapterUrl, - scanlator = null, - uploadDate = uploadDate, - branch = null, - source = MangaSource.MANHWA18, - ) - } - ) - } + return manga.copy( + altTitle = cardInfoElement?.selectFirst("b:contains(Other names)")?.parent()?.ownText()?.removePrefix(": "), + author = author, + description = docs.selectFirst(".series-summary .summary-content")?.html(), + tags = tags.orEmpty(), + state = state, + chapters = docs.select(".card-body > .list-chapters > a").asReversed().mapChapters { index, element -> + // attrAsRelativeUrl only return page url without the '/' + val chapterUrl = element.attrAsAbsoluteUrlOrNull("href")?.toRelativeUrl(domain) + ?: return@mapChapters null + val uploadDate = parseUploadDate(element.selectFirst(".chapter-time")?.text()) + MangaChapter( + id = generateUid(chapterUrl), + name = element.selectFirst(".chapter-name")?.text().orEmpty(), + number = index + 1, + url = chapterUrl, + scanlator = null, + uploadDate = uploadDate, + branch = null, + source = MangaSource.MANHWA18, + ) + }, + ) + } - // 7 minutes ago - // 5 hours ago - // 2 days ago - // 2 weeks ago - // 4 years ago - private fun parseUploadDate(timeStr: String?): Long { - timeStr ?: return 0 + // 7 minutes ago + // 5 hours ago + // 2 days ago + // 2 weeks ago + // 4 years ago + private fun parseUploadDate(timeStr: String?): Long { + timeStr ?: return 0 - val timeWords = timeStr.split(' ') - if (timeWords.size != 3) return 0 - val timeWord = timeWords[1] - val timeAmount = timeWords[0].toIntOrNull() ?: return 0 - val timeUnit = when (timeWord) { - "minute", "minutes" -> Calendar.MINUTE - "hour", "hours" -> Calendar.HOUR - "day", "days" -> Calendar.DAY_OF_YEAR - "week", "weeks" -> Calendar.WEEK_OF_YEAR - "month", "months" -> Calendar.MONTH - "year", "years" -> Calendar.YEAR - else -> return 0 - } - val cal = Calendar.getInstance() - cal.add(timeUnit, -timeAmount) - return cal.time.time - } + val timeWords = timeStr.split(' ') + if (timeWords.size != 3) return 0 + val timeWord = timeWords[1] + val timeAmount = timeWords[0].toIntOrNull() ?: return 0 + val timeUnit = when (timeWord) { + "minute", "minutes" -> Calendar.MINUTE + "hour", "hours" -> Calendar.HOUR + "day", "days" -> Calendar.DAY_OF_YEAR + "week", "weeks" -> Calendar.WEEK_OF_YEAR + "month", "months" -> Calendar.MONTH + "year", "years" -> Calendar.YEAR + else -> return 0 + } + val cal = Calendar.getInstance() + cal.add(timeUnit, -timeAmount) + return cal.time.time + } - override suspend fun getListPage( - page: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - val sortQuery = when(sortOrder) { - SortOrder.ALPHABETICAL -> "name" - SortOrder.POPULARITY -> "views" - SortOrder.UPDATED -> "last_update" - else -> "" - } + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + val sortQuery = when (sortOrder) { + SortOrder.ALPHABETICAL -> "name" + SortOrder.POPULARITY -> "views" + SortOrder.UPDATED -> "last_update" + else -> "" + } - val sortType = if (sortOrder == SortOrder.ALPHABETICAL) "ASC" else "DESC" - val tagQuery = tags?.joinToString(",") { it.key }.orEmpty() - val url = buildString { - append("https://") - append(getDomain()) - append("/manga-list.html?listType=pagination&page=") - append(page) - append("&artist=&author=&group=&m_status=&name=") - append(query?.urlEncoded().orEmpty()) - append("&genre=$tagQuery") - append("&ungenre=") - append("&sort=") - append(sortQuery) - append("&sort_type=") - append(sortType) - } + val sortType = if (sortOrder == SortOrder.ALPHABETICAL) "ASC" else "DESC" + val tagQuery = tags?.joinToString(",") { it.key }.orEmpty() + val url = buildString { + append("https://") + append(domain) + append("/manga-list.html?listType=pagination&page=") + append(page) + append("&artist=&author=&group=&m_status=&name=") + append(query?.urlEncoded().orEmpty()) + append("&genre=$tagQuery") + append("&ungenre=") + append("&sort=") + append(sortQuery) + append("&sort_type=") + append(sortType) + } - val docs = context.httpGet(url).parseHtml() - val actualPage = docs.selectFirst("ul.pagination a.active")?.text()?.toIntOrNull() - if (actualPage != page) { - return emptyList() - } + val docs = webClient.httpGet(url).parseHtml() + val actualPage = docs.selectFirst("ul.pagination a.active")?.text()?.toIntOrNull() + if (actualPage != page) { + return emptyList() + } - return docs.select(".card-body .thumb-item-flow") - .map { - val titleElement = it.selectFirstOrThrow(".thumb_attr.series-title > a") - val absUrl = titleElement.attrAsAbsoluteUrl("href") - Manga( - id = generateUid(absUrl.toRelativeUrl(getDomain())), - title = titleElement.text(), - altTitle = null, - url = absUrl.toRelativeUrl(getDomain()), - publicUrl = absUrl, - rating = RATING_UNKNOWN, - isNsfw = true, - coverUrl = it.selectFirst("div.img-in-ratio")?.attrAsAbsoluteUrl("data-bg").orEmpty(), - tags = emptySet(), - state = null, - author = null, - largeCoverUrl = null, - description = null, - source = MangaSource.MANHWA18, - ) - } - } + return docs.select(".card-body .thumb-item-flow") + .map { + val titleElement = it.selectFirstOrThrow(".thumb_attr.series-title > a") + val absUrl = titleElement.attrAsAbsoluteUrl("href") + Manga( + id = generateUid(absUrl.toRelativeUrl(domain)), + title = titleElement.text(), + altTitle = null, + url = absUrl.toRelativeUrl(domain), + publicUrl = absUrl, + rating = RATING_UNKNOWN, + isNsfw = true, + coverUrl = it.selectFirst("div.img-in-ratio")?.attrAsAbsoluteUrl("data-bg").orEmpty(), + tags = emptySet(), + state = null, + author = null, + largeCoverUrl = null, + description = null, + source = MangaSource.MANHWA18, + ) + } + } - override suspend fun getPages(chapter: MangaChapter): List { - val chapterUrl = chapter.url.toAbsoluteUrl(getDomain()) - return context.httpGet(chapterUrl).parseHtml() - .select("div.chapter-content > img").map { - val url = it.attrAsRelativeUrlOrNull("src").orEmpty() - MangaPage( - id = generateUid(url), - url = url, - referer = chapterUrl, - preview = null, - source = MangaSource.MANHWA18, - ) - } - } + override suspend fun getPages(chapter: MangaChapter): List { + val chapterUrl = chapter.url.toAbsoluteUrl(domain) + return webClient.httpGet(chapterUrl).parseHtml() + .select("div.chapter-content > img").map { + val url = it.attrAsRelativeUrlOrNull("src").orEmpty() + MangaPage( + id = generateUid(url), + url = url, + referer = chapterUrl, + preview = null, + source = MangaSource.MANHWA18, + ) + } + } - override suspend fun getTags(): Set { - return context.httpGet("https://${getDomain()}/").parseHtml().selectFirstOrThrow(".genres-menu") - .select("a.genres-item").orEmpty() - .mapToSet { - MangaTag( - title = it.text(), - key = it.text().lowercase(), - source = MangaSource.MANHWA18, - ) - } - } -} \ No newline at end of file + override suspend fun getTags(): Set { + return webClient.httpGet("https://${domain}/").parseHtml().selectFirstOrThrow(".genres-menu") + .select("a.genres-item").orEmpty() + .mapToSet { + MangaTag( + title = it.text(), + key = it.text().lowercase(), + source = MangaSource.MANHWA18, + ) + } + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NHentaiParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NHentaiParser.kt index 42170f497..5ca2f515a 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NHentaiParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NHentaiParser.kt @@ -15,174 +15,174 @@ import java.text.SimpleDateFormat import java.util.* @MangaSourceParser("NHENTAI", "N-Hentai") -class NHentaiParser(override val context: MangaLoaderContext) : PagedMangaParser(MangaSource.NHENTAI, pageSize = 25) { +class NHentaiParser(context: MangaLoaderContext) : PagedMangaParser(context, MangaSource.NHENTAI, pageSize = 25) { - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("nhentai.net", null) + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("nhentai.net", null) - override val sortOrders: Set - get() = EnumSet.of(SortOrder.NEWEST, SortOrder.POPULARITY) + override val sortOrders: Set + get() = EnumSet.of(SortOrder.NEWEST, SortOrder.POPULARITY) - override suspend fun getListPage( - page: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - if (query.isNullOrEmpty() && tags != null && tags.size > 1) { - return getListPage(page, buildQuery(tags), emptySet(), sortOrder) - } - val domain = getDomain() - val url = buildString { - append("https://") - append(domain) - if (!query.isNullOrEmpty()) { - append("/search/?q=") - append(query.urlEncoded()) - append("&page=") - append(page) - if (sortOrder == SortOrder.POPULARITY) { - append("&sort=popular") - } - } else { - append('/') - if (!tags.isNullOrEmpty()) { - val tag = tags.single() - append("tag/") - append(tag.key) - append('/') - if (sortOrder == SortOrder.POPULARITY) { - append("popular") - } - append("?page=") - append(page) - } else { - if (sortOrder == SortOrder.POPULARITY) { - append("?sort=popular&page=") - } else { - append("?page=") - } - append(page) - } - } - } - val root = context.httpGet(url).parseHtml().body().requireElementById("content") - .selectLastOrThrow("div.index-container") - val regexBrackets = Regex("\\[[^]]+]|\\([^)]+\\)") - val regexSpaces = Regex("\\s+") - return root.select(".gallery").map { div -> - val a = div.selectFirstOrThrow("a.cover") - val href = a.attrAsRelativeUrl("href") - val img = div.selectFirstOrThrow("img") - val title = div.selectFirstOrThrow(".caption").text() - Manga( - id = generateUid(href), - title = title.replace(regexBrackets, "") - .replace(regexSpaces, " ") - .trim(), - altTitle = null, - url = href, - publicUrl = href.toAbsoluteUrl(domain), - rating = RATING_UNKNOWN, - isNsfw = true, - coverUrl = img.attrAsAbsoluteUrlOrNull("data-src") - ?: img.attrAsAbsoluteUrl("src"), - tags = setOf(), - state = null, - author = null, - largeCoverUrl = null, - description = null, - chapters = listOf(), - source = source, - ) - } - } + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + if (query.isNullOrEmpty() && tags != null && tags.size > 1) { + return getListPage(page, buildQuery(tags), emptySet(), sortOrder) + } + val domain = domain + val url = buildString { + append("https://") + append(domain) + if (!query.isNullOrEmpty()) { + append("/search/?q=") + append(query.urlEncoded()) + append("&page=") + append(page) + if (sortOrder == SortOrder.POPULARITY) { + append("&sort=popular") + } + } else { + append('/') + if (!tags.isNullOrEmpty()) { + val tag = tags.single() + append("tag/") + append(tag.key) + append('/') + if (sortOrder == SortOrder.POPULARITY) { + append("popular") + } + append("?page=") + append(page) + } else { + if (sortOrder == SortOrder.POPULARITY) { + append("?sort=popular&page=") + } else { + append("?page=") + } + append(page) + } + } + } + val root = webClient.httpGet(url).parseHtml().body().requireElementById("content") + .selectLastOrThrow("div.index-container") + val regexBrackets = Regex("\\[[^]]+]|\\([^)]+\\)") + val regexSpaces = Regex("\\s+") + return root.select(".gallery").map { div -> + val a = div.selectFirstOrThrow("a.cover") + val href = a.attrAsRelativeUrl("href") + val img = div.selectFirstOrThrow("img") + val title = div.selectFirstOrThrow(".caption").text() + Manga( + id = generateUid(href), + title = title.replace(regexBrackets, "") + .replace(regexSpaces, " ") + .trim(), + altTitle = null, + url = href, + publicUrl = href.toAbsoluteUrl(domain), + rating = RATING_UNKNOWN, + isNsfw = true, + coverUrl = img.attrAsAbsoluteUrlOrNull("data-src") + ?: img.attrAsAbsoluteUrl("src"), + tags = setOf(), + state = null, + author = null, + largeCoverUrl = null, + description = null, + chapters = listOf(), + source = source, + ) + } + } - override suspend fun getDetails(manga: Manga): Manga { - val root = context.httpGet( - url = manga.url.toAbsoluteUrl(getDomain()), - ).parseHtml().body().requireElementById("bigcontainer") - val img = root.requireElementById("cover").selectFirstOrThrow("img") - val tagContainers = root.requireElementById("tags").select(".tag-container") - val dateFormat = SimpleDateFormat( - "yyyy-MM-dd'T'HH:mm:ss.SSSSSS'+00:00'", - Locale.ROOT, - ) - return manga.copy( - tags = tagContainers.find { x -> x.ownText() == "Tags:" }?.parseTags() ?: manga.tags, - author = tagContainers.find { x -> x.ownText() == "Artists:" } - ?.selectFirst("span.name")?.text()?.toCamelCase(), - largeCoverUrl = img.attrAsAbsoluteUrlOrNull("data-src") - ?: img.attrAsAbsoluteUrl("src"), - description = null, - chapters = listOf( - MangaChapter( - id = manga.id, - name = manga.title, - number = 1, - url = manga.url, - scanlator = null, - uploadDate = dateFormat.tryParse( - tagContainers.find { x -> x.ownText() == "Uploaded:" } - ?.selectFirst("time") - ?.attr("datetime"), - ), - branch = null, - source = source, - ), - ), - ) - } + override suspend fun getDetails(manga: Manga): Manga { + val root = webClient.httpGet( + url = manga.url.toAbsoluteUrl(domain), + ).parseHtml().body().requireElementById("bigcontainer") + val img = root.requireElementById("cover").selectFirstOrThrow("img") + val tagContainers = root.requireElementById("tags").select(".tag-container") + val dateFormat = SimpleDateFormat( + "yyyy-MM-dd'T'HH:mm:ss.SSSSSS'+00:00'", + Locale.ROOT, + ) + return manga.copy( + tags = tagContainers.find { x -> x.ownText() == "Tags:" }?.parseTags() ?: manga.tags, + author = tagContainers.find { x -> x.ownText() == "Artists:" } + ?.selectFirst("span.name")?.text()?.toCamelCase(), + largeCoverUrl = img.attrAsAbsoluteUrlOrNull("data-src") + ?: img.attrAsAbsoluteUrl("src"), + description = null, + chapters = listOf( + MangaChapter( + id = manga.id, + name = manga.title, + number = 1, + url = manga.url, + scanlator = null, + uploadDate = dateFormat.tryParse( + tagContainers.find { x -> x.ownText() == "Uploaded:" } + ?.selectFirst("time") + ?.attr("datetime"), + ), + branch = null, + source = source, + ), + ), + ) + } - override suspend fun getPages(chapter: MangaChapter): List { - val url = chapter.url.toAbsoluteUrl(getDomain()) - val root = context.httpGet(url).parseHtml().requireElementById("thumbnail-container") - return root.select(".thumb-container").map { div -> - val a = div.selectFirstOrThrow("a") - val img = div.selectFirstOrThrow("img") - val href = a.attrAsRelativeUrl("href") - MangaPage( - id = generateUid(href), - url = href, - referer = url, - preview = img.attrAsAbsoluteUrlOrNull("data-src") - ?: img.attrAsAbsoluteUrl("src"), - source = source, - ) - } - } + override suspend fun getPages(chapter: MangaChapter): List { + val url = chapter.url.toAbsoluteUrl(domain) + val root = webClient.httpGet(url).parseHtml().requireElementById("thumbnail-container") + return root.select(".thumb-container").map { div -> + val a = div.selectFirstOrThrow("a") + val img = div.selectFirstOrThrow("img") + val href = a.attrAsRelativeUrl("href") + MangaPage( + id = generateUid(href), + url = href, + referer = url, + preview = img.attrAsAbsoluteUrlOrNull("data-src") + ?: img.attrAsAbsoluteUrl("src"), + source = source, + ) + } + } - override suspend fun getPageUrl(page: MangaPage): String { - val root = context.httpGet(page.url.toAbsoluteUrl(getDomain())).parseHtml().body() - .requireElementById("image-container") - return root.selectFirstOrThrow("img").attrAsAbsoluteUrl("src") - } + override suspend fun getPageUrl(page: MangaPage): String { + val root = webClient.httpGet(page.url.toAbsoluteUrl(domain)).parseHtml().body() + .requireElementById("image-container") + return root.selectFirstOrThrow("img").attrAsAbsoluteUrl("src") + } - override suspend fun getTags(): Set { - return coroutineScope { - // parse first 3 pages of tags - (1..3).map { page -> - async { getTags(page) } - } - }.awaitAll().flattenTo(ArraySet(360)) - } + override suspend fun getTags(): Set { + return coroutineScope { + // parse first 3 pages of tags + (1..3).map { page -> + async { getTags(page) } + } + }.awaitAll().flattenTo(ArraySet(360)) + } - private suspend fun getTags(page: Int): Set { - val root = context.httpGet("https://${getDomain()}/tags/popular?page=$page").parseHtml().body() - .getElementById("tag-container") - return root?.parseTags().orEmpty() - } + private suspend fun getTags(page: Int): Set { + val root = webClient.httpGet("https://${domain}/tags/popular?page=$page").parseHtml().body() + .getElementById("tag-container") + return root?.parseTags().orEmpty() + } - private fun Element.parseTags() = select("a.tag").mapToSet { a -> - val href = a.attr("href").removeSuffix('/') - MangaTag( - title = a.selectFirstOrThrow(".name").text().toTitleCase(), - key = href.substringAfterLast('/'), - source = source, - ) - } + private fun Element.parseTags() = select("a.tag").mapToSet { a -> + val href = a.attr("href").removeSuffix('/') + MangaTag( + title = a.selectFirstOrThrow(".name").text().toTitleCase(), + key = href.substringAfterLast('/'), + source = source, + ) + } - private fun buildQuery(tags: Collection) = tags.joinToString(separator = " ") { tag -> - "tag:\"${tag.key}\"" - } -} \ No newline at end of file + private fun buildQuery(tags: Collection) = tags.joinToString(separator = " ") { tag -> + "tag:\"${tag.key}\"" + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NetTruyenParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NetTruyenParser.kt index afe8664fe..aaf4c956a 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NetTruyenParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NetTruyenParser.kt @@ -15,210 +15,210 @@ import java.text.SimpleDateFormat import java.util.* @MangaSourceParser("NETTRUYEN", "NetTruyen", "vi") -class NetTruyenParser(override val context: MangaLoaderContext) : - PagedMangaParser(MangaSource.NETTRUYEN, pageSize = 36) { - - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("nettruyenin.com", null) - - override val sortOrders: Set - get() = EnumSet.of(SortOrder.UPDATED, SortOrder.POPULARITY, SortOrder.NEWEST, SortOrder.RATING) - - private val mutex = Mutex() - private val dateFormat = SimpleDateFormat("dd/MM/yy", Locale.US) - private var tagCache: ArrayMap? = null - - override suspend fun getDetails(manga: Manga): Manga { - val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml() - val rating = doc.selectFirst("span[itemprop=ratingValue]") - ?.ownText() - ?.toFloatOrNull() ?: 0f - - val chapterElements = doc.getElementById("nt_listchapter")?.select("ul > li") ?: doc.parseFailed() - val chapters = chapterElements.asReversed().mapChapters { index, element -> - val a = element.selectFirst("div.chapter > a") ?: return@mapChapters null - val relativeUrl = a.attrAsRelativeUrlOrNull("href") ?: return@mapChapters null - val timeText = element.selectFirst("div.col-xs-4.text-center.no-wrap.small")?.text() - - MangaChapter( - id = generateUid(relativeUrl), - name = a.text(), - number = index + 1, - url = relativeUrl, - scanlator = null, - uploadDate = parseChapterTime(timeText), - branch = null, - source = source, - ) - } - - return manga.copy( - rating = rating / 5, - chapters = chapters, - description = doc.selectFirst("div.detail-content > p")?.html(), - isNsfw = doc.selectFirst("div.alert.alert-danger > strong:contains(Cảnh báo độ tuổi)") != null, - ) - } - - // 20 giây trước - // 52 phút trước - // 6 giờ trước - // 2 ngày trước - // 19:09 30/07 - // 23/12/21 - private fun parseChapterTime(timeText: String?): Long { - if (timeText.isNullOrEmpty()) { - return 0L - } - - val timeWords = arrayOf("giây", "phút", "giờ", "ngày") - val calendar = Calendar.getInstance() - val timeArr = timeText.split(' ') - if (WordSet(*timeWords).anyWordIn(timeText)) { - val timeSuffix = timeArr.getOrNull(1) - val timeDiff = timeArr.getOrNull(0)?.toIntOrNull() ?: return 0L - when (timeSuffix) { - timeWords[0] -> calendar.add(Calendar.SECOND, -timeDiff) - timeWords[1] -> calendar.add(Calendar.MINUTE, -timeDiff) - timeWords[2] -> calendar.add(Calendar.HOUR, -timeDiff) - timeWords[3] -> calendar.add(Calendar.DATE, -timeDiff) - else -> return 0L - } - } else { - val relativeDate = timeArr.lastOrNull() ?: return 0L - val dateString = when (relativeDate.split('/').size) { - 2 -> { - val currentYear = calendar.get(Calendar.YEAR).toString().takeLast(2) - "$relativeDate/$currentYear" - } - - 3 -> relativeDate - else -> return 0L - } - - calendar.timeInMillis = dateFormat.tryParse(dateString) - } - - - return calendar.time.time - } - - override suspend fun getListPage( - page: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - val isSearching = !query.isNullOrEmpty() - val url = buildString { - append("https://") - append(getDomain()) - if (isSearching) { - append("/tim-truyen?keyword=") - append(query!!.urlEncoded()) - append("&page=") - append(page) - } else { - val tagQuery = tags.orEmpty().joinToString(",") { it.key } - append("/tim-truyen-nang-cao?genres=$tagQuery") - append("¬genres=&gender=-1&status=-1&minchapter=1&sort=${getSortOrderKey(sortOrder)}") - append("&page=$page") - } - } - - val response = if (isSearching) { - val result = runCatchingCancellable { context.httpGet(url) } - val exception = result.exceptionOrNull() - if (exception is NotFoundException) { - return emptyList() - } - - result.getOrThrow() - } else { - context.httpGet(url) - } - - val itemsElements = response.parseHtml() - .select("div.ModuleContent > div.items") - .select("div.item") - return itemsElements.mapNotNull { item -> - val tooltipElement = item.selectFirst("div.box_tootip") ?: return@mapNotNull null - val absUrl = item.selectFirst("div.image > a")?.attrAsAbsoluteUrlOrNull("href") ?: return@mapNotNull null - val slug = absUrl.substringAfterLast('/') - val mangaState = when (tooltipElement.selectFirst("div.message_main > p:contains(Tình trạng)")?.ownText()) { - "Đang tiến hành" -> MangaState.ONGOING - "Hoàn thành" -> MangaState.FINISHED - else -> null - } - - val tagMap = getOrCreateTagMap() - val tagsElement = tooltipElement.selectFirst("div.message_main > p:contains(Thể loại)")?.ownText().orEmpty() - val mangaTags = tagsElement.split(',').mapNotNullToSet { tagMap[it.trim()] } - Manga( - id = generateUid(slug), - title = tooltipElement.selectFirst("div.title")?.text().orEmpty(), - altTitle = null, - url = absUrl.toRelativeUrl(getDomain()), - publicUrl = absUrl, - rating = RATING_UNKNOWN, - isNsfw = false, - coverUrl = item.selectFirst("div.image a img")?.absUrl("data-original").orEmpty(), - largeCoverUrl = null, - tags = mangaTags, - state = mangaState, - author = tooltipElement.selectFirst("div.message_main > p:contains(Tác giả)")?.ownText(), - description = tooltipElement.selectFirst("div.box_text")?.text(), - chapters = null, - source = source, - ) - } - } - - - override suspend fun getPages(chapter: MangaChapter): List { - val pageElements = context.httpGet(chapter.url.toAbsoluteUrl(getDomain())).parseHtml() - .select("div.reading-detail.box_doc > div img") - return pageElements.map { element -> - val url = element.attrAsAbsoluteUrl("data-original") - MangaPage( - id = generateUid(url), - url = url, - referer = getDomain(), - preview = null, - source = source, - ) - } - } - - override suspend fun getTags(): Set { - val map = getOrCreateTagMap() - val tagSet = ArraySet(map.size) - for (entry in map) { - tagSet.add(entry.value) - } - - return tagSet - } - - private suspend fun getOrCreateTagMap(): ArrayMap = mutex.withLock { - tagCache?.let { return@withLock it } - val doc = context.httpGet("/tim-truyen-nang-cao".toAbsoluteUrl(getDomain())).parseHtml() - val tagItems = doc.select("div.genre-item") - val result = ArrayMap(tagItems.size) - for (item in tagItems) { - val title = item.text().trim() - val key = item.select("span[data-id]").attr("data-id") - result[title] = MangaTag(title = title, key = key, source = source) - } - tagCache = result - result - } - - private fun getSortOrderKey(sortOrder: SortOrder) = when (sortOrder) { - SortOrder.UPDATED -> 0 - SortOrder.POPULARITY -> 10 - SortOrder.NEWEST -> 15 - SortOrder.RATING -> 20 - else -> throw IllegalArgumentException("Sort order ${sortOrder.name} not supported") - } +class NetTruyenParser(context: MangaLoaderContext) : + PagedMangaParser(context, MangaSource.NETTRUYEN, pageSize = 36) { + + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("nettruyenin.com", null) + + override val sortOrders: Set + get() = EnumSet.of(SortOrder.UPDATED, SortOrder.POPULARITY, SortOrder.NEWEST, SortOrder.RATING) + + private val mutex = Mutex() + private val dateFormat = SimpleDateFormat("dd/MM/yy", Locale.US) + private var tagCache: ArrayMap? = null + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val rating = doc.selectFirst("span[itemprop=ratingValue]") + ?.ownText() + ?.toFloatOrNull() ?: 0f + + val chapterElements = doc.getElementById("nt_listchapter")?.select("ul > li") ?: doc.parseFailed() + val chapters = chapterElements.asReversed().mapChapters { index, element -> + val a = element.selectFirst("div.chapter > a") ?: return@mapChapters null + val relativeUrl = a.attrAsRelativeUrlOrNull("href") ?: return@mapChapters null + val timeText = element.selectFirst("div.col-xs-4.text-center.no-wrap.small")?.text() + + MangaChapter( + id = generateUid(relativeUrl), + name = a.text(), + number = index + 1, + url = relativeUrl, + scanlator = null, + uploadDate = parseChapterTime(timeText), + branch = null, + source = source, + ) + } + + return manga.copy( + rating = rating / 5, + chapters = chapters, + description = doc.selectFirst("div.detail-content > p")?.html(), + isNsfw = doc.selectFirst("div.alert.alert-danger > strong:contains(Cảnh báo độ tuổi)") != null, + ) + } + + // 20 giây trước + // 52 phút trước + // 6 giờ trước + // 2 ngày trước + // 19:09 30/07 + // 23/12/21 + private fun parseChapterTime(timeText: String?): Long { + if (timeText.isNullOrEmpty()) { + return 0L + } + + val timeWords = arrayOf("giây", "phút", "giờ", "ngày") + val calendar = Calendar.getInstance() + val timeArr = timeText.split(' ') + if (WordSet(*timeWords).anyWordIn(timeText)) { + val timeSuffix = timeArr.getOrNull(1) + val timeDiff = timeArr.getOrNull(0)?.toIntOrNull() ?: return 0L + when (timeSuffix) { + timeWords[0] -> calendar.add(Calendar.SECOND, -timeDiff) + timeWords[1] -> calendar.add(Calendar.MINUTE, -timeDiff) + timeWords[2] -> calendar.add(Calendar.HOUR, -timeDiff) + timeWords[3] -> calendar.add(Calendar.DATE, -timeDiff) + else -> return 0L + } + } else { + val relativeDate = timeArr.lastOrNull() ?: return 0L + val dateString = when (relativeDate.split('/').size) { + 2 -> { + val currentYear = calendar.get(Calendar.YEAR).toString().takeLast(2) + "$relativeDate/$currentYear" + } + + 3 -> relativeDate + else -> return 0L + } + + calendar.timeInMillis = dateFormat.tryParse(dateString) + } + + + return calendar.time.time + } + + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + val isSearching = !query.isNullOrEmpty() + val url = buildString { + append("https://") + append(domain) + if (isSearching) { + append("/tim-truyen?keyword=") + append(query!!.urlEncoded()) + append("&page=") + append(page) + } else { + val tagQuery = tags.orEmpty().joinToString(",") { it.key } + append("/tim-truyen-nang-cao?genres=$tagQuery") + append("¬genres=&gender=-1&status=-1&minchapter=1&sort=${getSortOrderKey(sortOrder)}") + append("&page=$page") + } + } + + val response = if (isSearching) { + val result = runCatchingCancellable { webClient.httpGet(url) } + val exception = result.exceptionOrNull() + if (exception is NotFoundException) { + return emptyList() + } + + result.getOrThrow() + } else { + webClient.httpGet(url) + } + + val itemsElements = response.parseHtml() + .select("div.ModuleContent > div.items") + .select("div.item") + return itemsElements.mapNotNull { item -> + val tooltipElement = item.selectFirst("div.box_tootip") ?: return@mapNotNull null + val absUrl = item.selectFirst("div.image > a")?.attrAsAbsoluteUrlOrNull("href") ?: return@mapNotNull null + val slug = absUrl.substringAfterLast('/') + val mangaState = when (tooltipElement.selectFirst("div.message_main > p:contains(Tình trạng)")?.ownText()) { + "Đang tiến hành" -> MangaState.ONGOING + "Hoàn thành" -> MangaState.FINISHED + else -> null + } + + val tagMap = getOrCreateTagMap() + val tagsElement = tooltipElement.selectFirst("div.message_main > p:contains(Thể loại)")?.ownText().orEmpty() + val mangaTags = tagsElement.split(',').mapNotNullToSet { tagMap[it.trim()] } + Manga( + id = generateUid(slug), + title = tooltipElement.selectFirst("div.title")?.text().orEmpty(), + altTitle = null, + url = absUrl.toRelativeUrl(domain), + publicUrl = absUrl, + rating = RATING_UNKNOWN, + isNsfw = false, + coverUrl = item.selectFirst("div.image a img")?.absUrl("data-original").orEmpty(), + largeCoverUrl = null, + tags = mangaTags, + state = mangaState, + author = tooltipElement.selectFirst("div.message_main > p:contains(Tác giả)")?.ownText(), + description = tooltipElement.selectFirst("div.box_text")?.text(), + chapters = null, + source = source, + ) + } + } + + + override suspend fun getPages(chapter: MangaChapter): List { + val pageElements = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + .select("div.reading-detail.box_doc > div img") + return pageElements.map { element -> + val url = element.attrAsAbsoluteUrl("data-original") + MangaPage( + id = generateUid(url), + url = url, + referer = domain, + preview = null, + source = source, + ) + } + } + + override suspend fun getTags(): Set { + val map = getOrCreateTagMap() + val tagSet = ArraySet(map.size) + for (entry in map) { + tagSet.add(entry.value) + } + + return tagSet + } + + private suspend fun getOrCreateTagMap(): ArrayMap = mutex.withLock { + tagCache?.let { return@withLock it } + val doc = webClient.httpGet("/tim-truyen-nang-cao".toAbsoluteUrl(domain)).parseHtml() + val tagItems = doc.select("div.genre-item") + val result = ArrayMap(tagItems.size) + for (item in tagItems) { + val title = item.text().trim() + val key = item.select("span[data-id]").attr("data-id") + result[title] = MangaTag(title = title, key = key, source = source) + } + tagCache = result + result + } + + private fun getSortOrderKey(sortOrder: SortOrder) = when (sortOrder) { + SortOrder.UPDATED -> 0 + SortOrder.POPULARITY -> 10 + SortOrder.NEWEST -> 15 + SortOrder.RATING -> 20 + else -> throw IllegalArgumentException("Sort order ${sortOrder.name} not supported") + } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NicovideoSeigaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NicovideoSeigaParser.kt index de8953e45..2434199b4 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NicovideoSeigaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NicovideoSeigaParser.kt @@ -11,175 +11,175 @@ private const val STATUS_ONGOING = "連載" private const val STATUS_FINISHED = "完結" @MangaSourceParser("NICOVIDEO_SEIGA", "Nicovideo Seiga", "ja") -class NicovideoSeigaParser(override val context: MangaLoaderContext) : - MangaParser(MangaSource.NICOVIDEO_SEIGA), - MangaParserAuthProvider { +class NicovideoSeigaParser(context: MangaLoaderContext) : + MangaParser(context, MangaSource.NICOVIDEO_SEIGA), + MangaParserAuthProvider { - override val authUrl: String - get() = "https://${getDomain("account")}/login?site=seiga" + override val authUrl: String + get() = "https://${getDomain("account")}/login?site=seiga" - override val isAuthorized: Boolean - get() = context.cookieJar.getCookies(getDomain("seiga")).any { - it.name == "user_session" - } + override val isAuthorized: Boolean + get() = context.cookieJar.getCookies(getDomain("seiga")).any { + it.name == "user_session" + } - override suspend fun getUsername(): String { - val body = context.httpGet("https://${getDomain("app")}/my/apps").parseHtml().body() - return body.selectFirst("#userinfo > div > div > strong")?.text() ?: throw AuthRequiredException(source) - } + override suspend fun getUsername(): String { + val body = webClient.httpGet("https://${getDomain("app")}/my/apps").parseHtml().body() + return body.selectFirst("#userinfo > div > div > strong")?.text() ?: throw AuthRequiredException(source) + } - override val sortOrders: Set = EnumSet.of( - SortOrder.UPDATED, - SortOrder.POPULARITY, - ) + override val sortOrders: Set = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + ) - override val configKeyDomain: ConfigKey.Domain = ConfigKey.Domain("nicovideo.jp", null) + override val configKeyDomain: ConfigKey.Domain = ConfigKey.Domain("nicovideo.jp", null) - @InternalParsersApi - override suspend fun getList( - offset: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - val page = (offset / 20f).toIntUp().inc() - val domain = getDomain("seiga") - val url = when { - !query.isNullOrEmpty() -> return if (offset == 0) getSearchList(query, page) else emptyList() - tags.isNullOrEmpty() -> "https://$domain/manga/list?page=$page&sort=${getSortKey(sortOrder)}" - tags.size == 1 -> "https://$domain/manga/list?category=${tags.first().key}&page=$page" + - "&sort=${getSortKey(sortOrder)}" + @InternalParsersApi + override suspend fun getList( + offset: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + val page = (offset / 20f).toIntUp().inc() + val domain = getDomain("seiga") + val url = when { + !query.isNullOrEmpty() -> return if (offset == 0) getSearchList(query, page) else emptyList() + tags.isNullOrEmpty() -> "https://$domain/manga/list?page=$page&sort=${getSortKey(sortOrder)}" + tags.size == 1 -> "https://$domain/manga/list?category=${tags.first().key}&page=$page" + + "&sort=${getSortKey(sortOrder)}" - tags.size > 1 -> throw IllegalArgumentException("This source supports only 1 category") - else -> "https://$domain/manga/list?page=$page&sort=${getSortKey(sortOrder)}" - } - val doc = context.httpGet(url).parseHtml() - val comicList = doc.body().select("#comic_list > ul > li") ?: doc.parseFailed("Container not found") - val items = comicList.select("div > .description > div > div") - return items.mapNotNull { item -> - val href = - item.selectFirst(".comic_icon > div > a")?.attrAsRelativeUrlOrNull("href") ?: return@mapNotNull null - val statusText = item.selectFirst(".mg_description_header > .mg_icon > .content_status > span")?.text() - Manga( - id = generateUid(href), - title = item.selectFirst(".mg_body > .title > a")?.text() ?: return@mapNotNull null, - coverUrl = item.selectFirst(".comic_icon > div > a > img")?.attrAsAbsoluteUrl("src").orEmpty(), - altTitle = null, - author = item.selectFirst(".mg_description_header > .mg_author > a")?.text(), - rating = RATING_UNKNOWN, - url = href, - isNsfw = false, - tags = item.getElementsByAttributeValueContaining("href", "?category=").mapToSet { a -> - MangaTag( - key = a.attr("href").substringAfterLast('='), - title = a.ownText().trim(), - source = source, - ) - }, - state = when (statusText) { - STATUS_ONGOING -> MangaState.ONGOING - STATUS_FINISHED -> MangaState.FINISHED - else -> null - }, - publicUrl = href.toAbsoluteUrl(item.host ?: getDomain("seiga")), - source = source, - ) - } - } + tags.size > 1 -> throw IllegalArgumentException("This source supports only 1 category") + else -> "https://$domain/manga/list?page=$page&sort=${getSortKey(sortOrder)}" + } + val doc = webClient.httpGet(url).parseHtml() + val comicList = doc.body().select("#comic_list > ul > li") ?: doc.parseFailed("Container not found") + val items = comicList.select("div > .description > div > div") + return items.mapNotNull { item -> + val href = + item.selectFirst(".comic_icon > div > a")?.attrAsRelativeUrlOrNull("href") ?: return@mapNotNull null + val statusText = item.selectFirst(".mg_description_header > .mg_icon > .content_status > span")?.text() + Manga( + id = generateUid(href), + title = item.selectFirst(".mg_body > .title > a")?.text() ?: return@mapNotNull null, + coverUrl = item.selectFirst(".comic_icon > div > a > img")?.attrAsAbsoluteUrl("src").orEmpty(), + altTitle = null, + author = item.selectFirst(".mg_description_header > .mg_author > a")?.text(), + rating = RATING_UNKNOWN, + url = href, + isNsfw = false, + tags = item.getElementsByAttributeValueContaining("href", "?category=").mapToSet { a -> + MangaTag( + key = a.attr("href").substringAfterLast('='), + title = a.ownText().trim(), + source = source, + ) + }, + state = when (statusText) { + STATUS_ONGOING -> MangaState.ONGOING + STATUS_FINISHED -> MangaState.FINISHED + else -> null + }, + publicUrl = href.toAbsoluteUrl(item.host ?: getDomain("seiga")), + source = source, + ) + } + } - override suspend fun getDetails(manga: Manga): Manga { - val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain("seiga"))).parseHtml() - val contents = doc.body().selectFirstOrThrow("#contents") - val statusText = contents - .select("div.mg_work_detail > div > div:nth-child(2) > div.tip.content_status.status_series > span") - .text() - return manga.copy( - description = contents.selectFirst("div.mg_work_detail > div > div.row > div.description_text")?.html(), - largeCoverUrl = contents.selectFirst("div.primaries > div.main_visual > a > img") - ?.attrAsAbsoluteUrlOrNull("src"), - state = when (statusText) { - STATUS_ONGOING -> MangaState.ONGOING - STATUS_FINISHED -> MangaState.FINISHED - else -> null - }, - isNsfw = contents.select(".icon_adult").isNotEmpty(), - chapters = contents.select("#episode_list > ul > li").mapChapters { i, li -> - val href = li.selectFirst("div > div.description > div.title > a") - ?.attrAsRelativeUrl("href") ?: li.parseFailed() - MangaChapter( - id = generateUid(href), - name = li.select("div > div.description > div.title > a").text(), - number = i + 1, - url = href, - scanlator = null, - branch = null, - uploadDate = 0, - source = source, - ) - }, - ) - } + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(getDomain("seiga"))).parseHtml() + val contents = doc.body().selectFirstOrThrow("#contents") + val statusText = contents + .select("div.mg_work_detail > div > div:nth-child(2) > div.tip.content_status.status_series > span") + .text() + return manga.copy( + description = contents.selectFirst("div.mg_work_detail > div > div.row > div.description_text")?.html(), + largeCoverUrl = contents.selectFirst("div.primaries > div.main_visual > a > img") + ?.attrAsAbsoluteUrlOrNull("src"), + state = when (statusText) { + STATUS_ONGOING -> MangaState.ONGOING + STATUS_FINISHED -> MangaState.FINISHED + else -> null + }, + isNsfw = contents.select(".icon_adult").isNotEmpty(), + chapters = contents.select("#episode_list > ul > li").mapChapters { i, li -> + val href = li.selectFirst("div > div.description > div.title > a") + ?.attrAsRelativeUrl("href") ?: li.parseFailed() + MangaChapter( + id = generateUid(href), + name = li.select("div > div.description > div.title > a").text(), + number = i + 1, + url = href, + scanlator = null, + branch = null, + uploadDate = 0, + source = source, + ) + }, + ) + } - override suspend fun getPages(chapter: MangaChapter): List { - val fullUrl = chapter.url.toAbsoluteUrl(getDomain("seiga")) - val doc = context.httpGet(fullUrl).parseHtml() - if (!doc.select("#login_manga").isEmpty()) - throw AuthRequiredException(source) - val root = doc.body().select("#page_contents > li") - return root.map { li -> - val url = li.select("div > img").attr("data-original") - MangaPage( - id = generateUid(url), - url = url, - preview = null, - referer = fullUrl, - source = source, - ) - } - } + override suspend fun getPages(chapter: MangaChapter): List { + val fullUrl = chapter.url.toAbsoluteUrl(getDomain("seiga")) + val doc = webClient.httpGet(fullUrl).parseHtml() + if (!doc.select("#login_manga").isEmpty()) + throw AuthRequiredException(source) + val root = doc.body().select("#page_contents > li") + return root.map { li -> + val url = li.select("div > img").attr("data-original") + MangaPage( + id = generateUid(url), + url = url, + preview = null, + referer = fullUrl, + source = source, + ) + } + } - override suspend fun getTags(): Set { - val doc = context.httpGet("https://${getDomain("seiga")}/manga/list").parseHtml() - val root = doc.body().selectOrThrow("#mg_category_list > ul > li") - return root.mapToSet { li -> - val a = li.selectFirstOrThrow("a") - MangaTag( - title = a.text(), - key = a.attrAsRelativeUrlOrNull("href").orEmpty(), - source = source, - ) - } - } + override suspend fun getTags(): Set { + val doc = webClient.httpGet("https://${getDomain("seiga")}/manga/list").parseHtml() + val root = doc.body().selectOrThrow("#mg_category_list > ul > li") + return root.mapToSet { li -> + val a = li.selectFirstOrThrow("a") + MangaTag( + title = a.text(), + key = a.attrAsRelativeUrlOrNull("href").orEmpty(), + source = source, + ) + } + } - private suspend fun getSearchList(query: String, page: Int): List { - val domain = getDomain("seiga") - val doc = context.httpGet("https://$domain/manga/search/?q=$query&page=$page&sort=score").parseHtml() - val root = doc.body().select(".search_result__item") - return root.mapNotNull { item -> - val href = item.selectFirst(".search_result__item__thumbnail > a") - ?.attrAsRelativeUrl("href") ?: doc.parseFailed() - Manga( - id = generateUid(href), - url = href, - publicUrl = href.toAbsoluteUrl(item.host ?: domain), - title = item.selectFirst(".search_result__item__info > .search_result__item__info--title > a") - ?.text()?.trim() ?: return@mapNotNull null, - altTitle = null, - author = null, - tags = emptySet(), - rating = RATING_UNKNOWN, - state = null, - isNsfw = false, - source = source, - coverUrl = item.selectFirst(".search_result__item__thumbnail > a > img") - ?.attrAsAbsoluteUrl("data-original").orEmpty(), - ) - } - } + private suspend fun getSearchList(query: String, page: Int): List { + val domain = getDomain("seiga") + val doc = webClient.httpGet("https://$domain/manga/search/?q=$query&page=$page&sort=score").parseHtml() + val root = doc.body().select(".search_result__item") + return root.mapNotNull { item -> + val href = item.selectFirst(".search_result__item__thumbnail > a") + ?.attrAsRelativeUrl("href") ?: doc.parseFailed() + Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(item.host ?: domain), + title = item.selectFirst(".search_result__item__info > .search_result__item__info--title > a") + ?.text()?.trim() ?: return@mapNotNull null, + altTitle = null, + author = null, + tags = emptySet(), + rating = RATING_UNKNOWN, + state = null, + isNsfw = false, + source = source, + coverUrl = item.selectFirst(".search_result__item__thumbnail > a > img") + ?.attrAsAbsoluteUrl("data-original").orEmpty(), + ) + } + } - private fun getSortKey(sortOrder: SortOrder) = when (sortOrder) { - SortOrder.POPULARITY -> "manga_view" - SortOrder.UPDATED -> "manga_updated" - else -> "manga_view" - } -} \ No newline at end of file + private fun getSortKey(sortOrder: SortOrder) = when (sortOrder) { + SortOrder.POPULARITY -> "manga_view" + SortOrder.UPDATED -> "manga_updated" + else -> "manga_view" + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NineMangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NineMangaParser.kt index 0918ac86f..f3d6d2cee 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NineMangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NineMangaParser.kt @@ -12,249 +12,248 @@ import java.text.SimpleDateFormat import java.util.* internal abstract class NineMangaParser( - final override val context: MangaLoaderContext, - source: MangaSource, - defaultDomain: String, -) : PagedMangaParser(source, pageSize = 26) { + context: MangaLoaderContext, + source: MangaSource, + defaultDomain: String, +) : PagedMangaParser(context, source, pageSize = 26) { - override val configKeyDomain = ConfigKey.Domain(defaultDomain, null) + override val configKeyDomain = ConfigKey.Domain(defaultDomain, null) - init { - context.cookieJar.insertCookies(getDomain(), "ninemanga_template_desk=yes") - } + init { + context.cookieJar.insertCookies(domain, "ninemanga_template_desk=yes") + } - override val headers = Headers.Builder() - .add("Accept-Language", "en-US;q=0.7,en;q=0.3") - .build() + override val headers = Headers.Builder() + .add("Accept-Language", "en-US;q=0.7,en;q=0.3") + .build() - override val sortOrders: Set = Collections.singleton( - SortOrder.POPULARITY, - ) + override val sortOrders: Set = Collections.singleton( + SortOrder.POPULARITY, + ) - override suspend fun getListPage( - page: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - val url = buildString { - append("https://") - append(getDomain()) - when { - !query.isNullOrEmpty() -> { - append("/search/?name_sel=&wd=") - append(query.urlEncoded()) - append("&page=") - } + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + val url = buildString { + append("https://") + append(domain) + when { + !query.isNullOrEmpty() -> { + append("/search/?name_sel=&wd=") + append(query.urlEncoded()) + append("&page=") + } - !tags.isNullOrEmpty() -> { - append("/search/?category_id=") - for (tag in tags) { - append(tag.key) - append(',') - } - append("&page=") - } + !tags.isNullOrEmpty() -> { + append("/search/?category_id=") + for (tag in tags) { + append(tag.key) + append(',') + } + append("&page=") + } - else -> { - append("/category/index_") - } - } - append(page) - append(".html") - } - val doc = context.httpGet(url, headers).parseHtml() - val root = doc.body().selectFirst("ul.direlist") - ?: doc.parseFailed("Cannot find root") - val baseHost = root.baseUri().toHttpUrl().host - return root.select("li").map { node -> - val href = node.selectFirst("a")?.absUrl("href") - ?: node.parseFailed("Link not found") - val relUrl = href.toRelativeUrl(baseHost) - val dd = node.selectFirst("dd") - Manga( - id = generateUid(relUrl), - url = relUrl, - publicUrl = href, - title = dd?.selectFirst("a.bookname")?.text()?.toCamelCase().orEmpty(), - altTitle = null, - coverUrl = node.selectFirst("img")?.absUrl("src").orEmpty(), - rating = RATING_UNKNOWN, - author = null, - isNsfw = false, - tags = emptySet(), - state = null, - source = source, - description = dd?.selectFirst("p")?.html(), - ) - } - } + else -> { + append("/category/index_") + } + } + append(page) + append(".html") + } + val doc = webClient.httpGet(url).parseHtml() + val root = doc.body().selectFirst("ul.direlist") + ?: doc.parseFailed("Cannot find root") + val baseHost = root.baseUri().toHttpUrl().host + return root.select("li").map { node -> + val href = node.selectFirst("a")?.absUrl("href") + ?: node.parseFailed("Link not found") + val relUrl = href.toRelativeUrl(baseHost) + val dd = node.selectFirst("dd") + Manga( + id = generateUid(relUrl), + url = relUrl, + publicUrl = href, + title = dd?.selectFirst("a.bookname")?.text()?.toCamelCase().orEmpty(), + altTitle = null, + coverUrl = node.selectFirst("img")?.absUrl("src").orEmpty(), + rating = RATING_UNKNOWN, + author = null, + isNsfw = false, + tags = emptySet(), + state = null, + source = source, + description = dd?.selectFirst("p")?.html(), + ) + } + } - override suspend fun getDetails(manga: Manga): Manga { - val doc = context.httpGet( - manga.url.toAbsoluteUrl(getDomain()) + "?waring=1", - headers, - ).parseHtml() - val root = doc.body().selectFirstOrThrow("div.manga") - val infoRoot = root.selectFirstOrThrow("div.bookintro") - return manga.copy( - tags = infoRoot.getElementsByAttributeValue("itemprop", "genre").first() - ?.select("a")?.mapToSet { a -> - MangaTag( - title = a.text().toTitleCase(), - key = a.attr("href").substringBetween("/", "."), - source = source, - ) - }.orEmpty(), - author = infoRoot.getElementsByAttributeValue("itemprop", "author").first()?.text(), - state = parseStatus(infoRoot.select("li a.red").text()), - description = infoRoot.getElementsByAttributeValue("itemprop", "description").first() - ?.html()?.substringAfter(""), - chapters = root.selectFirst("div.chapterbox")?.select("ul.sub_vol_ul > li") - ?.asReversed()?.mapChapters { i, li -> - val a = li.selectFirst("a.chapter_list_a") - val href = a?.attrAsRelativeUrlOrNull("href") - ?.replace("%20", " ") ?: li.parseFailed("Link not found") - MangaChapter( - id = generateUid(href), - name = a.text(), - number = i + 1, - url = href, - uploadDate = parseChapterDateByLang(li.selectFirst("span")?.text().orEmpty()), - source = source, - scanlator = null, - branch = null, - ) - }, - ) - } + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet( + manga.url.toAbsoluteUrl(domain) + "?waring=1", + ).parseHtml() + val root = doc.body().selectFirstOrThrow("div.manga") + val infoRoot = root.selectFirstOrThrow("div.bookintro") + return manga.copy( + tags = infoRoot.getElementsByAttributeValue("itemprop", "genre").first() + ?.select("a")?.mapToSet { a -> + MangaTag( + title = a.text().toTitleCase(), + key = a.attr("href").substringBetween("/", "."), + source = source, + ) + }.orEmpty(), + author = infoRoot.getElementsByAttributeValue("itemprop", "author").first()?.text(), + state = parseStatus(infoRoot.select("li a.red").text()), + description = infoRoot.getElementsByAttributeValue("itemprop", "description").first() + ?.html()?.substringAfter(""), + chapters = root.selectFirst("div.chapterbox")?.select("ul.sub_vol_ul > li") + ?.asReversed()?.mapChapters { i, li -> + val a = li.selectFirst("a.chapter_list_a") + val href = a?.attrAsRelativeUrlOrNull("href") + ?.replace("%20", " ") ?: li.parseFailed("Link not found") + MangaChapter( + id = generateUid(href), + name = a.text(), + number = i + 1, + url = href, + uploadDate = parseChapterDateByLang(li.selectFirst("span")?.text().orEmpty()), + source = source, + scanlator = null, + branch = null, + ) + }, + ) + } - override suspend fun getPages(chapter: MangaChapter): List { - val doc = context.httpGet(chapter.url.toAbsoluteUrl(getDomain()), headers).parseHtml() - return doc.body().getElementById("page")?.select("option")?.map { option -> - val url = option.attr("value") - MangaPage( - id = generateUid(url), - url = url, - referer = chapter.url.toAbsoluteUrl(getDomain()), - preview = null, - source = source, - ) - } ?: doc.parseFailed("Pages list not found") - } + override suspend fun getPages(chapter: MangaChapter): List { + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + return doc.body().getElementById("page")?.select("option")?.map { option -> + val url = option.attr("value") + MangaPage( + id = generateUid(url), + url = url, + referer = chapter.url.toAbsoluteUrl(domain), + preview = null, + source = source, + ) + } ?: doc.parseFailed("Pages list not found") + } - override suspend fun getPageUrl(page: MangaPage): String { - val doc = context.httpGet(page.url.toAbsoluteUrl(getDomain()), headers).parseHtml() - val root = doc.body() - return root.selectFirst("a.pic_download")?.absUrl("href") - ?: doc.parseFailed("Page image not found") - } + override suspend fun getPageUrl(page: MangaPage): String { + val doc = webClient.httpGet(page.url.toAbsoluteUrl(domain)).parseHtml() + val root = doc.body() + return root.selectFirst("a.pic_download")?.absUrl("href") + ?: doc.parseFailed("Page image not found") + } - override suspend fun getTags(): Set { - val doc = context.httpGet("https://${getDomain()}/search/?type=high", headers) - .parseHtml() - val root = doc.body().getElementById("search_form") - return root?.select("li.cate_list")?.mapNotNullToSet { li -> - val cateId = li.attr("cate_id") ?: return@mapNotNullToSet null - val a = li.selectFirst("a") ?: return@mapNotNullToSet null - MangaTag( - title = a.text().toTitleCase(), - key = cateId, - source = source, - ) - } ?: doc.parseFailed("Root not found") - } + override suspend fun getTags(): Set { + val doc = webClient.httpGet("https://${domain}/search/?type=high") + .parseHtml() + val root = doc.body().getElementById("search_form") + return root?.select("li.cate_list")?.mapNotNullToSet { li -> + val cateId = li.attr("cate_id") ?: return@mapNotNullToSet null + val a = li.selectFirst("a") ?: return@mapNotNullToSet null + MangaTag( + title = a.text().toTitleCase(), + key = cateId, + source = source, + ) + } ?: doc.parseFailed("Root not found") + } - private fun parseStatus(status: String) = when { - status.contains("Ongoing") -> MangaState.ONGOING - status.contains("Completed") -> MangaState.FINISHED - else -> null - } + private fun parseStatus(status: String) = when { + status.contains("Ongoing") -> MangaState.ONGOING + status.contains("Completed") -> MangaState.FINISHED + else -> null + } - private fun parseChapterDateByLang(date: String): Long { - val dateWords = date.split(" ") + private fun parseChapterDateByLang(date: String): Long { + val dateWords = date.split(" ") - if (dateWords.size == 3) { - if (dateWords[1].contains(",")) { - SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH).tryParse(date) - } else { - val timeAgo = Integer.parseInt(dateWords[0]) - return Calendar.getInstance().apply { - when (dateWords[1]) { - "minutes" -> Calendar.MINUTE // EN-FR - "hours" -> Calendar.HOUR // EN + if (dateWords.size == 3) { + if (dateWords[1].contains(",")) { + SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH).tryParse(date) + } else { + val timeAgo = Integer.parseInt(dateWords[0]) + return Calendar.getInstance().apply { + when (dateWords[1]) { + "minutes" -> Calendar.MINUTE // EN-FR + "hours" -> Calendar.HOUR // EN - "minutos" -> Calendar.MINUTE // ES - "horas" -> Calendar.HOUR + "minutos" -> Calendar.MINUTE // ES + "horas" -> Calendar.HOUR - // "minutos" -> Calendar.MINUTE // BR - "hora" -> Calendar.HOUR + // "minutos" -> Calendar.MINUTE // BR + "hora" -> Calendar.HOUR - "минут" -> Calendar.MINUTE // RU - "часа" -> Calendar.HOUR + "минут" -> Calendar.MINUTE // RU + "часа" -> Calendar.HOUR - "Stunden" -> Calendar.HOUR // DE + "Stunden" -> Calendar.HOUR // DE - "minuti" -> Calendar.MINUTE // IT - "ore" -> Calendar.HOUR + "minuti" -> Calendar.MINUTE // IT + "ore" -> Calendar.HOUR - "heures" -> Calendar.HOUR // FR ("minutes" also French word) - else -> null - }?.let { - add(it, -timeAgo) - } - }.timeInMillis - } - } - return 0L - } + "heures" -> Calendar.HOUR // FR ("minutes" also French word) + else -> null + }?.let { + add(it, -timeAgo) + } + }.timeInMillis + } + } + return 0L + } - @MangaSourceParser("NINEMANGA_EN", "NineManga English", "en") - class English(context: MangaLoaderContext) : NineMangaParser( - context, - MangaSource.NINEMANGA_EN, - "www.ninemanga.com", - ) + @MangaSourceParser("NINEMANGA_EN", "NineManga English", "en") + class English(context: MangaLoaderContext) : NineMangaParser( + context, + MangaSource.NINEMANGA_EN, + "www.ninemanga.com", + ) - @MangaSourceParser("NINEMANGA_ES", "NineManga Español", "es") - class Spanish(context: MangaLoaderContext) : NineMangaParser( - context, - MangaSource.NINEMANGA_ES, - "es.ninemanga.com", - ) + @MangaSourceParser("NINEMANGA_ES", "NineManga Español", "es") + class Spanish(context: MangaLoaderContext) : NineMangaParser( + context, + MangaSource.NINEMANGA_ES, + "es.ninemanga.com", + ) - @MangaSourceParser("NINEMANGA_RU", "NineManga Русский", "ru") - class Russian(context: MangaLoaderContext) : NineMangaParser( - context, - MangaSource.NINEMANGA_RU, - "ru.ninemanga.com", - ) + @MangaSourceParser("NINEMANGA_RU", "NineManga Русский", "ru") + class Russian(context: MangaLoaderContext) : NineMangaParser( + context, + MangaSource.NINEMANGA_RU, + "ru.ninemanga.com", + ) - @MangaSourceParser("NINEMANGA_DE", "NineManga Deutsch", "de") - class Deutsch(context: MangaLoaderContext) : NineMangaParser( - context, - MangaSource.NINEMANGA_DE, - "de.ninemanga.com", - ) + @MangaSourceParser("NINEMANGA_DE", "NineManga Deutsch", "de") + class Deutsch(context: MangaLoaderContext) : NineMangaParser( + context, + MangaSource.NINEMANGA_DE, + "de.ninemanga.com", + ) - @MangaSourceParser("NINEMANGA_BR", "NineManga Brasil", "pt") - class Brazil(context: MangaLoaderContext) : NineMangaParser( - context, - MangaSource.NINEMANGA_BR, - "br.ninemanga.com", - ) + @MangaSourceParser("NINEMANGA_BR", "NineManga Brasil", "pt") + class Brazil(context: MangaLoaderContext) : NineMangaParser( + context, + MangaSource.NINEMANGA_BR, + "br.ninemanga.com", + ) - @MangaSourceParser("NINEMANGA_IT", "NineManga Italiano", "it") - class Italiano(context: MangaLoaderContext) : NineMangaParser( - context, - MangaSource.NINEMANGA_IT, - "it.ninemanga.com", - ) + @MangaSourceParser("NINEMANGA_IT", "NineManga Italiano", "it") + class Italiano(context: MangaLoaderContext) : NineMangaParser( + context, + MangaSource.NINEMANGA_IT, + "it.ninemanga.com", + ) - @MangaSourceParser("NINEMANGA_FR", "NineManga Français", "fr") - class Francais(context: MangaLoaderContext) : NineMangaParser( - context, - MangaSource.NINEMANGA_FR, - "fr.ninemanga.com", - ) -} \ No newline at end of file + @MangaSourceParser("NINEMANGA_FR", "NineManga Français", "fr") + class Francais(context: MangaLoaderContext) : NineMangaParser( + context, + MangaSource.NINEMANGA_FR, + "fr.ninemanga.com", + ) +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NudeMoonParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NudeMoonParser.kt index 0bde4f090..6341dcd72 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NudeMoonParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NudeMoonParser.kt @@ -15,217 +15,217 @@ private const val MAX_THUMB_INDEX = 19 @MangaSourceParser("NUDEMOON", "Nude-Moon", "ru") internal class NudeMoonParser( - override val context: MangaLoaderContext, -) : MangaParser(MangaSource.NUDEMOON), MangaParserAuthProvider { - - override val configKeyDomain = ConfigKey.Domain( - defaultValue = "nude-moon.org", - presetValues = arrayOf("nude-moon.org", "nude-moon.net"), - ) - override val authUrl: String - get() = "https://${getDomain()}/index.php" - - override val isAuthorized: Boolean - get() { - return context.cookieJar.getCookies(getDomain()).any { - it.name == "fusion_user" - } - } - - override val sortOrders: Set = EnumSet.of( - SortOrder.NEWEST, - SortOrder.POPULARITY, - SortOrder.RATING, - ) - - init { - context.cookieJar.insertCookies( - getDomain(), - "NMfYa=1;", - "nm_mobile=0;", - ) - } - - override suspend fun getList( - offset: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - val domain = getDomain() - val url = when { - !query.isNullOrEmpty() -> "https://$domain/search?stext=${query.urlEncoded()}&rowstart=$offset" - !tags.isNullOrEmpty() -> tags.joinToString( - separator = "_", - prefix = "https://$domain/tags/", - postfix = "&rowstart=$offset", - transform = { it.key.urlEncoded() }, - ) - - else -> "https://$domain/all_manga?${getSortKey(sortOrder)}&rowstart=$offset" - } - val doc = context.httpGet(url).parseHtml() - val root = doc.body().run { - selectFirst("td.main-bg") ?: selectFirst("td.main-body") - } ?: doc.parseFailed("Cannot find root") - return root.select("table.news_pic2").mapNotNull { row -> - val a = row.selectFirst("td.bg_style1")?.selectFirst("a") - ?: return@mapNotNull null - val href = a.attrAsRelativeUrl("href") - val title = a.selectFirst("h2")?.text().orEmpty() - val info = row.selectFirst("td[width=100%]") ?: return@mapNotNull null - Manga( - id = generateUid(href), - url = href, - title = title.substringAfter(" / "), - altTitle = title.substringBefore(" / ", "") - .takeUnless { it.isBlank() }, - author = info.getElementsContainingOwnText("Автор:").firstOrNull() - ?.nextElementSibling()?.ownText(), - coverUrl = row.selectFirst("img.news_pic2")?.absUrl("data-src") - .orEmpty(), - tags = row.selectFirst("span.tag-links")?.select("a") - ?.mapToSet { - MangaTag( - title = it.text().toTitleCase(), - key = it.attr("href").substringAfterLast('/'), - source = source, - ) - }.orEmpty(), - source = source, - publicUrl = a.absUrl("href"), - rating = RATING_UNKNOWN, - isNsfw = true, - description = row.selectFirst("div.description")?.html(), - state = null, - ) - } - } - - override suspend fun getDetails(manga: Manga): Manga { - val body = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml().body() - val root = body.selectFirst("table.shoutbox") - ?: body.parseFailed("Cannot find root") - val info = root.select("div.tbl2") - val lastInfo = info.last() - return manga.copy( - largeCoverUrl = body.selectFirst("img.news_pic2")?.absUrl("src"), - description = info.select("div.blockquote").lastOrNull()?.html() ?: manga.description, - tags = info.select("span.tag-links").firstOrNull()?.select("a")?.mapToSet { - MangaTag( - title = it.text().toTitleCase(), - key = it.attr("href").substringAfterLast('/'), - source = source, - ) - }?.plus(manga.tags) ?: manga.tags, - author = lastInfo?.getElementsByAttributeValueContaining("href", "mangaka/")?.text() - ?: manga.author, - chapters = listOf( - MangaChapter( - id = manga.id, - url = getReadLink(manga.url), - source = source, - number = 1, - name = manga.title, - scanlator = lastInfo?.getElementsByAttributeValueContaining("href", "perevod/")?.text(), - uploadDate = lastInfo?.getElementsContainingOwnText("Дата:") - ?.firstOrNull() - ?.html() - ?.parseDate() ?: 0L, - branch = null, - ), - ), - ) - } - - override suspend fun getPages(chapter: MangaChapter): List { - val fullUrl = chapter.url.toAbsoluteUrl(getDomain()) - val doc = context.httpGet(fullUrl).parseHtml() - val mangaId = chapter.url.substringAfterLast('/').substringBefore('-').toIntOrNull() - - val script = doc.select("script").firstNotNullOfOrNull { - it.html().takeIf { x -> x.contains(" images = new ") } - } ?: if (isAuthorized) { - doc.parseFailed("Cannot find pages list") - } else { - throw AuthRequiredException(source) - } - val pagesRegex = Regex("images\\[(\\d+)].src\\s*=\\s*'([^']+)'", RegexOption.MULTILINE) - return pagesRegex.findAll(script).map { match -> - val i = match.groupValues[1].toInt() - val url = match.groupValues[2] - MangaPage( - id = generateUid(url), - url = url, - referer = fullUrl, - preview = if (i <= MAX_THUMB_INDEX && mangaId != null) { - val part2 = url.substringBeforeLast('/') - val part3 = url.substringAfterLast('/') - val part1 = part2.substringBeforeLast('/') - "$part1/thumb/$mangaId/thumb_$part3" - } else { - null - }, - source = source, - ) - }.toList() - } - - override suspend fun getTags(): Set { - val domain = getDomain() - val doc = context.httpGet("https://$domain/all_manga").parseHtml() - val root = doc.body().getElementsContainingOwnText("Поиск манги по тегам") - .firstOrNull()?.parents()?.find { it.tag().normalName() == "tbody" } - ?.selectFirst("td.textbox")?.selectFirst("td.small") - ?: doc.parseFailed("Tags root not found") - return root.select("a").mapToSet { - MangaTag( - title = it.text().toTitleCase(), - key = it.attr("href").substringAfterLast('/') - .removeSuffix("+"), - source = source, - ) - } - } - - override suspend fun getUsername(): String { - val body = context.httpGet("https://${getDomain()}/").parseHtml() - .body() - return body - .getElementsContainingOwnText("Профиль") - .firstOrNull() - ?.attr("href") - ?.substringAfterLast('/') - ?: run { - throw if (body.selectFirst("form[name=\"loginform\"]") != null) { - AuthRequiredException(source) - } else { - body.parseFailed("Cannot find username") - } - } - } - - override fun getFaviconUrl(): String { - return "https://${getDomain()}/favicon.jpg" - } - - private fun getSortKey(sortOrder: SortOrder) = - when (sortOrder) { - SortOrder.POPULARITY -> "views" - SortOrder.NEWEST -> "date" - SortOrder.RATING -> "like" - else -> "like" - } - - private fun String.parseDate(): Long { - val dateString = substringBetweenFirst("Дата:", "<")?.trim() ?: return 0 - val dateFormat = SimpleDateFormat("d MMMM yyyy", Locale("ru")) - return dateFormat.tryParse(dateString) - } - - private fun getReadLink(url: String): String { - val prefix = url.substringBefore('-', "") - val suffix = url.substringAfter('-').trimStart('-') - return "$prefix-online-$suffix" - } -} \ No newline at end of file + context: MangaLoaderContext, +) : MangaParser(context, MangaSource.NUDEMOON), MangaParserAuthProvider { + + override val configKeyDomain = ConfigKey.Domain( + defaultValue = "nude-moon.org", + presetValues = arrayOf("nude-moon.org", "nude-moon.net"), + ) + override val authUrl: String + get() = "https://${domain}/index.php" + + override val isAuthorized: Boolean + get() { + return context.cookieJar.getCookies(domain).any { + it.name == "fusion_user" + } + } + + override val sortOrders: Set = EnumSet.of( + SortOrder.NEWEST, + SortOrder.POPULARITY, + SortOrder.RATING, + ) + + init { + context.cookieJar.insertCookies( + domain, + "NMfYa=1;", + "nm_mobile=0;", + ) + } + + override suspend fun getList( + offset: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + val domain = domain + val url = when { + !query.isNullOrEmpty() -> "https://$domain/search?stext=${query.urlEncoded()}&rowstart=$offset" + !tags.isNullOrEmpty() -> tags.joinToString( + separator = "_", + prefix = "https://$domain/tags/", + postfix = "&rowstart=$offset", + transform = { it.key.urlEncoded() }, + ) + + else -> "https://$domain/all_manga?${getSortKey(sortOrder)}&rowstart=$offset" + } + val doc = webClient.httpGet(url).parseHtml() + val root = doc.body().run { + selectFirst("td.main-bg") ?: selectFirst("td.main-body") + } ?: doc.parseFailed("Cannot find root") + return root.select("table.news_pic2").mapNotNull { row -> + val a = row.selectFirst("td.bg_style1")?.selectFirst("a") + ?: return@mapNotNull null + val href = a.attrAsRelativeUrl("href") + val title = a.selectFirst("h2")?.text().orEmpty() + val info = row.selectFirst("td[width=100%]") ?: return@mapNotNull null + Manga( + id = generateUid(href), + url = href, + title = title.substringAfter(" / "), + altTitle = title.substringBefore(" / ", "") + .takeUnless { it.isBlank() }, + author = info.getElementsContainingOwnText("Автор:").firstOrNull() + ?.nextElementSibling()?.ownText(), + coverUrl = row.selectFirst("img.news_pic2")?.absUrl("data-src") + .orEmpty(), + tags = row.selectFirst("span.tag-links")?.select("a") + ?.mapToSet { + MangaTag( + title = it.text().toTitleCase(), + key = it.attr("href").substringAfterLast('/'), + source = source, + ) + }.orEmpty(), + source = source, + publicUrl = a.absUrl("href"), + rating = RATING_UNKNOWN, + isNsfw = true, + description = row.selectFirst("div.description")?.html(), + state = null, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val body = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml().body() + val root = body.selectFirst("table.shoutbox") + ?: body.parseFailed("Cannot find root") + val info = root.select("div.tbl2") + val lastInfo = info.last() + return manga.copy( + largeCoverUrl = body.selectFirst("img.news_pic2")?.absUrl("src"), + description = info.select("div.blockquote").lastOrNull()?.html() ?: manga.description, + tags = info.select("span.tag-links").firstOrNull()?.select("a")?.mapToSet { + MangaTag( + title = it.text().toTitleCase(), + key = it.attr("href").substringAfterLast('/'), + source = source, + ) + }?.plus(manga.tags) ?: manga.tags, + author = lastInfo?.getElementsByAttributeValueContaining("href", "mangaka/")?.text() + ?: manga.author, + chapters = listOf( + MangaChapter( + id = manga.id, + url = getReadLink(manga.url), + source = source, + number = 1, + name = manga.title, + scanlator = lastInfo?.getElementsByAttributeValueContaining("href", "perevod/")?.text(), + uploadDate = lastInfo?.getElementsContainingOwnText("Дата:") + ?.firstOrNull() + ?.html() + ?.parseDate() ?: 0L, + branch = null, + ), + ), + ) + } + + override suspend fun getPages(chapter: MangaChapter): List { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + val mangaId = chapter.url.substringAfterLast('/').substringBefore('-').toIntOrNull() + + val script = doc.select("script").firstNotNullOfOrNull { + it.html().takeIf { x -> x.contains(" images = new ") } + } ?: if (isAuthorized) { + doc.parseFailed("Cannot find pages list") + } else { + throw AuthRequiredException(source) + } + val pagesRegex = Regex("images\\[(\\d+)].src\\s*=\\s*'([^']+)'", RegexOption.MULTILINE) + return pagesRegex.findAll(script).map { match -> + val i = match.groupValues[1].toInt() + val url = match.groupValues[2] + MangaPage( + id = generateUid(url), + url = url, + referer = fullUrl, + preview = if (i <= MAX_THUMB_INDEX && mangaId != null) { + val part2 = url.substringBeforeLast('/') + val part3 = url.substringAfterLast('/') + val part1 = part2.substringBeforeLast('/') + "$part1/thumb/$mangaId/thumb_$part3" + } else { + null + }, + source = source, + ) + }.toList() + } + + override suspend fun getTags(): Set { + val domain = domain + val doc = webClient.httpGet("https://$domain/all_manga").parseHtml() + val root = doc.body().getElementsContainingOwnText("Поиск манги по тегам") + .firstOrNull()?.parents()?.find { it.tag().normalName() == "tbody" } + ?.selectFirst("td.textbox")?.selectFirst("td.small") + ?: doc.parseFailed("Tags root not found") + return root.select("a").mapToSet { + MangaTag( + title = it.text().toTitleCase(), + key = it.attr("href").substringAfterLast('/') + .removeSuffix("+"), + source = source, + ) + } + } + + override suspend fun getUsername(): String { + val body = webClient.httpGet("https://${domain}/").parseHtml() + .body() + return body + .getElementsContainingOwnText("Профиль") + .firstOrNull() + ?.attr("href") + ?.substringAfterLast('/') + ?: run { + throw if (body.selectFirst("form[name=\"loginform\"]") != null) { + AuthRequiredException(source) + } else { + body.parseFailed("Cannot find username") + } + } + } + + override fun getFaviconUrl(): String { + return "https://${domain}/favicon.jpg" + } + + private fun getSortKey(sortOrder: SortOrder) = + when (sortOrder) { + SortOrder.POPULARITY -> "views" + SortOrder.NEWEST -> "date" + SortOrder.RATING -> "like" + else -> "like" + } + + private fun String.parseDate(): Long { + val dateString = substringBetweenFirst("Дата:", "<")?.trim() ?: return 0 + val dateFormat = SimpleDateFormat("d MMMM yyyy", Locale("ru")) + return dateFormat.tryParse(dateString) + } + + private fun getReadLink(url: String): String { + val prefix = url.substringBefore('-', "") + val suffix = url.substringAfter('-').trimStart('-') + return "$prefix-online-$suffix" + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/RemangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/RemangaParser.kt index 48f4530ea..fa3a24dcf 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/RemangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/RemangaParser.kt @@ -28,259 +28,259 @@ private const val STATUS_FINISHED = 0 @MangaSourceParser("REMANGA", "Remanga", "ru") internal class RemangaParser( - override val context: MangaLoaderContext, -) : PagedMangaParser(MangaSource.REMANGA, PAGE_SIZE), MangaParserAuthProvider { + context: MangaLoaderContext, +) : PagedMangaParser(context, MangaSource.REMANGA, PAGE_SIZE), MangaParserAuthProvider { - override val headers = Headers.Builder() - .add("User-Agent", "Mozilla/5.0 (Android 13; Mobile; rv:68.0) Gecko/68.0 Firefox/109.0") - .build() + private val baseHeaders = Headers.Builder() + .add("User-Agent", "Mozilla/5.0 (Android 13; Mobile; rv:68.0) Gecko/68.0 Firefox/109.0") + .build() - override val configKeyDomain = ConfigKey.Domain("remanga.org", null) - override val authUrl: String - get() = "https://${getDomain()}/user/login" + override val headers + get() = getApiHeaders() - override val sortOrders: Set = EnumSet.of( - SortOrder.UPDATED, - SortOrder.POPULARITY, - SortOrder.RATING, - SortOrder.NEWEST, - ) + override val configKeyDomain = ConfigKey.Domain("remanga.org", null) + override val authUrl: String + get() = "https://${domain}/user/login" - override val isAuthorized: Boolean - get() { - return context.cookieJar.getCookies(getDomain()).any { - it.name == "user" - } - } + override val sortOrders: Set = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.RATING, + SortOrder.NEWEST, + ) - private val regexLastUrlPath = Regex("/[^/]+/?$") + override val isAuthorized: Boolean + get() { + return context.cookieJar.getCookies(domain).any { + it.name == "user" + } + } - override suspend fun getListPage( - page: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - copyCookies() - val domain = getDomain() - val urlBuilder = StringBuilder() - .append("https://api.") - .append(domain) - if (query != null) { - urlBuilder.append("/api/search/?query=") - .append(query.urlEncoded()) - } else { - urlBuilder.append("/api/search/catalog/?ordering=") - .append(getSortKey(sortOrder)) - tags?.forEach { tag -> - urlBuilder.append("&genres=") - urlBuilder.append(tag.key) - } - } - urlBuilder - .append("&page=") - .append(page) - .append("&count=") - .append(PAGE_SIZE) - val content = context.httpGet(urlBuilder.toString(), getApiHeaders()).parseJson() - .getJSONArray("content") - return content.mapJSON { jo -> - val url = "/manga/${jo.getString("dir")}" - val img = jo.getJSONObject("img") - Manga( - id = generateUid(url), - url = url, - publicUrl = "https://$domain$url", - title = jo.getString("rus_name"), - altTitle = jo.getString("en_name"), - rating = jo.getString("avg_rating").toFloatOrNull()?.div(10f) ?: RATING_UNKNOWN, - coverUrl = "https://api.$domain${img.getString("mid")}", - largeCoverUrl = "https://api.$domain${img.getString("high")}", - author = null, - isNsfw = false, - state = null, - tags = jo.optJSONArray("genres")?.mapJSONToSet { g -> - MangaTag( - title = g.getString("name").toTitleCase(), - key = g.getInt("id").toString(), - source = MangaSource.REMANGA, - ) - }.orEmpty(), - source = MangaSource.REMANGA, - ) - } - } + private val regexLastUrlPath = Regex("/[^/]+/?$") - override suspend fun getDetails(manga: Manga): Manga { - copyCookies() - val domain = getDomain() - val slug = manga.url.find(regexLastUrlPath) - ?: throw ParseException("Cannot obtain slug from ${manga.url}", manga.publicUrl) - val data = context.httpGet( - url = "https://api.$domain/api/titles$slug/", - headers = getApiHeaders(), - ).handle401().parseJson() - val content = try { - data.getJSONObject("content") - } catch (e: JSONException) { - throw ParseException(data.optString("msg"), manga.publicUrl, e) - } - val branchId = content.getJSONArray("branches").optJSONObject(0) - ?.getLong("id") ?: throw ParseException("No branches found", manga.publicUrl) - val chapters = grabChapters(domain, branchId) - val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.US) - return manga.copy( - description = content.getString("description"), - state = when (content.optJSONObject("status")?.getInt("id")) { - STATUS_ONGOING -> MangaState.ONGOING - STATUS_FINISHED -> MangaState.FINISHED - else -> null - }, - tags = content.getJSONArray("genres").mapJSONToSet { g -> - MangaTag( - title = g.getString("name").toTitleCase(), - key = g.getInt("id").toString(), - source = MangaSource.REMANGA, - ) - }, - chapters = chapters.mapChapters { i, jo -> - if ( - jo.getBooleanOrDefault("is_paid", false) && - !jo.getBooleanOrDefault("is_bought", false) - ) { - return@mapChapters null - } - val id = jo.getLong("id") - val name = jo.getString("name").toTitleCase(Locale.ROOT) - val publishers = jo.optJSONArray("publishers") - MangaChapter( - id = generateUid(id), - url = "/api/titles/chapters/$id/", - number = chapters.size - i, - name = buildString { - append("Том ") - append(jo.optString("tome", "0")) - append(". ") - append("Глава ") - append(jo.optString("chapter", "0")) - if (name.isNotEmpty()) { - append(" - ") - append(name) - } - }, - uploadDate = dateFormat.tryParse(jo.getString("upload_date")), - scanlator = publishers?.optJSONObject(0)?.getStringOrNull("name"), - source = MangaSource.REMANGA, - branch = null, - ) - }.asReversed(), - ) - } + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + copyCookies() + val domain = domain + val urlBuilder = StringBuilder() + .append("https://api.") + .append(domain) + if (query != null) { + urlBuilder.append("/api/search/?query=") + .append(query.urlEncoded()) + } else { + urlBuilder.append("/api/search/catalog/?ordering=") + .append(getSortKey(sortOrder)) + tags?.forEach { tag -> + urlBuilder.append("&genres=") + urlBuilder.append(tag.key) + } + } + urlBuilder + .append("&page=") + .append(page) + .append("&count=") + .append(PAGE_SIZE) + val content = webClient.httpGet(urlBuilder.toString()).parseJson() + .getJSONArray("content") + return content.mapJSON { jo -> + val url = "/manga/${jo.getString("dir")}" + val img = jo.getJSONObject("img") + Manga( + id = generateUid(url), + url = url, + publicUrl = "https://$domain$url", + title = jo.getString("rus_name"), + altTitle = jo.getString("en_name"), + rating = jo.getString("avg_rating").toFloatOrNull()?.div(10f) ?: RATING_UNKNOWN, + coverUrl = "https://api.$domain${img.getString("mid")}", + largeCoverUrl = "https://api.$domain${img.getString("high")}", + author = null, + isNsfw = false, + state = null, + tags = jo.optJSONArray("genres")?.mapJSONToSet { g -> + MangaTag( + title = g.getString("name").toTitleCase(), + key = g.getInt("id").toString(), + source = MangaSource.REMANGA, + ) + }.orEmpty(), + source = MangaSource.REMANGA, + ) + } + } - override suspend fun getPages(chapter: MangaChapter): List { - val referer = "https://${getDomain()}/" - val content = context.httpGet(chapter.url.toAbsoluteUrl(getDomain("api")), getApiHeaders()) - .handle401() - .parseJson() - .getJSONObject("content") - val pages = content.optJSONArray("pages") - if (pages == null) { - val pubDate = content.getStringOrNull("pub_date")?.let { - SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.US).tryParse(it) - } - if (pubDate != null && pubDate > System.currentTimeMillis()) { - val at = SimpleDateFormat.getDateInstance(DateFormat.LONG).format(Date(pubDate)) - throw ContentUnavailableException("Глава станет доступной $at") - } else { - throw ContentUnavailableException("Глава недоступна") - } - } - val result = ArrayList(pages.length()) - for (i in 0 until pages.length()) { - when (val item = pages.get(i)) { - is JSONObject -> result += parsePage(item, referer) - is JSONArray -> item.mapJSONTo(result) { parsePage(it, referer) } - else -> throw ParseException("Unknown json item $item", chapter.url) - } - } - return result - } + override suspend fun getDetails(manga: Manga): Manga { + copyCookies() + val domain = domain + val slug = manga.url.find(regexLastUrlPath) + ?: throw ParseException("Cannot obtain slug from ${manga.url}", manga.publicUrl) + val data = webClient.httpGet( + url = "https://api.$domain/api/titles$slug/", + ).handle401().parseJson() + val content = try { + data.getJSONObject("content") + } catch (e: JSONException) { + throw ParseException(data.optString("msg"), manga.publicUrl, e) + } + val branchId = content.getJSONArray("branches").optJSONObject(0) + ?.getLong("id") ?: throw ParseException("No branches found", manga.publicUrl) + val chapters = grabChapters(domain, branchId) + val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.US) + return manga.copy( + description = content.getString("description"), + state = when (content.optJSONObject("status")?.getInt("id")) { + STATUS_ONGOING -> MangaState.ONGOING + STATUS_FINISHED -> MangaState.FINISHED + else -> null + }, + tags = content.getJSONArray("genres").mapJSONToSet { g -> + MangaTag( + title = g.getString("name").toTitleCase(), + key = g.getInt("id").toString(), + source = MangaSource.REMANGA, + ) + }, + chapters = chapters.mapChapters { i, jo -> + if ( + jo.getBooleanOrDefault("is_paid", false) && + !jo.getBooleanOrDefault("is_bought", false) + ) { + return@mapChapters null + } + val id = jo.getLong("id") + val name = jo.getString("name").toTitleCase(Locale.ROOT) + val publishers = jo.optJSONArray("publishers") + MangaChapter( + id = generateUid(id), + url = "/api/titles/chapters/$id/", + number = chapters.size - i, + name = buildString { + append("Том ") + append(jo.optString("tome", "0")) + append(". ") + append("Глава ") + append(jo.optString("chapter", "0")) + if (name.isNotEmpty()) { + append(" - ") + append(name) + } + }, + uploadDate = dateFormat.tryParse(jo.getString("upload_date")), + scanlator = publishers?.optJSONObject(0)?.getStringOrNull("name"), + source = MangaSource.REMANGA, + branch = null, + ) + }.asReversed(), + ) + } - override suspend fun getTags(): Set { - val domain = getDomain() - val content = context.httpGet("https://api.$domain/api/forms/titles/?get=genres", getApiHeaders()) - .parseJson().getJSONObject("content").getJSONArray("genres") - return content.mapJSONToSet { jo -> - MangaTag( - title = jo.getString("name").toTitleCase(), - key = jo.getInt("id").toString(), - source = source, - ) - } - } + override suspend fun getPages(chapter: MangaChapter): List { + val referer = "https://${domain}/" + val content = webClient.httpGet(chapter.url.toAbsoluteUrl(getDomain("api"))) + .handle401() + .parseJson() + .getJSONObject("content") + val pages = content.optJSONArray("pages") + if (pages == null) { + val pubDate = content.getStringOrNull("pub_date")?.let { + SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.US).tryParse(it) + } + if (pubDate != null && pubDate > System.currentTimeMillis()) { + val at = SimpleDateFormat.getDateInstance(DateFormat.LONG).format(Date(pubDate)) + throw ContentUnavailableException("Глава станет доступной $at") + } else { + throw ContentUnavailableException("Глава недоступна") + } + } + val result = ArrayList(pages.length()) + for (i in 0 until pages.length()) { + when (val item = pages.get(i)) { + is JSONObject -> result += parsePage(item, referer) + is JSONArray -> item.mapJSONTo(result) { parsePage(it, referer) } + else -> throw ParseException("Unknown json item $item", chapter.url) + } + } + return result + } - override suspend fun getUsername(): String { - val jo = context.httpGet( - url = "https://api.${getDomain()}/api/users/current/", - headers = getApiHeaders(), - ).handle401().parseJson() - return jo.getJSONObject("content").getString("username") - } + override suspend fun getTags(): Set { + val domain = domain + val content = webClient.httpGet("https://api.$domain/api/forms/titles/?get=genres") + .parseJson().getJSONObject("content").getJSONArray("genres") + return content.mapJSONToSet { jo -> + MangaTag( + title = jo.getString("name").toTitleCase(), + key = jo.getInt("id").toString(), + source = source, + ) + } + } - private fun getApiHeaders(): Headers { - val userCookie = context.cookieJar.getCookies(getDomain()).find { - it.name == "user" - } ?: return headers - val jo = JSONObject(URLDecoder.decode(userCookie.value, Charsets.UTF_8.name())) - val accessToken = jo.getStringOrNull("access_token") ?: return headers - return headers.newBuilder().add("authorization", "bearer $accessToken").build() - } + override suspend fun getUsername(): String { + val jo = webClient.httpGet( + url = "https://api.${domain}/api/users/current/", + ).handle401().parseJson() + return jo.getJSONObject("content").getString("username") + } - private fun copyCookies() { - val domain = getDomain() - context.cookieJar.copyCookies(domain, "api.$domain") - } + private fun getApiHeaders(): Headers { + val userCookie = context.cookieJar.getCookies(domain).find { + it.name == "user" + } ?: return baseHeaders + val jo = JSONObject(URLDecoder.decode(userCookie.value, Charsets.UTF_8.name())) + val accessToken = jo.getStringOrNull("access_token") ?: return baseHeaders + return baseHeaders.newBuilder().add("authorization", "bearer $accessToken").build() + } - private fun getSortKey(order: SortOrder?) = when (order) { - SortOrder.UPDATED -> "-chapter_date" - SortOrder.POPULARITY -> "-rating" - SortOrder.RATING -> "-votes" - SortOrder.NEWEST -> "-id" - else -> "-chapter_date" - } + private fun copyCookies() { + val domain = domain + context.cookieJar.copyCookies(domain, "api.$domain") + } - private fun parsePage(jo: JSONObject, referer: String) = MangaPage( - id = generateUid(jo.getLong("id")), - url = jo.getString("link"), - preview = null, - referer = referer, - source = source, - ) + private fun getSortKey(order: SortOrder?) = when (order) { + SortOrder.UPDATED -> "-chapter_date" + SortOrder.POPULARITY -> "-rating" + SortOrder.RATING -> "-votes" + SortOrder.NEWEST -> "-id" + else -> "-chapter_date" + } - private suspend fun grabChapters(domain: String, branchId: Long): List { - val result = ArrayList(100) - var page = 1 - while (true) { - val content = context.httpGet( - url = "https://api.$domain/api/titles/chapters/?branch_id=$branchId&page=$page&count=100", - headers = getApiHeaders(), - ).handle401().parseJson().getJSONArray("content") - val len = content.length() - if (len == 0) { - break - } - result.ensureCapacity(result.size + len) - for (i in 0 until len) { - result.add(content.getJSONObject(i)) - } - page++ - } - return result - } + private fun parsePage(jo: JSONObject, referer: String) = MangaPage( + id = generateUid(jo.getLong("id")), + url = jo.getString("link"), + preview = null, + referer = referer, + source = source, + ) - private fun Response.handle401() = apply { - if (code == HttpURLConnection.HTTP_UNAUTHORIZED) { - throw AuthRequiredException(source) - } - } -} \ No newline at end of file + private suspend fun grabChapters(domain: String, branchId: Long): List { + val result = ArrayList(100) + var page = 1 + while (true) { + val content = webClient.httpGet( + url = "https://api.$domain/api/titles/chapters/?branch_id=$branchId&page=$page&count=100", + ).handle401().parseJson().getJSONArray("content") + val len = content.length() + if (len == 0) { + break + } + result.ensureCapacity(result.size + len) + for (i in 0 until len) { + result.add(content.getJSONObject(i)) + } + page++ + } + return result + } + + private fun Response.handle401() = apply { + if (code == HttpURLConnection.HTTP_UNAUTHORIZED) { + throw AuthRequiredException(source) + } + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/UnionMangasParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/UnionMangasParser.kt index 7e1eb6431..ea07c0fc4 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/UnionMangasParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/UnionMangasParser.kt @@ -13,165 +13,165 @@ import java.text.SimpleDateFormat import java.util.* @MangaSourceParser("UNION_MANGAS", "Union Mangás", "pt") -class UnionMangasParser(override val context: MangaLoaderContext) : PagedMangaParser(MangaSource.UNION_MANGAS, 40) { +class UnionMangasParser(context: MangaLoaderContext) : PagedMangaParser(context, MangaSource.UNION_MANGAS, 40) { - override val sortOrders = EnumSet.of( - SortOrder.ALPHABETICAL, - SortOrder.POPULARITY, - ) + override val sortOrders = EnumSet.of( + SortOrder.ALPHABETICAL, + SortOrder.POPULARITY, + ) - override val configKeyDomain = ConfigKey.Domain("unionleitor.top", emptyArray()) + override val configKeyDomain = ConfigKey.Domain("unionleitor.top", emptyArray()) - override suspend fun getListPage( - page: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - if (!query.isNullOrEmpty()) { - return if (page == searchPaginator.firstPage) { - search(query) - } else { - emptyList() - } - } - val tag = tags.oneOrThrowIfMany() - val url = urlBuilder() - .addPathSegment("lista-mangas") - .addPathSegment( - when { - tag != null -> tag.key - sortOrder == SortOrder.ALPHABETICAL -> "a-z" - else -> "visualizacoes" - }, - ).addPathSegment(page.toString()) - val doc = context.httpGet(url.build()).parseHtml() - val root = doc.selectFirstOrThrow("div.tamanho-bloco-perfil") - return root.select(".lista-mangas-novos").map { div -> - val a = div.selectFirstOrThrow("a") - val img = div.selectFirstOrThrow("img") - val href = a.attrAsRelativeUrl("href") - Manga( - id = generateUid(href), - url = href, - publicUrl = a.attrAsAbsoluteUrl("href"), - title = div.selectLastOrThrow("a").text(), - coverUrl = img.attrAsAbsoluteUrl("src"), - altTitle = null, - rating = RATING_UNKNOWN, - tags = emptySet(), - description = div.selectLast("div")?.ownText(), - state = null, - author = null, - isNsfw = false, - source = source, - ) - } - } + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + if (!query.isNullOrEmpty()) { + return if (page == searchPaginator.firstPage) { + search(query) + } else { + emptyList() + } + } + val tag = tags.oneOrThrowIfMany() + val url = urlBuilder() + .addPathSegment("lista-mangas") + .addPathSegment( + when { + tag != null -> tag.key + sortOrder == SortOrder.ALPHABETICAL -> "a-z" + else -> "visualizacoes" + }, + ).addPathSegment(page.toString()) + val doc = webClient.httpGet(url.build()).parseHtml() + val root = doc.selectFirstOrThrow("div.tamanho-bloco-perfil") + return root.select(".lista-mangas-novos").map { div -> + val a = div.selectFirstOrThrow("a") + val img = div.selectFirstOrThrow("img") + val href = a.attrAsRelativeUrl("href") + Manga( + id = generateUid(href), + url = href, + publicUrl = a.attrAsAbsoluteUrl("href"), + title = div.selectLastOrThrow("a").text(), + coverUrl = img.attrAsAbsoluteUrl("src"), + altTitle = null, + rating = RATING_UNKNOWN, + tags = emptySet(), + description = div.selectLast("div")?.ownText(), + state = null, + author = null, + isNsfw = false, + source = source, + ) + } + } - override suspend fun getDetails(manga: Manga): Manga { - val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml() - val root = doc.selectFirstOrThrow(".perfil-manga") - val dateFormat = SimpleDateFormat("dd/MM/yyyy", Locale.ROOT) - return manga.copy( - rating = root.select("h2") - .find { it.ownText().startsWith('#') } - ?.ownText()?.drop(1)?.toFloatOrNull()?.div(10f) ?: manga.rating, - largeCoverUrl = root.selectFirst("img.img-thumbnail")?.attrAsAbsoluteUrlOrNull("src"), - description = root.selectFirst(".panel-default")?.selectFirst(".panel-body")?.html(), - author = root.tableValue("Autor")?.ownText(), - altTitle = root.tableValue("Título(s) Alternativo(s)")?.ownText(), - state = when (root.tableValue("Status")?.selectLast(".label")?.text()) { - "Completo" -> MangaState.FINISHED - "Ativo" -> MangaState.ONGOING - else -> null - }, - tags = root.tableValue("Gênero(s)")?.select("a")?.mapToSet { - it.toMangaTag() - } ?: manga.tags, - isNsfw = root.selectFirst(".alert-danger")?.html()?.contains("18 anos") == true, - chapters = root.select("div.row.capitulos").asReversed().mapChapters { i, div -> - val a = div.selectFirstOrThrow("a") - val href = a.attrAsRelativeUrl("href") - val title = a.text() - MangaChapter( - id = generateUid(href), - name = title, - number = i + 1, - url = href, - scanlator = div.selectLast("a")?.text()?.takeUnless { it == title }, - uploadDate = dateFormat.tryParse( - a.nextElementSibling()?.text()?.removeSurrounding("(", ")"), - ), - branch = null, - source = source, - ) - }, - ) - } + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val root = doc.selectFirstOrThrow(".perfil-manga") + val dateFormat = SimpleDateFormat("dd/MM/yyyy", Locale.ROOT) + return manga.copy( + rating = root.select("h2") + .find { it.ownText().startsWith('#') } + ?.ownText()?.drop(1)?.toFloatOrNull()?.div(10f) ?: manga.rating, + largeCoverUrl = root.selectFirst("img.img-thumbnail")?.attrAsAbsoluteUrlOrNull("src"), + description = root.selectFirst(".panel-default")?.selectFirst(".panel-body")?.html(), + author = root.tableValue("Autor")?.ownText(), + altTitle = root.tableValue("Título(s) Alternativo(s)")?.ownText(), + state = when (root.tableValue("Status")?.selectLast(".label")?.text()) { + "Completo" -> MangaState.FINISHED + "Ativo" -> MangaState.ONGOING + else -> null + }, + tags = root.tableValue("Gênero(s)")?.select("a")?.mapToSet { + it.toMangaTag() + } ?: manga.tags, + isNsfw = root.selectFirst(".alert-danger")?.html()?.contains("18 anos") == true, + chapters = root.select("div.row.capitulos").asReversed().mapChapters { i, div -> + val a = div.selectFirstOrThrow("a") + val href = a.attrAsRelativeUrl("href") + val title = a.text() + MangaChapter( + id = generateUid(href), + name = title, + number = i + 1, + url = href, + scanlator = div.selectLast("a")?.text()?.takeUnless { it == title }, + uploadDate = dateFormat.tryParse( + a.nextElementSibling()?.text()?.removeSurrounding("(", ")"), + ), + branch = null, + source = source, + ) + }, + ) + } - override suspend fun getPages(chapter: MangaChapter): List { - val fullUrl = chapter.url.toAbsoluteUrl(getDomain()) - val doc = context.httpGet(fullUrl).parseHtml() - val root = doc.body().selectFirstOrThrow("article") - return root.selectOrThrow("img[pag]").mapNotNull { img -> - val href = img.attrAsRelativeUrl("src") - if (href.startsWith("/images/banner")) { - return@mapNotNull null - } - MangaPage( - id = generateUid(href), - url = href, - referer = fullUrl, - preview = null, - source = source, - ) - } - } + override suspend fun getPages(chapter: MangaChapter): List { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + val root = doc.body().selectFirstOrThrow("article") + return root.selectOrThrow("img[pag]").mapNotNull { img -> + val href = img.attrAsRelativeUrl("src") + if (href.startsWith("/images/banner")) { + return@mapNotNull null + } + MangaPage( + id = generateUid(href), + url = href, + referer = fullUrl, + preview = null, + source = source, + ) + } + } - override suspend fun getTags(): Set { - val doc = context.httpGet(urlBuilder().addPathSegment("lista-mangas").build()).parseHtml() - val ul = doc.body().selectFirstOrThrow(".nav-tabs").selectFirstOrThrow("ul.dropdown-menu") - return ul.select("li").mapToSet { li -> - li.selectFirstOrThrow("a").toMangaTag() - } - } + override suspend fun getTags(): Set { + val doc = webClient.httpGet(urlBuilder().addPathSegment("lista-mangas").build()).parseHtml() + val ul = doc.body().selectFirstOrThrow(".nav-tabs").selectFirstOrThrow("ul.dropdown-menu") + return ul.select("li").mapToSet { li -> + li.selectFirstOrThrow("a").toMangaTag() + } + } - private suspend fun search(query: String): List { - val domain = getDomain() - val json = context.httpGet( - urlBuilder() - .addPathSegments("assets/busca.php") - .addQueryParameter("nomeManga", query) - .build(), - ).parseJson() - return json.getJSONArray("items").mapJSON { jo -> - val href = "/pagina-manga/" + jo.getString("url") - Manga( - id = generateUid(href), - url = href, - publicUrl = href.toAbsoluteUrl(domain), - title = jo.getString("titulo"), - rating = RATING_UNKNOWN, - tags = emptySet(), - author = jo.getStringOrNull("autor"), - coverUrl = jo.getString("imagem"), - state = null, - isNsfw = false, - altTitle = null, - source = source, - ) - } - } + private suspend fun search(query: String): List { + val domain = domain + val json = webClient.httpGet( + urlBuilder() + .addPathSegments("assets/busca.php") + .addQueryParameter("nomeManga", query) + .build(), + ).parseJson() + return json.getJSONArray("items").mapJSON { jo -> + val href = "/pagina-manga/" + jo.getString("url") + Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + title = jo.getString("titulo"), + rating = RATING_UNKNOWN, + tags = emptySet(), + author = jo.getStringOrNull("autor"), + coverUrl = jo.getString("imagem"), + state = null, + isNsfw = false, + altTitle = null, + source = source, + ) + } + } - private fun Element.tableValue(title: String): Element? { - return select("h4.media-heading") - .find { it.selectFirst("label.subtit-manga")?.text()?.contains(title, ignoreCase = true) == true } - } + private fun Element.tableValue(title: String): Element? { + return select("h4.media-heading") + .find { it.selectFirst("label.subtit-manga")?.text()?.contains(title, ignoreCase = true) == true } + } - private fun Element.toMangaTag() = MangaTag( - title = text().toTitleCase(sourceLocale ?: Locale.ROOT), - key = attr("href").removeSuffix('/').substringAfterLast('/'), - source = source, - ) -} \ No newline at end of file + private fun Element.toMangaTag() = MangaTag( + title = text().toTitleCase(sourceLocale ?: Locale.ROOT), + key = attr("href").removeSuffix('/').substringAfterLast('/'), + source = source, + ) +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt index 04fb602db..e8f3a94bd 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt @@ -4,6 +4,7 @@ import okhttp3.Headers import okhttp3.HttpUrl.Companion.toHttpUrl import okhttp3.Response import org.json.JSONArray +import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaParser import org.koitharu.kotatsu.parsers.MangaParserAuthProvider import org.koitharu.kotatsu.parsers.exception.AuthRequiredException @@ -20,301 +21,298 @@ private const val NSFW_ALERT = "сексуальные сцены" private const val NOTHING_FOUND = "Ничего не найдено" internal abstract class GroupleParser( - source: MangaSource, - userAgent: String, - private val siteId: Int, -) : MangaParser(source), MangaParserAuthProvider { + context: MangaLoaderContext, + source: MangaSource, + userAgent: String, + private val siteId: Int, +) : MangaParser(context, source), MangaParserAuthProvider { - override val headers = Headers.Builder() - .add("User-Agent", userAgent) - .build() + override val headers = Headers.Builder() + .add("User-Agent", userAgent) + .build() - override val sortOrders: Set = EnumSet.of( - SortOrder.UPDATED, - SortOrder.POPULARITY, - SortOrder.NEWEST, - SortOrder.RATING, - ) + override val sortOrders: Set = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.NEWEST, + SortOrder.RATING, + ) - override val authUrl: String - get() { - val targetUri = "https://${getDomain()}/".urlEncoded() - return "https://grouple.co/internal/auth/sso?siteId=$siteId&=targetUri=$targetUri" - } + override val authUrl: String + get() { + val targetUri = "https://${domain}/".urlEncoded() + return "https://grouple.co/internal/auth/sso?siteId=$siteId&=targetUri=$targetUri" + } - override val isAuthorized: Boolean - get() = context.cookieJar.getCookies(getDomain()).any { it.name == "gwt" } + override val isAuthorized: Boolean + get() = context.cookieJar.getCookies(domain).any { it.name == "gwt" } - override suspend fun getList( - offset: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - val domain = getDomain() - val doc = when { - !query.isNullOrEmpty() -> context.httpPost( - "https://$domain/search", - mapOf( - "q" to query.urlEncoded(), - "offset" to (offset upBy PAGE_SIZE_SEARCH).toString(), - ), - headers, - ) + override suspend fun getList( + offset: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + val domain = domain + val doc = when { + !query.isNullOrEmpty() -> webClient.httpPost( + "https://$domain/search", + mapOf( + "q" to query.urlEncoded(), + "offset" to (offset upBy PAGE_SIZE_SEARCH).toString(), + ), + ) - tags.isNullOrEmpty() -> context.httpGet( - "https://$domain/list?sortType=${ - getSortKey(sortOrder) - }&offset=${offset upBy PAGE_SIZE}", - headers, - ) + tags.isNullOrEmpty() -> webClient.httpGet( + "https://$domain/list?sortType=${ + getSortKey(sortOrder) + }&offset=${offset upBy PAGE_SIZE}", + ) - tags.size == 1 -> context.httpGet( - "https://$domain/list/genre/${tags.first().key}?sortType=${ - getSortKey(sortOrder) - }&offset=${offset upBy PAGE_SIZE}", - headers, - ) + tags.size == 1 -> webClient.httpGet( + "https://$domain/list/genre/${tags.first().key}?sortType=${ + getSortKey(sortOrder) + }&offset=${offset upBy PAGE_SIZE}", + ) - offset > 0 -> return emptyList() - else -> advancedSearch(domain, tags) - }.parseHtml().body() - val root = (doc.getElementById("mangaBox") ?: doc.getElementById("mangaResults")) - ?: doc.parseFailed("Cannot find root") - val tiles = root.selectFirst("div.tiles.row") ?: if ( - root.select(".alert").any { it.ownText() == NOTHING_FOUND } - ) { - return emptyList() - } else { - doc.parseFailed("No tiles found") - } - val baseHost = root.baseUri().toHttpUrl().host - return tiles.select("div.tile").mapNotNull { node -> - val imgDiv = node.selectFirst("div.img") ?: return@mapNotNull null - val descDiv = node.selectFirst("div.desc") ?: return@mapNotNull null - if (descDiv.selectFirst("i.fa-user") != null) { - return@mapNotNull null // skip author - } - val href = imgDiv.selectFirst("a")?.attrAsAbsoluteUrlOrNull("href") - if (href == null || href.toHttpUrl().host != baseHost) { - return@mapNotNull null // skip external links - } - val title = descDiv.selectFirst("h3")?.selectFirst("a")?.text() - ?: return@mapNotNull null - val tileInfo = descDiv.selectFirst("div.tile-info") - val relUrl = href.toRelativeUrl(baseHost) - Manga( - id = generateUid(relUrl), - url = relUrl, - publicUrl = href, - title = title, - altTitle = descDiv.selectFirst("h4")?.text(), - coverUrl = imgDiv.selectFirst("img.lazy")?.attr("data-original")?.replace("_p.", ".").orEmpty(), - rating = runCatching { - node.selectFirst(".compact-rate") - ?.attr("title") - ?.toFloatOrNull() - ?.div(5f) - }.getOrNull() ?: RATING_UNKNOWN, - author = tileInfo?.selectFirst("a.person-link")?.text(), - isNsfw = false, - tags = runCatching { - tileInfo?.select("a.element-link") - ?.mapToSet { - MangaTag( - title = it.text().toTitleCase(), - key = it.attr("href").substringAfterLast('/'), - source = source, - ) - } - }.getOrNull().orEmpty(), - state = when { - node.selectFirst("div.tags") - ?.selectFirst("span.mangaCompleted") != null -> MangaState.FINISHED + offset > 0 -> return emptyList() + else -> advancedSearch(domain, tags) + }.parseHtml().body() + val root = (doc.getElementById("mangaBox") ?: doc.getElementById("mangaResults")) + ?: doc.parseFailed("Cannot find root") + val tiles = root.selectFirst("div.tiles.row") ?: if ( + root.select(".alert").any { it.ownText() == NOTHING_FOUND } + ) { + return emptyList() + } else { + doc.parseFailed("No tiles found") + } + val baseHost = root.baseUri().toHttpUrl().host + return tiles.select("div.tile").mapNotNull { node -> + val imgDiv = node.selectFirst("div.img") ?: return@mapNotNull null + val descDiv = node.selectFirst("div.desc") ?: return@mapNotNull null + if (descDiv.selectFirst("i.fa-user") != null) { + return@mapNotNull null // skip author + } + val href = imgDiv.selectFirst("a")?.attrAsAbsoluteUrlOrNull("href") + if (href == null || href.toHttpUrl().host != baseHost) { + return@mapNotNull null // skip external links + } + val title = descDiv.selectFirst("h3")?.selectFirst("a")?.text() + ?: return@mapNotNull null + val tileInfo = descDiv.selectFirst("div.tile-info") + val relUrl = href.toRelativeUrl(baseHost) + Manga( + id = generateUid(relUrl), + url = relUrl, + publicUrl = href, + title = title, + altTitle = descDiv.selectFirst("h4")?.text(), + coverUrl = imgDiv.selectFirst("img.lazy")?.attr("data-original")?.replace("_p.", ".").orEmpty(), + rating = runCatching { + node.selectFirst(".compact-rate") + ?.attr("title") + ?.toFloatOrNull() + ?.div(5f) + }.getOrNull() ?: RATING_UNKNOWN, + author = tileInfo?.selectFirst("a.person-link")?.text(), + isNsfw = false, + tags = runCatching { + tileInfo?.select("a.element-link") + ?.mapToSet { + MangaTag( + title = it.text().toTitleCase(), + key = it.attr("href").substringAfterLast('/'), + source = source, + ) + } + }.getOrNull().orEmpty(), + state = when { + node.selectFirst("div.tags") + ?.selectFirst("span.mangaCompleted") != null -> MangaState.FINISHED - else -> null - }, - source = source, - ) - } - } + else -> null + }, + source = source, + ) + } + } - override suspend fun getDetails(manga: Manga): Manga { - val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain()), headers).checkAuthRequired().parseHtml() - val root = doc.body().getElementById("mangaBox")?.selectFirst("div.leftContent") - ?: doc.parseFailed("Cannot find root") - val dateFormat = SimpleDateFormat("dd.MM.yy", Locale.US) - val coverImg = root.selectFirst("div.subject-cover")?.selectFirst("img") - return manga.copy( - description = root.selectFirst("div.manga-description")?.html(), - largeCoverUrl = coverImg?.attr("data-full"), - coverUrl = coverImg?.attr("data-thumb") ?: manga.coverUrl, - tags = manga.tags + root.select("div.subject-meta").select("span.elem_genre ") - .mapNotNull { - val a = it.selectFirst("a.element-link") ?: return@mapNotNull null - MangaTag( - title = a.text().toTitleCase(), - key = a.attr("href").substringAfterLast('/'), - source = source, - ) - }, - author = root.selectFirst("a.person-link")?.text() ?: manga.author, - isNsfw = root.select(".alert-warning").any { it.ownText().contains(NSFW_ALERT) }, - chapters = root.selectFirst("div.chapters-link")?.selectFirst("table") - ?.select("tr:has(td > a)")?.asReversed()?.mapChapters { i, tr -> - val a = tr.selectFirst("a.chapter-link") ?: return@mapChapters null - val href = a.attrAsRelativeUrl("href") - var translators = "" - val translatorElement = a.attr("title") - if (!translatorElement.isNullOrBlank()) { - translators = translatorElement - .replace("(Переводчик),", "&") - .removeSuffix(" (Переводчик)") - } - MangaChapter( - id = generateUid(href), - name = tr.selectFirst("a")?.text().orEmpty().removePrefix(manga.title).trim(), - number = i + 1, - url = href, - uploadDate = dateFormat.tryParse(tr.selectFirst("td.date")?.text()), - scanlator = translators, - source = source, - branch = null, - ) - }, - ) - } + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).checkAuthRequired().parseHtml() + val root = doc.body().getElementById("mangaBox")?.selectFirst("div.leftContent") + ?: doc.parseFailed("Cannot find root") + val dateFormat = SimpleDateFormat("dd.MM.yy", Locale.US) + val coverImg = root.selectFirst("div.subject-cover")?.selectFirst("img") + return manga.copy( + description = root.selectFirst("div.manga-description")?.html(), + largeCoverUrl = coverImg?.attr("data-full"), + coverUrl = coverImg?.attr("data-thumb") ?: manga.coverUrl, + tags = manga.tags + root.select("div.subject-meta").select("span.elem_genre ") + .mapNotNull { + val a = it.selectFirst("a.element-link") ?: return@mapNotNull null + MangaTag( + title = a.text().toTitleCase(), + key = a.attr("href").substringAfterLast('/'), + source = source, + ) + }, + author = root.selectFirst("a.person-link")?.text() ?: manga.author, + isNsfw = root.select(".alert-warning").any { it.ownText().contains(NSFW_ALERT) }, + chapters = root.selectFirst("div.chapters-link")?.selectFirst("table") + ?.select("tr:has(td > a)")?.asReversed()?.mapChapters { i, tr -> + val a = tr.selectFirst("a.chapter-link") ?: return@mapChapters null + val href = a.attrAsRelativeUrl("href") + var translators = "" + val translatorElement = a.attr("title") + if (!translatorElement.isNullOrBlank()) { + translators = translatorElement + .replace("(Переводчик),", "&") + .removeSuffix(" (Переводчик)") + } + MangaChapter( + id = generateUid(href), + name = tr.selectFirst("a")?.text().orEmpty().removePrefix(manga.title).trim(), + number = i + 1, + url = href, + uploadDate = dateFormat.tryParse(tr.selectFirst("td.date")?.text()), + scanlator = translators, + source = source, + branch = null, + ) + }, + ) + } - override suspend fun getPages(chapter: MangaChapter): List { - val doc = context.httpGet(chapter.url.toAbsoluteUrl(getDomain()) + "?mtr=1", headers) - .checkAuthRequired() - .parseHtml() - val scripts = doc.select("script") - for (script in scripts) { - val data = script.html() - val pos = data.indexOf("rm_h.readerInit( 0,") - if (pos == -1) { - continue - } - val json = data.substring(pos) - .substringAfter('(') - .substringBefore('\n') - .substringBeforeLast(')') - if (json.isEmpty()) { - continue - } - val ja = JSONArray("[$json]") - val pages = ja.getJSONArray(1) - val servers = ja.getJSONArray(3).mapJSON { it.getString("path") } - val serversStr = servers.joinToString("|") - return (0 until pages.length()).map { i -> - val page = pages.getJSONArray(i) - val primaryServer = page.getString(0) - val url = page.getString(2) - MangaPage( - id = generateUid(url), - url = "$primaryServer|$serversStr|$url", - preview = null, - referer = chapter.url, - source = source, - ) - } - } - doc.parseFailed("Pages list not found at ${chapter.url}") - } + override suspend fun getPages(chapter: MangaChapter): List { + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain) + "?mtr=1") + .checkAuthRequired() + .parseHtml() + val scripts = doc.select("script") + for (script in scripts) { + val data = script.html() + val pos = data.indexOf("rm_h.readerInit( 0,") + if (pos == -1) { + continue + } + val json = data.substring(pos) + .substringAfter('(') + .substringBefore('\n') + .substringBeforeLast(')') + if (json.isEmpty()) { + continue + } + val ja = JSONArray("[$json]") + val pages = ja.getJSONArray(1) + val servers = ja.getJSONArray(3).mapJSON { it.getString("path") } + val serversStr = servers.joinToString("|") + return (0 until pages.length()).map { i -> + val page = pages.getJSONArray(i) + val primaryServer = page.getString(0) + val url = page.getString(2) + MangaPage( + id = generateUid(url), + url = "$primaryServer|$serversStr|$url", + preview = null, + referer = chapter.url, + source = source, + ) + } + } + doc.parseFailed("Pages list not found at ${chapter.url}") + } - override suspend fun getPageUrl(page: MangaPage): String { - val parts = page.url.split('|') - val path = parts.last() - val servers = parts.dropLast(1).toSet() - val headers = Headers.headersOf("Referer", page.referer) - for (server in servers) { - val url = server + path - if (tryHead(url, headers)) { - return url - } - } - val fallbackServer = servers.firstOrNull() ?: throw ParseException("Cannot find any page url", page.url) - return fallbackServer + path - } + override suspend fun getPageUrl(page: MangaPage): String { + val parts = page.url.split('|') + val path = parts.last() + val servers = parts.dropLast(1).toSet() + for (server in servers) { + val url = server + path + if (tryHead(url)) { + return url + } + } + val fallbackServer = servers.firstOrNull() ?: throw ParseException("Cannot find any page url", page.url) + return fallbackServer + path + } - override suspend fun getTags(): Set { - val doc = context.httpGet("https://${getDomain()}/list/genres/sort_name", headers).parseHtml() - val root = doc.body().getElementById("mangaBox")?.selectFirst("div.leftContent") - ?.selectFirst("table.table") ?: doc.parseFailed("Cannot find root") - return root.select("a.element-link").mapToSet { a -> - MangaTag( - title = a.text().toTitleCase(), - key = a.attr("href").substringAfterLast('/'), - source = source, - ) - } - } + override suspend fun getTags(): Set { + val doc = webClient.httpGet("https://${domain}/list/genres/sort_name").parseHtml() + val root = doc.body().getElementById("mangaBox")?.selectFirst("div.leftContent") + ?.selectFirst("table.table") ?: doc.parseFailed("Cannot find root") + return root.select("a.element-link").mapToSet { a -> + MangaTag( + title = a.text().toTitleCase(), + key = a.attr("href").substringAfterLast('/'), + source = source, + ) + } + } - override suspend fun getUsername(): String { - val root = context.httpGet("https://grouple.co/").parseHtml().body() - val element = root.selectFirst("img.user-avatar") ?: throw AuthRequiredException(source) - val res = element.parent()?.text() - return if (res.isNullOrEmpty()) { - root.parseFailed("Cannot find username") - } else res - } + override suspend fun getUsername(): String { + val root = webClient.httpGet("https://grouple.co/").parseHtml().body() + val element = root.selectFirst("img.user-avatar") ?: throw AuthRequiredException(source) + val res = element.parent()?.text() + return if (res.isNullOrEmpty()) { + root.parseFailed("Cannot find username") + } else res + } - private fun getSortKey(sortOrder: SortOrder) = - when (sortOrder) { - SortOrder.ALPHABETICAL -> "name" - SortOrder.POPULARITY -> "rate" - SortOrder.UPDATED -> "updated" - SortOrder.NEWEST -> "created" - SortOrder.RATING -> "votes" - } + private fun getSortKey(sortOrder: SortOrder) = + when (sortOrder) { + SortOrder.ALPHABETICAL -> "name" + SortOrder.POPULARITY -> "rate" + SortOrder.UPDATED -> "updated" + SortOrder.NEWEST -> "created" + SortOrder.RATING -> "votes" + } - private suspend fun advancedSearch(domain: String, tags: Set): Response { - val url = "https://$domain/search/advanced" - // Step 1: map catalog genres names to advanced-search genres ids - val tagsIndex = context.httpGet(url, headers).parseHtml() - .body().selectFirst("form.search-form") - ?.select("div.form-group") - ?.get(1) ?: throw ParseException("Genres filter element not found", url) - val tagNames = tags.map { it.title.lowercase() } - val payload = HashMap() - var foundGenres = 0 - tagsIndex.select("li.property").forEach { li -> - val name = li.text().trim().lowercase() - val id = li.selectFirst("input")?.id() - ?: li.parseFailed("Id for tag $name not found") - payload[id] = if (name in tagNames) { - foundGenres++ - "in" - } else "" - } - if (foundGenres != tags.size) { - tagsIndex.parseFailed("Some genres are not found") - } - // Step 2: advanced search - payload["q"] = "" - payload["s_high_rate"] = "" - payload["s_single"] = "" - payload["s_mature"] = "" - payload["s_completed"] = "" - payload["s_translated"] = "" - payload["s_many_chapters"] = "" - payload["s_wait_upload"] = "" - payload["s_sale"] = "" - payload["years"] = "1900,2099" - payload["+"] = "Искать".urlEncoded() - return context.httpPost(url, payload, headers) - } + private suspend fun advancedSearch(domain: String, tags: Set): Response { + val url = "https://$domain/search/advanced" + // Step 1: map catalog genres names to advanced-search genres ids + val tagsIndex = webClient.httpGet(url).parseHtml() + .body().selectFirst("form.search-form") + ?.select("div.form-group") + ?.get(1) ?: throw ParseException("Genres filter element not found", url) + val tagNames = tags.map { it.title.lowercase() } + val payload = HashMap() + var foundGenres = 0 + tagsIndex.select("li.property").forEach { li -> + val name = li.text().trim().lowercase() + val id = li.selectFirst("input")?.id() + ?: li.parseFailed("Id for tag $name not found") + payload[id] = if (name in tagNames) { + foundGenres++ + "in" + } else "" + } + if (foundGenres != tags.size) { + tagsIndex.parseFailed("Some genres are not found") + } + // Step 2: advanced search + payload["q"] = "" + payload["s_high_rate"] = "" + payload["s_single"] = "" + payload["s_mature"] = "" + payload["s_completed"] = "" + payload["s_translated"] = "" + payload["s_many_chapters"] = "" + payload["s_wait_upload"] = "" + payload["s_sale"] = "" + payload["years"] = "1900,2099" + payload["+"] = "Искать".urlEncoded() + return webClient.httpPost(url, payload) + } - private suspend fun tryHead(url: String, headers: Headers): Boolean = runCatchingCancellable { - context.httpHead(url, headers).isSuccessful - }.getOrDefault(false) + private suspend fun tryHead(url: String): Boolean = runCatchingCancellable { + webClient.httpHead(url).isSuccessful + }.getOrDefault(false) - private fun Response.checkAuthRequired(): Response { - val lastPathSegment = request.url.pathSegments.lastOrNull() ?: return this - if (lastPathSegment == "login") { - throw AuthRequiredException(source) - } - return this - } -} \ No newline at end of file + private fun Response.checkAuthRequired(): Response { + val lastPathSegment = request.url.pathSegments.lastOrNull() ?: return this + if (lastPathSegment == "login") { + throw AuthRequiredException(source) + } + return this + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/MintMangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/MintMangaParser.kt index ecce1f714..ddbbdd3b1 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/MintMangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/MintMangaParser.kt @@ -7,15 +7,15 @@ import org.koitharu.kotatsu.parsers.model.MangaSource @MangaSourceParser("MINTMANGA", "MintManga", "ru") internal class MintMangaParser( - override val context: MangaLoaderContext, -) : GroupleParser(MangaSource.MINTMANGA, "mintmangafun", 2) { + context: MangaLoaderContext, +) : GroupleParser(context, MangaSource.MINTMANGA, "mintmangafun", 2) { - override val configKeyDomain = ConfigKey.Domain( - "mintmanga.live", - arrayOf("mintmanga.live", "mintmanga.com"), - ) + override val configKeyDomain = ConfigKey.Domain( + "mintmanga.live", + arrayOf("mintmanga.live", "mintmanga.com"), + ) - override fun getFaviconUrl(): String { - return "https://resmm.rmr.rocks/static/apple-touch-icon-8fff291039c140493adb0c7ba81065ad.png" - } -} \ No newline at end of file + override fun getFaviconUrl(): String { + return "https://resmm.rmr.rocks/static/apple-touch-icon-8fff291039c140493adb0c7ba81065ad.png" + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/ReadmangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/ReadmangaParser.kt index 6ab267714..f4fe90d48 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/ReadmangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/ReadmangaParser.kt @@ -7,15 +7,15 @@ import org.koitharu.kotatsu.parsers.model.MangaSource @MangaSourceParser("READMANGA_RU", "ReadManga", "ru") internal class ReadmangaParser( - override val context: MangaLoaderContext, -) : GroupleParser(MangaSource.READMANGA_RU, "readmangafun", 1) { + context: MangaLoaderContext, +) : GroupleParser(context, MangaSource.READMANGA_RU, "readmangafun", 1) { - override val configKeyDomain = ConfigKey.Domain( - "readmanga.live", - arrayOf("readmanga.io", "readmanga.live", "readmanga.me"), - ) + override val configKeyDomain = ConfigKey.Domain( + "readmanga.live", + arrayOf("readmanga.io", "readmanga.live", "readmanga.me"), + ) - override fun getFaviconUrl(): String { - return "https://resrm.rmr.rocks/static/apple-touch-icon-3162037c9df9f28dca0f9a4092cb0f65.png" - } -} \ No newline at end of file + override fun getFaviconUrl(): String { + return "https://resrm.rmr.rocks/static/apple-touch-icon-3162037c9df9f28dca0f9a4092cb0f65.png" + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/SelfMangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/SelfMangaParser.kt index 3591eb710..912958086 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/SelfMangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/SelfMangaParser.kt @@ -7,12 +7,12 @@ import org.koitharu.kotatsu.parsers.model.MangaSource @MangaSourceParser("SELFMANGA", "SelfManga", "ru") internal class SelfMangaParser( - override val context: MangaLoaderContext, -) : GroupleParser(MangaSource.SELFMANGA, "selfmangafun", 3) { + context: MangaLoaderContext, +) : GroupleParser(context, MangaSource.SELFMANGA, "selfmangafun", 3) { - override val configKeyDomain = ConfigKey.Domain("selfmanga.live", null) + override val configKeyDomain = ConfigKey.Domain("selfmanga.live", null) - override fun getFaviconUrl(): String { - return "https://ressm.rmr.rocks/static/apple-touch-icon-a769ea533d811b73ac3eedde658bb1d3.png" - } -} \ No newline at end of file + override fun getFaviconUrl(): String { + return "https://ressm.rmr.rocks/static/apple-touch-icon-a769ea533d811b73ac3eedde658bb1d3.png" + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/IsekaiScanEu.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/IsekaiScanEu.kt index f2b2c6b9c..526353e58 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/IsekaiScanEu.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/IsekaiScanEu.kt @@ -12,36 +12,36 @@ import java.util.* @MangaSourceParser("ISEKAISCAN_EU", "IsekaiScan (eu)", "en") internal class IsekaiScanEuParser(context: MangaLoaderContext) : - MadaraParser(context, MangaSource.ISEKAISCAN_EU, "isekaiscan.eu") { + MadaraParser(context, MangaSource.ISEKAISCAN_EU, "isekaiscan.eu") { - override val datePattern = "MM/dd/yyyy" + override val datePattern = "MM/dd/yyyy" - override suspend fun getChapters(manga: Manga, doc: Document): List { - val mangaId = doc.body().requireElementById("manga-chapters-holder").attr("data-id") - val ul = context.httpPost( - "https://${getDomain()}/wp-admin/admin-ajax.php", - mapOf( - "action" to "manga_get_chapters", - "manga" to mangaId, - ), - ).parseHtml().body().selectFirstOrThrow("ul") - val dateFormat = SimpleDateFormat(datePattern, Locale.US) - return ul.select("li").asReversed().mapChapters { i, li -> - val a = li.selectFirst("a") - val href = a?.attrAsRelativeUrlOrNull("href") ?: li.parseFailed("Link is missing") - MangaChapter( - id = generateUid(href), - name = a.ownText(), - number = i + 1, - url = href, - uploadDate = parseChapterDate( - dateFormat, - li.selectFirst("span.chapter-release-date i")?.text(), - ), - source = source, - scanlator = null, - branch = null, - ) - } - } -} \ No newline at end of file + override suspend fun getChapters(manga: Manga, doc: Document): List { + val mangaId = doc.body().requireElementById("manga-chapters-holder").attr("data-id") + val ul = webClient.httpPost( + "https://${domain}/wp-admin/admin-ajax.php", + mapOf( + "action" to "manga_get_chapters", + "manga" to mangaId, + ), + ).parseHtml().body().selectFirstOrThrow("ul") + val dateFormat = SimpleDateFormat(datePattern, Locale.US) + return ul.select("li").asReversed().mapChapters { i, li -> + val a = li.selectFirst("a") + val href = a?.attrAsRelativeUrlOrNull("href") ?: li.parseFailed("Link is missing") + MangaChapter( + id = generateUid(href), + name = a.ownText(), + number = i + 1, + url = href, + uploadDate = parseChapterDate( + dateFormat, + li.selectFirst("span.chapter-release-date i")?.text(), + ), + source = source, + scanlator = null, + branch = null, + ) + } + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Madara5Parser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Madara5Parser.kt index 26072e613..ae2e4110a 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Madara5Parser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Madara5Parser.kt @@ -13,175 +13,175 @@ import java.text.SimpleDateFormat import java.util.* abstract class Madara5Parser @InternalParsersApi constructor( - override val context: MangaLoaderContext, - source: MangaSource, - domain: String, -) : PagedMangaParser(source, pageSize = 22) { - - protected open val datePattern = "MMMM dd, HH:mm" - protected open val tagPrefix = "/mangas/" - protected open val nsfwTags = arraySetOf("yaoi", "yuri", "mature") - - override val sortOrders: Set = EnumSet.of(SortOrder.UPDATED) - - override val configKeyDomain = ConfigKey.Domain(domain, null) - - override suspend fun getListPage( - page: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - val domain = getDomain() - val url = buildString { - append("https://") - append(domain) - append("/search?s=") - if (!query.isNullOrEmpty()) { - append(query.urlEncoded()) - } - append("&post_type=wp-manga") - if (!tags.isNullOrEmpty()) { - for (tag in tags) { - append("&genre%5B%5D=") - append(tag.key) - } - } - append("&op=1&author=&artist=&page=") - append(page) - } - val root = context.httpGet(url).parseHtml().body().selectFirstOrThrow(".search-wrap") - return root.select(".c-tabs-item__content").map { div -> - val a = div.selectFirstOrThrow("a") - val img = div.selectLastOrThrow("img") - val href = a.attrAsRelativeUrl("href") - val postContent = root.selectFirstOrThrow(".post-content") - val tagSet = postContent.getElementsContainingOwnText("Genre") - .firstOrNull()?.tableValue() - ?.getElementsByAttributeValueContaining("href", tagPrefix) - ?.mapToSet { it.asMangaTag() }.orEmpty() - Manga( - id = generateUid(href), - title = a.attr("title"), - altTitle = postContent.getElementsContainingOwnText("Alternative") - .firstOrNull()?.tableValue()?.text()?.trim(), - url = href, - publicUrl = a.attrAsAbsoluteUrl("href"), - coverUrl = img.src().orEmpty(), - author = postContent.getElementsContainingOwnText("Author") - .firstOrNull()?.tableValue()?.text()?.trim(), - state = postContent.getElementsContainingOwnText("Status") - .firstOrNull()?.tableValue()?.text()?.asMangaState(), - isNsfw = isNsfw(tagSet), - rating = div.selectFirstOrThrow(".score").text() - .toFloatOrNull()?.div(5f) ?: RATING_UNKNOWN, - tags = tagSet, - source = source, - ) - } - } - - override suspend fun getDetails(manga: Manga): Manga { - val root = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml().body() - .selectFirstOrThrow(".site-content") - val postContent = root.selectFirstOrThrow(".post-content") - val tags = postContent.getElementsContainingOwnText("Genre") - .firstOrNull()?.tableValue() - ?.getElementsByAttributeValueContaining("href", tagPrefix) - ?.mapToSet { a -> a.asMangaTag() } ?: manga.tags - val mangaId = root.getElementById("manga-chapters-holder")?.attr("data-id")?.toLongOrNull() - ?: root.parseFailed("Cannot find mangaId") - return manga.copy( - description = (root.selectFirst(".detail-content") - ?: root.selectFirstOrThrow(".description-summary")).html(), - author = postContent.getElementsContainingOwnText("Author") - .firstOrNull()?.tableValue()?.text()?.trim(), - state = postContent.getElementsContainingOwnText("Status") - .firstOrNull()?.tableValue()?.text()?.asMangaState(), - tags = tags, - isNsfw = isNsfw(tags), - chapters = loadChapters(mangaId), - ) - } - - override suspend fun getPages(chapter: MangaChapter): List { - val fullUrl = chapter.url.toAbsoluteUrl(getDomain()) - val doc = context.httpGet(fullUrl).parseHtml() - val arrayData = doc.getElementById("arraydata") ?: doc.parseFailed("#arraydata not found") - return arrayData.html().split(',').map { url -> - MangaPage( - id = generateUid(url), - url = url, - referer = fullUrl, - preview = null, - source = source, - ) - } - } - - override suspend fun getTags(): Set { - val doc = context.httpGet("http://${getDomain()}/").parseHtml().body() - return doc.getElementsByAttributeValueContaining("href", tagPrefix) - .mapToSet { it.asMangaTag() } - } - - private suspend fun loadChapters(mangaId: Long): List { - val dateFormat = SimpleDateFormat(datePattern, sourceLocale ?: Locale.US) - val doc = context.httpGet("https://${getDomain()}/ajax-list-chapter?mangaID=$mangaId").parseHtml() - return doc.select("li.wp-manga-chapter").asReversed().mapChapters { i, li -> - val a = li.selectFirstOrThrow("a") - val href = a.attrAsRelativeUrl("href") - MangaChapter( - id = generateUid(href), - url = href, - name = a.text(), - number = i + 1, - branch = null, - uploadDate = dateFormat.tryParse( - li.selectFirst(".chapter-release-date")?.text()?.trim(), - ), - scanlator = null, - source = source, - ) - } - } - - protected fun isNsfw(tags: Set): Boolean { - return tags.any { it.key in nsfwTags } - } - - private fun Element.src(): String? { - return absUrl("data-src").ifEmpty { - absUrl("src") - }.takeUnless { it.isEmpty() } - } - - private fun Element.tableValue(): Element { - for (p in parents()) { - val children = p.children() - if (children.size == 2) { - return children[1] - } - } - parseFailed("Cannot find tableValue for node ${text()}") - } - - private fun String.asMangaState() = when (trim().lowercase(sourceLocale ?: Locale.US)) { - "ongoing" -> MangaState.ONGOING - "completed" -> MangaState.FINISHED - else -> null - } - - private fun Element.asMangaTag() = MangaTag( - title = ownText(), - key = attr("href").removeSuffix('/').substringAfterLast('/') - .replace('-', '+'), - source = source, - ) - - @MangaSourceParser("MANGAOWLS", "BeautyManga", "en") - class BeautyManga(context: MangaLoaderContext) : Madara5Parser(context, MangaSource.MANGAOWLS, "beautymanga.com") { - - override fun getFaviconUrl() = "http://${getDomain()}/frontend/images/favico.png" - } + context: MangaLoaderContext, + source: MangaSource, + domain: String, +) : PagedMangaParser(context, source, pageSize = 22) { + + protected open val datePattern = "MMMM dd, HH:mm" + protected open val tagPrefix = "/mangas/" + protected open val nsfwTags = arraySetOf("yaoi", "yuri", "mature") + + override val sortOrders: Set = EnumSet.of(SortOrder.UPDATED) + + override val configKeyDomain = ConfigKey.Domain(domain, null) + + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + val domain = domain + val url = buildString { + append("https://") + append(domain) + append("/search?s=") + if (!query.isNullOrEmpty()) { + append(query.urlEncoded()) + } + append("&post_type=wp-manga") + if (!tags.isNullOrEmpty()) { + for (tag in tags) { + append("&genre%5B%5D=") + append(tag.key) + } + } + append("&op=1&author=&artist=&page=") + append(page) + } + val root = webClient.httpGet(url).parseHtml().body().selectFirstOrThrow(".search-wrap") + return root.select(".c-tabs-item__content").map { div -> + val a = div.selectFirstOrThrow("a") + val img = div.selectLastOrThrow("img") + val href = a.attrAsRelativeUrl("href") + val postContent = root.selectFirstOrThrow(".post-content") + val tagSet = postContent.getElementsContainingOwnText("Genre") + .firstOrNull()?.tableValue() + ?.getElementsByAttributeValueContaining("href", tagPrefix) + ?.mapToSet { it.asMangaTag() }.orEmpty() + Manga( + id = generateUid(href), + title = a.attr("title"), + altTitle = postContent.getElementsContainingOwnText("Alternative") + .firstOrNull()?.tableValue()?.text()?.trim(), + url = href, + publicUrl = a.attrAsAbsoluteUrl("href"), + coverUrl = img.src().orEmpty(), + author = postContent.getElementsContainingOwnText("Author") + .firstOrNull()?.tableValue()?.text()?.trim(), + state = postContent.getElementsContainingOwnText("Status") + .firstOrNull()?.tableValue()?.text()?.asMangaState(), + isNsfw = isNsfw(tagSet), + rating = div.selectFirstOrThrow(".score").text() + .toFloatOrNull()?.div(5f) ?: RATING_UNKNOWN, + tags = tagSet, + source = source, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val root = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml().body() + .selectFirstOrThrow(".site-content") + val postContent = root.selectFirstOrThrow(".post-content") + val tags = postContent.getElementsContainingOwnText("Genre") + .firstOrNull()?.tableValue() + ?.getElementsByAttributeValueContaining("href", tagPrefix) + ?.mapToSet { a -> a.asMangaTag() } ?: manga.tags + val mangaId = root.getElementById("manga-chapters-holder")?.attr("data-id")?.toLongOrNull() + ?: root.parseFailed("Cannot find mangaId") + return manga.copy( + description = (root.selectFirst(".detail-content") + ?: root.selectFirstOrThrow(".description-summary")).html(), + author = postContent.getElementsContainingOwnText("Author") + .firstOrNull()?.tableValue()?.text()?.trim(), + state = postContent.getElementsContainingOwnText("Status") + .firstOrNull()?.tableValue()?.text()?.asMangaState(), + tags = tags, + isNsfw = isNsfw(tags), + chapters = loadChapters(mangaId), + ) + } + + override suspend fun getPages(chapter: MangaChapter): List { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + val arrayData = doc.getElementById("arraydata") ?: doc.parseFailed("#arraydata not found") + return arrayData.html().split(',').map { url -> + MangaPage( + id = generateUid(url), + url = url, + referer = fullUrl, + preview = null, + source = source, + ) + } + } + + override suspend fun getTags(): Set { + val doc = webClient.httpGet("http://${domain}/").parseHtml().body() + return doc.getElementsByAttributeValueContaining("href", tagPrefix) + .mapToSet { it.asMangaTag() } + } + + private suspend fun loadChapters(mangaId: Long): List { + val dateFormat = SimpleDateFormat(datePattern, sourceLocale ?: Locale.US) + val doc = webClient.httpGet("https://${domain}/ajax-list-chapter?mangaID=$mangaId").parseHtml() + return doc.select("li.wp-manga-chapter").asReversed().mapChapters { i, li -> + val a = li.selectFirstOrThrow("a") + val href = a.attrAsRelativeUrl("href") + MangaChapter( + id = generateUid(href), + url = href, + name = a.text(), + number = i + 1, + branch = null, + uploadDate = dateFormat.tryParse( + li.selectFirst(".chapter-release-date")?.text()?.trim(), + ), + scanlator = null, + source = source, + ) + } + } + + protected fun isNsfw(tags: Set): Boolean { + return tags.any { it.key in nsfwTags } + } + + private fun Element.src(): String? { + return absUrl("data-src").ifEmpty { + absUrl("src") + }.takeUnless { it.isEmpty() } + } + + private fun Element.tableValue(): Element { + for (p in parents()) { + val children = p.children() + if (children.size == 2) { + return children[1] + } + } + parseFailed("Cannot find tableValue for node ${text()}") + } + + private fun String.asMangaState() = when (trim().lowercase(sourceLocale ?: Locale.US)) { + "ongoing" -> MangaState.ONGOING + "completed" -> MangaState.FINISHED + else -> null + } + + private fun Element.asMangaTag() = MangaTag( + title = ownText(), + key = attr("href").removeSuffix('/').substringAfterLast('/') + .replace('-', '+'), + source = source, + ) + + @MangaSourceParser("MANGAOWLS", "BeautyManga", "en") + class BeautyManga(context: MangaLoaderContext) : Madara5Parser(context, MangaSource.MANGAOWLS, "beautymanga.com") { + + override fun getFaviconUrl() = "http://${domain}/frontend/images/favico.png" + } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Madara6Parser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Madara6Parser.kt index 5fa04a6ce..a38f4b97b 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Madara6Parser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Madara6Parser.kt @@ -10,61 +10,61 @@ import java.text.SimpleDateFormat import java.util.* internal abstract class Madara6Parser( - context: MangaLoaderContext, - source: MangaSource, - domain: String, + context: MangaLoaderContext, + source: MangaSource, + domain: String, ) : MadaraParser(context, source, domain) { - override val datePattern: String = "dd MMMM yyyy" + override val datePattern: String = "dd MMMM yyyy" - override suspend fun getDetails(manga: Manga): Manga { - return coroutineScope { - val chapters = async { loadChapters(manga.url) } - val body = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml().body() - parseDetails(manga, body, chapters.await()) - } - } + override suspend fun getDetails(manga: Manga): Manga { + return coroutineScope { + val chapters = async { loadChapters(manga.url) } + val body = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml().body() + parseDetails(manga, body, chapters.await()) + } + } - protected fun Element.tableValue(): Element { - for (p in parents()) { - val children = p.children() - if (children.size == 2) { - return children[1] - } - } - parseFailed("Cannot find tableValue for node ${text()}") - } + protected fun Element.tableValue(): Element { + for (p in parents()) { + val children = p.children() + if (children.size == 2) { + return children[1] + } + } + parseFailed("Cannot find tableValue for node ${text()}") + } - protected abstract fun String.asMangaState(): MangaState? + protected abstract fun String.asMangaState(): MangaState? - protected fun Element.asMangaTag() = MangaTag( - title = ownText(), - key = attr("href").removeSuffix('/').substringAfterLast('/') - .replace('-', '+'), - source = source, - ) + protected fun Element.asMangaTag() = MangaTag( + title = ownText(), + key = attr("href").removeSuffix('/').substringAfterLast('/') + .replace('-', '+'), + source = source, + ) - protected open suspend fun loadChapters(mangaUrl: String): List { - val url = mangaUrl.toAbsoluteUrl(getDomain()).removeSuffix('/') + "/ajax/chapters/" - val dateFormat = SimpleDateFormat(datePattern, sourceLocale ?: Locale.ROOT) - val doc = context.httpPost(url, emptyMap()).parseHtml() - return doc.select("li.wp-manga-chapter").asReversed().mapChapters { i, li -> - val a = li.selectFirstOrThrow("a") - val href = a.attrAsRelativeUrl("href") - MangaChapter( - id = generateUid(href), - url = href, - name = a.text(), - number = i + 1, - branch = null, - uploadDate = dateFormat.tryParse( - li.selectFirst(".chapter-release-date")?.text()?.trim(), - ), - scanlator = null, - source = source, - ) - } - } + protected open suspend fun loadChapters(mangaUrl: String): List { + val url = mangaUrl.toAbsoluteUrl(domain).removeSuffix('/') + "/ajax/chapters/" + val dateFormat = SimpleDateFormat(datePattern, sourceLocale ?: Locale.ROOT) + val doc = webClient.httpPost(url, emptyMap()).parseHtml() + return doc.select("li.wp-manga-chapter").asReversed().mapChapters { i, li -> + val a = li.selectFirstOrThrow("a") + val href = a.attrAsRelativeUrl("href") + MangaChapter( + id = generateUid(href), + url = href, + name = a.text(), + number = i + 1, + branch = null, + uploadDate = dateFormat.tryParse( + li.selectFirst(".chapter-release-date")?.text()?.trim(), + ), + scanlator = null, + source = source, + ) + } + } - protected abstract fun parseDetails(manga: Manga, body: Element, chapters: List): Manga -} \ No newline at end of file + protected abstract fun parseDetails(manga: Manga, body: Element, chapters: List): Manga +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt index 3885663a1..ad7a6b59e 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt @@ -16,503 +16,503 @@ import java.text.SimpleDateFormat import java.util.* internal abstract class MadaraParser( - override val context: MangaLoaderContext, - source: MangaSource, - domain: String, -) : PagedMangaParser(source, pageSize = 12) { - - override val configKeyDomain = ConfigKey.Domain(domain, null) - - override val sortOrders: Set = EnumSet.of( - SortOrder.UPDATED, - SortOrder.POPULARITY, - ) - - protected open val tagPrefix = "manga-genre/" - protected open val isNsfwSource = false - protected open val datePattern = "MMMM dd, yyyy" - - init { - paginator.firstPage = 0 - searchPaginator.firstPage = 0 - } - - override suspend fun getListPage( - page: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - val tag = tags.oneOrThrowIfMany() - val payload = createRequestTemplate() - payload["page"] = page.toString() - payload["vars[meta_key]"] = when (sortOrder) { - SortOrder.POPULARITY -> "_wp_manga_views" - SortOrder.UPDATED -> "_latest_update" - else -> "_wp_manga_views" - } - payload["vars[wp-manga-genre]"] = tag?.key.orEmpty() - payload["vars[s]"] = query?.urlEncoded().orEmpty() - val doc = context.httpPost( - "https://${getDomain()}/wp-admin/admin-ajax.php", - payload, - ).parseHtml() - return doc.select("div.row.c-tabs-item__content").ifEmpty { - doc.select("div.page-item-detail.manga") - }.map { div -> - val href = div.selectFirst("a")?.attrAsRelativeUrlOrNull("href") - ?: div.parseFailed("Link not found") - val summary = div.selectFirst(".tab-summary") ?: div.selectFirst(".item-summary") - Manga( - id = generateUid(href), - url = href, - publicUrl = href.toAbsoluteUrl(div.host ?: getDomain()), - coverUrl = div.selectFirst("img")?.src().orEmpty(), - title = (summary?.selectFirst("h3") ?: summary?.selectFirst("h4"))?.text().orEmpty(), - altTitle = null, - rating = div.selectFirst("span.total_votes")?.ownText() - ?.toFloatOrNull()?.div(5f) ?: -1f, - tags = summary?.selectFirst(".mg_genres")?.select("a")?.mapToSet { a -> - MangaTag( - key = a.attr("href").removeSuffix("/").substringAfterLast('/'), - title = a.text().toTitleCase(), - source = source, - ) - }.orEmpty(), - author = summary?.selectFirst(".mg_author")?.selectFirst("a")?.ownText(), - state = when ( - summary?.selectFirst(".mg_status")?.selectFirst(".summary-content") - ?.ownText()?.trim()?.lowercase() - ) { - "ongoing" -> MangaState.ONGOING - "completed" -> MangaState.FINISHED - else -> null - }, - source = source, - isNsfw = isNsfwSource, - ) - } - } - - override suspend fun getTags(): Set { - val doc = context.httpGet("https://${getDomain()}/manga/").parseHtml() - val body = doc.body() - val root1 = body.selectFirst("header")?.selectFirst("ul.second-menu") - val root2 = body.selectFirst("div.genres_wrap")?.selectFirst("ul.list-unstyled") - if (root1 == null && root2 == null) { - doc.parseFailed("Root not found") - } - val list = root1?.select("li").orEmpty() + root2?.select("li").orEmpty() - val keySet = HashSet(list.size) - return list.mapNotNullToSet { li -> - val a = li.selectFirst("a") ?: return@mapNotNullToSet null - val href = a.attr("href").removeSuffix("/") - .substringAfterLast(tagPrefix, "") - if (href.isEmpty() || !keySet.add(href)) { - return@mapNotNullToSet null - } - MangaTag( - key = href, - title = a.ownText().trim().ifEmpty { - a.selectFirst(".menu-image-title")?.text()?.trim() ?: return@mapNotNullToSet null - }.toTitleCase(), - source = source, - ) - } - } - - override suspend fun getDetails(manga: Manga): Manga = coroutineScope { - val fullUrl = manga.url.toAbsoluteUrl(getDomain()) - val doc = context.httpGet(fullUrl).parseHtml() - val chaptersDeferred = async { getChapters(manga, doc) } - val root = doc.body().selectFirst("div.profile-manga") - ?.selectFirst("div.summary_content") - ?.selectFirst("div.post-content") - ?: throw ParseException("Root not found", fullUrl) - val root2 = doc.body().selectFirst("div.content-area") - ?.selectFirst("div.c-page") - ?: throw ParseException("Root2 not found", fullUrl) - manga.copy( - tags = root.selectFirst("div.genres-content")?.select("a") - ?.mapNotNullToSet { a -> - MangaTag( - key = a.attr("href").removeSuffix("/").substringAfterLast('/'), - title = a.text().toTitleCase(), - source = source, - ) - } ?: manga.tags, - description = root2.selectFirst("div.description-summary") - ?.selectFirst("div.summary__content") - ?.select("p") - ?.filterNot { it.ownText().startsWith("A brief description") } - ?.joinToString { it.html() }, - chapters = chaptersDeferred.await(), - ) - } - - protected open suspend fun getChapters(manga: Manga, doc: Document): List { - val root2 = doc.body().selectFirstOrThrow("div.content-area") - .selectFirstOrThrow("div.c-page") - val dateFormat = SimpleDateFormat(datePattern, sourceLocale ?: Locale.US) - return root2.select("li").asReversed().mapChapters { i, li -> - val a = li.selectFirst("a") - val href = a?.attrAsRelativeUrlOrNull("href") ?: li.parseFailed("Link is missing") - MangaChapter( - id = generateUid(href), - name = a.ownText(), - number = i + 1, - url = href, - uploadDate = parseChapterDate( - dateFormat, - li.selectFirst("span.chapter-release-date i")?.text(), - ), - source = source, - scanlator = null, - branch = null, - ) - } - } - - override suspend fun getPages(chapter: MangaChapter): List { - val fullUrl = chapter.url.toAbsoluteUrl(getDomain()) - val doc = context.httpGet(fullUrl).parseHtml() - val root = doc.body().selectFirst("div.main-col-inner") - ?.selectFirst("div.reading-content") - ?: throw ParseException("Root not found", fullUrl) - return root.select("div.page-break").map { div -> - val img = div.selectFirst("img") ?: div.parseFailed("Page image not found") - val url = img.src()?.toRelativeUrl(getDomain()) ?: div.parseFailed("Image src not found") - MangaPage( - id = generateUid(url), - url = url, - preview = null, - referer = fullUrl, - source = source, - ) - } - } - - protected fun parseChapterDate(dateFormat: DateFormat, date: String?): Long { - date ?: return 0 - return when { - date.endsWith(" ago", ignoreCase = true) -> { - parseRelativeDate(date) - } - // Handle translated 'ago' in Portuguese. - date.endsWith(" atrás", ignoreCase = true) -> { - parseRelativeDate(date) - } - // Handle translated 'ago' in Turkish. - date.endsWith(" önce", ignoreCase = true) -> { - parseRelativeDate(date) - } - // Handle 'yesterday' and 'today', using midnight - date.startsWith("year", ignoreCase = true) -> { - Calendar.getInstance().apply { - add(Calendar.DAY_OF_MONTH, -1) // yesterday - set(Calendar.HOUR_OF_DAY, 0) - set(Calendar.MINUTE, 0) - set(Calendar.SECOND, 0) - set(Calendar.MILLISECOND, 0) - }.timeInMillis - } - - date.startsWith("today", ignoreCase = true) -> { - Calendar.getInstance().apply { - set(Calendar.HOUR_OF_DAY, 0) - set(Calendar.MINUTE, 0) - set(Calendar.SECOND, 0) - set(Calendar.MILLISECOND, 0) - }.timeInMillis - } - - date.contains(Regex("""\d(st|nd|rd|th)""")) -> { - // Clean date (e.g. 5th December 2019 to 5 December 2019) before parsing it - date.split(" ").map { - if (it.contains(Regex("""\d\D\D"""))) { - it.replace(Regex("""\D"""), "") - } else { - it - } - } - .let { dateFormat.tryParse(it.joinToString(" ")) } - } - - else -> dateFormat.tryParse(date) - } - } - - // Parses dates in this form: - // 21 hours ago - private fun parseRelativeDate(date: String): Long { - val number = Regex("""(\d+)""").find(date)?.value?.toIntOrNull() ?: return 0 - val cal = Calendar.getInstance() - - return when { - WordSet( - "hari", - "gün", - "jour", - "día", - "dia", - "day", - ).anyWordIn(date) -> cal.apply { add(Calendar.DAY_OF_MONTH, -number) }.timeInMillis - - WordSet("jam", "saat", "heure", "hora", "hour").anyWordIn(date) -> cal.apply { - add( - Calendar.HOUR, - -number, - ) - }.timeInMillis - - WordSet("menit", "dakika", "min", "minute", "minuto").anyWordIn(date) -> cal.apply { - add( - Calendar.MINUTE, - -number, - ) - }.timeInMillis - - WordSet("detik", "segundo", "second").anyWordIn(date) -> cal.apply { - add( - Calendar.SECOND, - -number, - ) - }.timeInMillis - - WordSet("month").anyWordIn(date) -> cal.apply { add(Calendar.MONTH, -number) }.timeInMillis - WordSet("year").anyWordIn(date) -> cal.apply { add(Calendar.YEAR, -number) }.timeInMillis - else -> 0 - } - } - - private fun Element.src(): String? { - var result = absUrl("data-src") - if (result.isEmpty()) result = absUrl("data-cfsrc") - if (result.isEmpty()) result = absUrl("src") - return if (result.isEmpty()) null else result - } - - private fun createRequestTemplate() = - ( - "action=madara_load_more&page=1&template=madara-core%2Fcontent%2Fcontent-search&vars%5Bs%5D=&vars%5B" + - "orderby%5D=meta_value_num&vars%5Bpaged%5D=1&vars%5Btemplate%5D=search&vars%5Bmeta_query" + - "%5D%5B0%5D%5Brelation%5D=AND&vars%5Bmeta_query%5D%5Brelation%5D=OR&vars%5Bpost_type" + - "%5D=wp-manga&vars%5Bpost_status%5D=publish&vars%5Bmeta_key%5D=_latest_update&vars%5Border" + - "%5D=desc&vars%5Bmanga_archives_item_layout%5D=default" - ).split('&') - .map { - val pos = it.indexOf('=') - it.substring(0, pos) to it.substring(pos + 1) - }.toMutableMap() - - @MangaSourceParser("MANGAREAD", "MangaRead", "en") - class MangaRead(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGAREAD, "www.mangaread.org") { - override val tagPrefix = "genres/" - override val datePattern = "dd.MM.yyyy" - } - - @MangaSourceParser("MANGAWEEBS", "MangaWeebs", "en") - class MangaWeebs(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGAWEEBS, "mangaweebs.in") { - override val datePattern = "dd MMMM HH:mm" - } - - @MangaSourceParser("HachiManga", "HACHIMANGA", "ja") - class HachiManga(context: MangaLoaderContext) : - MadaraParser(context, MangaSource.HACHIMANGA, "hachimanga.com") { - - override val sourceLocale: Locale = Locale.ENGLISH - } - - @MangaSourceParser("PIANMANGA", "PianManga", "en") - class PianManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.PIANMANGA, "pianmanga.me") - - @MangaSourceParser("MANGAROSIE", "MangaRosie", "en") - class MangaRosie(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGAROSIE, "mangarosie.in") - - @MangaSourceParser("MANGATX", "MangaTx", "en") - class MangaTx(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGATX, "mangatx.com") - - @MangaSourceParser("MANGAEFFECT", "MangaEffect", "en") - class MangaEffect(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGAEFFECT, "mangaeffect.com") { - override val datePattern = "dd.MM.yyyy" - } - - @MangaSourceParser("AQUAMANGA", "AquaManga", "en") - class AquaManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.AQUAMANGA, "aquamanga.com") { - - override fun getFaviconUrl(): String { - return "https://${getDomain()}/wp-content/uploads/2021/03/cropped-cropped-favicon-1-32x32.png" - } - } - - @MangaSourceParser("MANGALEK", "MangaLek", "ar") - class MangaLek(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGALEK, "mangalek.com") { - - override fun getFaviconUrl(): String { - return "https://${getDomain()}/wp-content/logo192.png" - } - } - - @MangaSourceParser("HARIMANGA", "HariManga", "en") - class HariManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.HARIMANGA, "harimanga.com") { - override val datePattern = "MM/dd/yyyy" - } - - @MangaSourceParser("KISSMANGA", "KissManga", "en") - class KissManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.KISSMANGA, "kissmanga.in") - - @MangaSourceParser("MANGAROCK", "MangaRock", "en") - class MangaRock(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGAROCK, "mangarockteam.com") - - @MangaSourceParser("FREEMANGA", "FreeManga", "en") - class FreeManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.FREEMANGA, "freemanga.me") - - @MangaSourceParser("MANGA_KOMI", "MangaKomi", "en") - class MangaKomi(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGA_KOMI, "mangakomi.io") - - @MangaSourceParser("NEATMANGA", "NeatManga", "en") - class NeatManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.NEATMANGA, "neatmangas.com") - - @MangaSourceParser("MANHWACLAN", "ManhwaClan", "en") - class ManhwaClan(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANHWACLAN, "manhwaclan.com") - - @MangaSourceParser("MANGA_3S", "Manga3s", "en") - class Manga3s(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGA_3S, "manga3s.com") { - override val tagPrefix = "manhwa-genre/" - } - - @MangaSourceParser("MANHWAKOOL", "Manhwa Kool", "en") - class ManhwaKool(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANHWAKOOL, "manhwakool.com") { - override val datePattern: String = "MM/dd" - override fun getFaviconUrl(): String { - return "https://${getDomain()}/wp-content/uploads/2021/10/cropped-logo-kool-32x32.jpeg" - } - } - - @MangaSourceParser("TOPMANHUA", "Top Manhua", "en") - class TopManhua(context: MangaLoaderContext) : MadaraParser(context, MangaSource.TOPMANHUA, "www.topmanhua.com") { - override val tagPrefix = "manhua-genre/" - override val datePattern = "MM/dd/yyyy" - } - - @MangaSourceParser("S2MANGA", "S2Manga", "en") - class S2Manga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.S2MANGA, "s2manga.com") - - @MangaSourceParser("SKY_MANGA", "Sky Manga", "en") - class SkyManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.SKY_MANGA, "skymanga.xyz") { + context: MangaLoaderContext, + source: MangaSource, + domain: String, +) : PagedMangaParser(context, source, pageSize = 12) { + + override val configKeyDomain = ConfigKey.Domain(domain, null) + + override val sortOrders: Set = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + ) + + protected open val tagPrefix = "manga-genre/" + protected open val isNsfwSource = false + protected open val datePattern = "MMMM dd, yyyy" + + init { + paginator.firstPage = 0 + searchPaginator.firstPage = 0 + } + + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + val tag = tags.oneOrThrowIfMany() + val payload = createRequestTemplate() + payload["page"] = page.toString() + payload["vars[meta_key]"] = when (sortOrder) { + SortOrder.POPULARITY -> "_wp_manga_views" + SortOrder.UPDATED -> "_latest_update" + else -> "_wp_manga_views" + } + payload["vars[wp-manga-genre]"] = tag?.key.orEmpty() + payload["vars[s]"] = query?.urlEncoded().orEmpty() + val doc = webClient.httpPost( + "https://$domain/wp-admin/admin-ajax.php", + payload, + ).parseHtml() + return doc.select("div.row.c-tabs-item__content").ifEmpty { + doc.select("div.page-item-detail.manga") + }.map { div -> + val href = div.selectFirst("a")?.attrAsRelativeUrlOrNull("href") + ?: div.parseFailed("Link not found") + val summary = div.selectFirst(".tab-summary") ?: div.selectFirst(".item-summary") + Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(div.host ?: domain), + coverUrl = div.selectFirst("img")?.src().orEmpty(), + title = (summary?.selectFirst("h3") ?: summary?.selectFirst("h4"))?.text().orEmpty(), + altTitle = null, + rating = div.selectFirst("span.total_votes")?.ownText() + ?.toFloatOrNull()?.div(5f) ?: -1f, + tags = summary?.selectFirst(".mg_genres")?.select("a")?.mapToSet { a -> + MangaTag( + key = a.attr("href").removeSuffix("/").substringAfterLast('/'), + title = a.text().toTitleCase(), + source = source, + ) + }.orEmpty(), + author = summary?.selectFirst(".mg_author")?.selectFirst("a")?.ownText(), + state = when ( + summary?.selectFirst(".mg_status")?.selectFirst(".summary-content") + ?.ownText()?.trim()?.lowercase() + ) { + "ongoing" -> MangaState.ONGOING + "completed" -> MangaState.FINISHED + else -> null + }, + source = source, + isNsfw = isNsfwSource, + ) + } + } + + override suspend fun getTags(): Set { + val doc = webClient.httpGet("https://$domain/manga/").parseHtml() + val body = doc.body() + val root1 = body.selectFirst("header")?.selectFirst("ul.second-menu") + val root2 = body.selectFirst("div.genres_wrap")?.selectFirst("ul.list-unstyled") + if (root1 == null && root2 == null) { + doc.parseFailed("Root not found") + } + val list = root1?.select("li").orEmpty() + root2?.select("li").orEmpty() + val keySet = HashSet(list.size) + return list.mapNotNullToSet { li -> + val a = li.selectFirst("a") ?: return@mapNotNullToSet null + val href = a.attr("href").removeSuffix("/") + .substringAfterLast(tagPrefix, "") + if (href.isEmpty() || !keySet.add(href)) { + return@mapNotNullToSet null + } + MangaTag( + key = href, + title = a.ownText().trim().ifEmpty { + a.selectFirst(".menu-image-title")?.text()?.trim() ?: return@mapNotNullToSet null + }.toTitleCase(), + source = source, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { + val fullUrl = manga.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + val chaptersDeferred = async { getChapters(manga, doc) } + val root = doc.body().selectFirst("div.profile-manga") + ?.selectFirst("div.summary_content") + ?.selectFirst("div.post-content") + ?: throw ParseException("Root not found", fullUrl) + val root2 = doc.body().selectFirst("div.content-area") + ?.selectFirst("div.c-page") + ?: throw ParseException("Root2 not found", fullUrl) + manga.copy( + tags = root.selectFirst("div.genres-content")?.select("a") + ?.mapNotNullToSet { a -> + MangaTag( + key = a.attr("href").removeSuffix("/").substringAfterLast('/'), + title = a.text().toTitleCase(), + source = source, + ) + } ?: manga.tags, + description = root2.selectFirst("div.description-summary") + ?.selectFirst("div.summary__content") + ?.select("p") + ?.filterNot { it.ownText().startsWith("A brief description") } + ?.joinToString { it.html() }, + chapters = chaptersDeferred.await(), + ) + } + + protected open suspend fun getChapters(manga: Manga, doc: Document): List { + val root2 = doc.body().selectFirstOrThrow("div.content-area") + .selectFirstOrThrow("div.c-page") + val dateFormat = SimpleDateFormat(datePattern, sourceLocale ?: Locale.US) + return root2.select("li").asReversed().mapChapters { i, li -> + val a = li.selectFirst("a") + val href = a?.attrAsRelativeUrlOrNull("href") ?: li.parseFailed("Link is missing") + MangaChapter( + id = generateUid(href), + name = a.ownText(), + number = i + 1, + url = href, + uploadDate = parseChapterDate( + dateFormat, + li.selectFirst("span.chapter-release-date i")?.text(), + ), + source = source, + scanlator = null, + branch = null, + ) + } + } + + override suspend fun getPages(chapter: MangaChapter): List { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + val root = doc.body().selectFirst("div.main-col-inner") + ?.selectFirst("div.reading-content") + ?: throw ParseException("Root not found", fullUrl) + return root.select("div.page-break").map { div -> + val img = div.selectFirst("img") ?: div.parseFailed("Page image not found") + val url = img.src()?.toRelativeUrl(domain) ?: div.parseFailed("Image src not found") + MangaPage( + id = generateUid(url), + url = url, + preview = null, + referer = fullUrl, + source = source, + ) + } + } + + protected fun parseChapterDate(dateFormat: DateFormat, date: String?): Long { + date ?: return 0 + return when { + date.endsWith(" ago", ignoreCase = true) -> { + parseRelativeDate(date) + } + // Handle translated 'ago' in Portuguese. + date.endsWith(" atrás", ignoreCase = true) -> { + parseRelativeDate(date) + } + // Handle translated 'ago' in Turkish. + date.endsWith(" önce", ignoreCase = true) -> { + parseRelativeDate(date) + } + // Handle 'yesterday' and 'today', using midnight + date.startsWith("year", ignoreCase = true) -> { + Calendar.getInstance().apply { + add(Calendar.DAY_OF_MONTH, -1) // yesterday + set(Calendar.HOUR_OF_DAY, 0) + set(Calendar.MINUTE, 0) + set(Calendar.SECOND, 0) + set(Calendar.MILLISECOND, 0) + }.timeInMillis + } + + date.startsWith("today", ignoreCase = true) -> { + Calendar.getInstance().apply { + set(Calendar.HOUR_OF_DAY, 0) + set(Calendar.MINUTE, 0) + set(Calendar.SECOND, 0) + set(Calendar.MILLISECOND, 0) + }.timeInMillis + } + + date.contains(Regex("""\d(st|nd|rd|th)""")) -> { + // Clean date (e.g. 5th December 2019 to 5 December 2019) before parsing it + date.split(" ").map { + if (it.contains(Regex("""\d\D\D"""))) { + it.replace(Regex("""\D"""), "") + } else { + it + } + } + .let { dateFormat.tryParse(it.joinToString(" ")) } + } + + else -> dateFormat.tryParse(date) + } + } + + // Parses dates in this form: + // 21 hours ago + private fun parseRelativeDate(date: String): Long { + val number = Regex("""(\d+)""").find(date)?.value?.toIntOrNull() ?: return 0 + val cal = Calendar.getInstance() + + return when { + WordSet( + "hari", + "gün", + "jour", + "día", + "dia", + "day", + ).anyWordIn(date) -> cal.apply { add(Calendar.DAY_OF_MONTH, -number) }.timeInMillis + + WordSet("jam", "saat", "heure", "hora", "hour").anyWordIn(date) -> cal.apply { + add( + Calendar.HOUR, + -number, + ) + }.timeInMillis + + WordSet("menit", "dakika", "min", "minute", "minuto").anyWordIn(date) -> cal.apply { + add( + Calendar.MINUTE, + -number, + ) + }.timeInMillis + + WordSet("detik", "segundo", "second").anyWordIn(date) -> cal.apply { + add( + Calendar.SECOND, + -number, + ) + }.timeInMillis + + WordSet("month").anyWordIn(date) -> cal.apply { add(Calendar.MONTH, -number) }.timeInMillis + WordSet("year").anyWordIn(date) -> cal.apply { add(Calendar.YEAR, -number) }.timeInMillis + else -> 0 + } + } + + private fun Element.src(): String? { + var result = absUrl("data-src") + if (result.isEmpty()) result = absUrl("data-cfsrc") + if (result.isEmpty()) result = absUrl("src") + return if (result.isEmpty()) null else result + } + + private fun createRequestTemplate() = + ( + "action=madara_load_more&page=1&template=madara-core%2Fcontent%2Fcontent-search&vars%5Bs%5D=&vars%5B" + + "orderby%5D=meta_value_num&vars%5Bpaged%5D=1&vars%5Btemplate%5D=search&vars%5Bmeta_query" + + "%5D%5B0%5D%5Brelation%5D=AND&vars%5Bmeta_query%5D%5Brelation%5D=OR&vars%5Bpost_type" + + "%5D=wp-manga&vars%5Bpost_status%5D=publish&vars%5Bmeta_key%5D=_latest_update&vars%5Border" + + "%5D=desc&vars%5Bmanga_archives_item_layout%5D=default" + ).split('&') + .map { + val pos = it.indexOf('=') + it.substring(0, pos) to it.substring(pos + 1) + }.toMutableMap() + + @MangaSourceParser("MANGAREAD", "MangaRead", "en") + class MangaRead(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGAREAD, "www.mangaread.org") { + override val tagPrefix = "genres/" + override val datePattern = "dd.MM.yyyy" + } + + @MangaSourceParser("MANGAWEEBS", "MangaWeebs", "en") + class MangaWeebs(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGAWEEBS, "mangaweebs.in") { + override val datePattern = "dd MMMM HH:mm" + } + + @MangaSourceParser("HachiManga", "HACHIMANGA", "ja") + class HachiManga(context: MangaLoaderContext) : + MadaraParser(context, MangaSource.HACHIMANGA, "hachimanga.com") { + + override val sourceLocale: Locale = Locale.ENGLISH + } + + @MangaSourceParser("PIANMANGA", "PianManga", "en") + class PianManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.PIANMANGA, "pianmanga.me") + + @MangaSourceParser("MANGAROSIE", "MangaRosie", "en") + class MangaRosie(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGAROSIE, "mangarosie.in") + + @MangaSourceParser("MANGATX", "MangaTx", "en") + class MangaTx(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGATX, "mangatx.com") + + @MangaSourceParser("MANGAEFFECT", "MangaEffect", "en") + class MangaEffect(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGAEFFECT, "mangaeffect.com") { + override val datePattern = "dd.MM.yyyy" + } + + @MangaSourceParser("AQUAMANGA", "AquaManga", "en") + class AquaManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.AQUAMANGA, "aquamanga.com") { + + override fun getFaviconUrl(): String { + return "https://$domain/wp-content/uploads/2021/03/cropped-cropped-favicon-1-32x32.png" + } + } + + @MangaSourceParser("MANGALEK", "MangaLek", "ar") + class MangaLek(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGALEK, "mangalek.com") { + + override fun getFaviconUrl(): String { + return "https://$domain/wp-content/logo192.png" + } + } + + @MangaSourceParser("HARIMANGA", "HariManga", "en") + class HariManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.HARIMANGA, "harimanga.com") { + override val datePattern = "MM/dd/yyyy" + } + + @MangaSourceParser("KISSMANGA", "KissManga", "en") + class KissManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.KISSMANGA, "kissmanga.in") + + @MangaSourceParser("MANGAROCK", "MangaRock", "en") + class MangaRock(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGAROCK, "mangarockteam.com") + + @MangaSourceParser("FREEMANGA", "FreeManga", "en") + class FreeManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.FREEMANGA, "freemanga.me") + + @MangaSourceParser("MANGA_KOMI", "MangaKomi", "en") + class MangaKomi(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGA_KOMI, "mangakomi.io") + + @MangaSourceParser("NEATMANGA", "NeatManga", "en") + class NeatManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.NEATMANGA, "neatmangas.com") + + @MangaSourceParser("MANHWACLAN", "ManhwaClan", "en") + class ManhwaClan(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANHWACLAN, "manhwaclan.com") + + @MangaSourceParser("MANGA_3S", "Manga3s", "en") + class Manga3s(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGA_3S, "manga3s.com") { + override val tagPrefix = "manhwa-genre/" + } + + @MangaSourceParser("MANHWAKOOL", "Manhwa Kool", "en") + class ManhwaKool(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANHWAKOOL, "manhwakool.com") { + override val datePattern: String = "MM/dd" + override fun getFaviconUrl(): String { + return "https://$domain/wp-content/uploads/2021/10/cropped-logo-kool-32x32.jpeg" + } + } + + @MangaSourceParser("TOPMANHUA", "Top Manhua", "en") + class TopManhua(context: MangaLoaderContext) : MadaraParser(context, MangaSource.TOPMANHUA, "www.topmanhua.com") { + override val tagPrefix = "manhua-genre/" + override val datePattern = "MM/dd/yyyy" + } + + @MangaSourceParser("S2MANGA", "S2Manga", "en") + class S2Manga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.S2MANGA, "s2manga.com") + + @MangaSourceParser("SKY_MANGA", "Sky Manga", "en") + class SkyManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.SKY_MANGA, "skymanga.xyz") { - override val isNsfwSource = true + override val isNsfwSource = true - override fun getFaviconUrl(): String { - return "https://${getDomain()}/wp-content/uploads/cropped-sky-tv-1-32x32.png" - } - } + override fun getFaviconUrl(): String { + return "https://$domain/wp-content/uploads/cropped-sky-tv-1-32x32.png" + } + } - @MangaSourceParser("BAKAMAN", "BakaMan", "th") - class BakaMan(context: MangaLoaderContext) : MadaraParser(context, MangaSource.BAKAMAN, "bakaman.net") { + @MangaSourceParser("BAKAMAN", "BakaMan", "th") + class BakaMan(context: MangaLoaderContext) : MadaraParser(context, MangaSource.BAKAMAN, "bakaman.net") { - override val isNsfwSource = true - } + override val isNsfwSource = true + } - @MangaSourceParser("MANGA_DISTRICT", "Manga District", "en") - class MangaDistrict(context: MangaLoaderContext) : - MadaraParser(context, MangaSource.MANGA_DISTRICT, "mangadistrict.com") { + @MangaSourceParser("MANGA_DISTRICT", "Manga District", "en") + class MangaDistrict(context: MangaLoaderContext) : + MadaraParser(context, MangaSource.MANGA_DISTRICT, "mangadistrict.com") { - override val tagPrefix = "publication-genre/" + override val tagPrefix = "publication-genre/" - override val isNsfwSource = true - } + override val isNsfwSource = true + } - @MangaSourceParser("HENTAI_4FREE", "Hentai4Free", "en") - class Hentai4Free(context: MangaLoaderContext) : - MadaraParser(context, MangaSource.HENTAI_4FREE, "hentai4free.net") { + @MangaSourceParser("HENTAI_4FREE", "Hentai4Free", "en") + class Hentai4Free(context: MangaLoaderContext) : + MadaraParser(context, MangaSource.HENTAI_4FREE, "hentai4free.net") { - override val tagPrefix = "hentai-tag/" + override val tagPrefix = "hentai-tag/" - override val isNsfwSource = true + override val isNsfwSource = true - override suspend fun getTags(): Set { - val doc = context.httpGet("https://${getDomain()}/").parseHtml() - val body = doc.body() - val root1 = body.selectFirst("header")?.selectFirst("ul.second-menu") - val list = root1?.select("li").orEmpty() - val keySet = HashSet(list.size) - return list.mapNotNullToSet { li -> - val a = li.selectFirst("a") ?: return@mapNotNullToSet null - val href = a.attr("href").removeSuffix("/") - .substringAfterLast(tagPrefix, "") - if (href.isEmpty() || !keySet.add(href)) { - return@mapNotNullToSet null - } - MangaTag( - key = href, - title = a.ownText().trim().toTitleCase(), - source = source, - ) - } - } - } + override suspend fun getTags(): Set { + val doc = webClient.httpGet("https://$domain/").parseHtml() + val body = doc.body() + val root1 = body.selectFirst("header")?.selectFirst("ul.second-menu") + val list = root1?.select("li").orEmpty() + val keySet = HashSet(list.size) + return list.mapNotNullToSet { li -> + val a = li.selectFirst("a") ?: return@mapNotNullToSet null + val href = a.attr("href").removeSuffix("/") + .substringAfterLast(tagPrefix, "") + if (href.isEmpty() || !keySet.add(href)) { + return@mapNotNullToSet null + } + MangaTag( + key = href, + title = a.ownText().trim().toTitleCase(), + source = source, + ) + } + } + } - @MangaSourceParser("ALLPORN_COMIC", "All Porn Comic", "en") - class AllPornComic(context: MangaLoaderContext) : - MadaraParser(context, MangaSource.ALLPORN_COMIC, "allporncomic.com") { + @MangaSourceParser("ALLPORN_COMIC", "All Porn Comic", "en") + class AllPornComic(context: MangaLoaderContext) : + MadaraParser(context, MangaSource.ALLPORN_COMIC, "allporncomic.com") { - override val tagPrefix = "porncomic-genre/" + override val tagPrefix = "porncomic-genre/" - override val isNsfwSource = true + override val isNsfwSource = true - override fun getFaviconUrl(): String { - return "https://cdn.${getDomain()}/wp-content/uploads/2019/01/cropped-cropped-pcround-32x32.png" - } - } + override fun getFaviconUrl(): String { + return "https://cdn.$domain/wp-content/uploads/2019/01/cropped-cropped-pcround-32x32.png" + } + } - @MangaSourceParser("CAT_300", "Cat300", "th") - class Cat300(context: MangaLoaderContext) : MadaraParser(context, MangaSource.CAT_300, "cat300.com") { + @MangaSourceParser("CAT_300", "Cat300", "th") + class Cat300(context: MangaLoaderContext) : MadaraParser(context, MangaSource.CAT_300, "cat300.com") { - override val isNsfwSource = true - } - - @MangaSourceParser("BIBIMANGA", "BibiManga", "en") - class BibiManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.BIBIMANGA, "bibimanga.com") { - - override val isNsfwSource = true - } - - @MangaSourceParser("TREE_MANGA", "Tree Manga", "en") - class TreeManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.TREE_MANGA, "treemanga.com") { + override val isNsfwSource = true + } + + @MangaSourceParser("BIBIMANGA", "BibiManga", "en") + class BibiManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.BIBIMANGA, "bibimanga.com") { + + override val isNsfwSource = true + } + + @MangaSourceParser("TREE_MANGA", "Tree Manga", "en") + class TreeManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.TREE_MANGA, "treemanga.com") { - override val datePattern = "MM/dd/yyyy" - - override fun getFaviconUrl(): String { - return "https://${getDomain()}/wp-content/uploads/2017/10/lgoo-treemanga-2-1.jpg" - } - } + override val datePattern = "MM/dd/yyyy" + + override fun getFaviconUrl(): String { + return "https://$domain/wp-content/uploads/2017/10/lgoo-treemanga-2-1.jpg" + } + } - @MangaSourceParser("MANGACV", "Manga Cv", "en") - class MangaCv(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGACV, "mangacv.com") { + @MangaSourceParser("MANGACV", "Manga Cv", "en") + class MangaCv(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGACV, "mangacv.com") { - override fun getFaviconUrl(): String { - return "https://${getDomain()}/wp-content/uploads/2020/10/cropped-mangaCV-32x32.png" - } - } + override fun getFaviconUrl(): String { + return "https://$domain/wp-content/uploads/2020/10/cropped-mangaCV-32x32.png" + } + } - @MangaSourceParser("MANGA_MANHUA", "Manga Manhua", "en") - class MangaManhua(context: MangaLoaderContext) : - MadaraParser(context, MangaSource.MANGA_MANHUA, "mangamanhua.online") + @MangaSourceParser("MANGA_MANHUA", "Manga Manhua", "en") + class MangaManhua(context: MangaLoaderContext) : + MadaraParser(context, MangaSource.MANGA_MANHUA, "mangamanhua.online") - @MangaSourceParser("MANGA_247", "247MANGA", "en") - class Manga247(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGA_247, "247manga.com") { - override val tagPrefix = "manhwa-genre/" - } + @MangaSourceParser("MANGA_247", "247MANGA", "en") + class Manga247(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGA_247, "247manga.com") { + override val tagPrefix = "manhwa-genre/" + } - @MangaSourceParser("MANGA_365", "365Manga", "en") - class Manga365(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGA_365, "365manga.com") + @MangaSourceParser("MANGA_365", "365Manga", "en") + class Manga365(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGA_365, "365manga.com") - @MangaSourceParser("MANGACLASH", "Mangaclash", "en") - class Mangaclash(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGACLASH, "mangaclash.com") { - override val datePattern = "MM/dd/yyyy" - } + @MangaSourceParser("MANGACLASH", "Mangaclash", "en") + class Mangaclash(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGACLASH, "mangaclash.com") { + override val datePattern = "MM/dd/yyyy" + } - @MangaSourceParser("ZINMANGA", "ZINMANGA", "en") - class Zinmanga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.ZINMANGA, "zinmanga.com") + @MangaSourceParser("ZINMANGA", "ZINMANGA", "en") + class Zinmanga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.ZINMANGA, "zinmanga.com") } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangalinkParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangalinkParser.kt index 6303c813c..ff78aab41 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangalinkParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangalinkParser.kt @@ -8,44 +8,41 @@ import org.koitharu.kotatsu.parsers.exception.ParseException import org.koitharu.kotatsu.parsers.model.Manga import org.koitharu.kotatsu.parsers.model.MangaSource import org.koitharu.kotatsu.parsers.model.MangaTag -import org.koitharu.kotatsu.parsers.util.mapNotNullToSet -import org.koitharu.kotatsu.parsers.util.parseHtml -import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl -import org.koitharu.kotatsu.parsers.util.toTitleCase +import org.koitharu.kotatsu.parsers.util.* @MangaSourceParser("MANGALINK_AR", "Mangalink", "ar") internal class MangalinkParser(context: MangaLoaderContext) : - MadaraParser(context, MangaSource.MANGALINK_AR, "mangalink.online") { + MadaraParser(context, MangaSource.MANGALINK_AR, "mangalink.online") { - override suspend fun getDetails(manga: Manga): Manga = coroutineScope { - val fullUrl = manga.url.toAbsoluteUrl(getDomain()) - val doc = context.httpGet(fullUrl).parseHtml() - val chaptersDeferred = async { getChapters(manga, doc) } - val root = doc.body().selectFirst("div.profile-manga") - ?.selectFirst("div.summary_content") - ?.selectFirst("div.post-content") - ?: throw ParseException("Root not found", fullUrl) - val root2 = doc.body().selectFirst("div.content-area") - ?.selectFirst("div.c-page") - ?: throw ParseException("Root2 not found", fullUrl) - manga.copy( - tags = root.selectFirst("div.genres-content")?.select("a") - ?.mapNotNullToSet { a -> - MangaTag( - key = a.attr("href").removeSuffix("/").substringAfterLast('/'), - title = a.text().toTitleCase(), - source = source, - ) - } ?: manga.tags, - description = root2.selectFirst("div.description-summary") - ?.selectFirst("div.summary__content") - ?.select("p") - ?.filterNot { it.ownText().startsWith("A brief description") } - ?.joinToString { it.html() }, - chapters = chaptersDeferred.await(), - ) - } + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { + val fullUrl = manga.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + val chaptersDeferred = async { getChapters(manga, doc) } + val root = doc.body().selectFirst("div.profile-manga") + ?.selectFirst("div.summary_content") + ?.selectFirst("div.post-content") + ?: throw ParseException("Root not found", fullUrl) + val root2 = doc.body().selectFirst("div.content-area") + ?.selectFirst("div.c-page") + ?: throw ParseException("Root2 not found", fullUrl) + manga.copy( + tags = root.selectFirst("div.genres-content")?.select("a") + ?.mapNotNullToSet { a -> + MangaTag( + key = a.attr("href").removeSuffix("/").substringAfterLast('/'), + title = a.text().toTitleCase(), + source = source, + ) + } ?: manga.tags, + description = root2.selectFirst("div.description-summary") + ?.selectFirst("div.summary__content") + ?.select("p") + ?.filterNot { it.ownText().startsWith("A brief description") } + ?.joinToString { it.html() }, + chapters = chaptersDeferred.await(), + ) + } - override fun getFaviconUrl(): String = - "https://cdn.${getDomain()}/wp-content/uploads/2020/05/cropped-mangalink-180x180.jpg" -} \ No newline at end of file + override fun getFaviconUrl(): String = + "https://cdn.${domain}/wp-content/uploads/2020/05/cropped-mangalink-180x180.jpg" +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangasOriginesParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangasOriginesParser.kt index 8b9b567d9..af1c3801a 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangasOriginesParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangasOriginesParser.kt @@ -8,49 +8,50 @@ import org.koitharu.kotatsu.parsers.model.MangaChapter import org.koitharu.kotatsu.parsers.model.MangaSource import org.koitharu.kotatsu.parsers.model.MangaState import org.koitharu.kotatsu.parsers.util.attrAsAbsoluteUrlOrNull +import org.koitharu.kotatsu.parsers.util.domain import org.koitharu.kotatsu.parsers.util.mapToSet import org.koitharu.kotatsu.parsers.util.selectFirstOrThrow import java.util.* @MangaSourceParser("MANGAS_ORIGINES", "Mangas Origines", "fr") internal class MangasOriginesParser(context: MangaLoaderContext) : - Madara6Parser(context, MangaSource.MANGAS_ORIGINES, "mangas-origines.fr") { + Madara6Parser(context, MangaSource.MANGAS_ORIGINES, "mangas-origines.fr") { - override val tagPrefix = "catalogues-genre/" + override val tagPrefix = "catalogues-genre/" - override fun getFaviconUrl(): String { - return "https://${getDomain()}/wp-content/uploads/2020/11/Mangas-150x150.png" - } + override fun getFaviconUrl(): String { + return "https://${domain}/wp-content/uploads/2020/11/Mangas-150x150.png" + } - override fun parseDetails(manga: Manga, body: Element, chapters: List): Manga { - val root = body.selectFirstOrThrow(".site-content") - val postContent = root.selectFirstOrThrow(".post-content") - val tags = postContent.getElementsContainingOwnText("Genre") - .firstOrNull()?.tableValue() - ?.getElementsByAttributeValueContaining("href", tagPrefix) - ?.mapToSet { a -> a.asMangaTag() } ?: manga.tags - return manga.copy( - largeCoverUrl = root.selectFirst("picture") - ?.selectFirst("img[data-src]") - ?.attrAsAbsoluteUrlOrNull("data-src"), - description = (root.selectFirst(".detail-content") - ?: root.selectFirstOrThrow(".manga-excerpt")).html(), - author = postContent.getElementsContainingOwnText("Auteur") - .firstOrNull()?.tableValue()?.text()?.trim(), - state = postContent.getElementsContainingOwnText("STATUS") - .firstOrNull()?.tableValue()?.text()?.asMangaState(), - tags = tags, - isNsfw = body.hasClass("adult-content"), - chapters = chapters, - ) - } + override fun parseDetails(manga: Manga, body: Element, chapters: List): Manga { + val root = body.selectFirstOrThrow(".site-content") + val postContent = root.selectFirstOrThrow(".post-content") + val tags = postContent.getElementsContainingOwnText("Genre") + .firstOrNull()?.tableValue() + ?.getElementsByAttributeValueContaining("href", tagPrefix) + ?.mapToSet { a -> a.asMangaTag() } ?: manga.tags + return manga.copy( + largeCoverUrl = root.selectFirst("picture") + ?.selectFirst("img[data-src]") + ?.attrAsAbsoluteUrlOrNull("data-src"), + description = (root.selectFirst(".detail-content") + ?: root.selectFirstOrThrow(".manga-excerpt")).html(), + author = postContent.getElementsContainingOwnText("Auteur") + .firstOrNull()?.tableValue()?.text()?.trim(), + state = postContent.getElementsContainingOwnText("STATUS") + .firstOrNull()?.tableValue()?.text()?.asMangaState(), + tags = tags, + isNsfw = body.hasClass("adult-content"), + chapters = chapters, + ) + } - override fun String.asMangaState() = when (trim().lowercase(Locale.FRANCE)) { - "en cours" -> MangaState.ONGOING - "abandonné", - "terminé", - -> MangaState.FINISHED + override fun String.asMangaState() = when (trim().lowercase(Locale.FRANCE)) { + "en cours" -> MangaState.ONGOING + "abandonné", + "terminé", + -> MangaState.FINISHED - else -> null - } -} \ No newline at end of file + else -> null + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/PrismaScansParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/PrismaScansParser.kt index 6a95d095a..b48770b08 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/PrismaScansParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/PrismaScansParser.kt @@ -8,51 +8,52 @@ import org.koitharu.kotatsu.parsers.model.MangaChapter import org.koitharu.kotatsu.parsers.model.MangaSource import org.koitharu.kotatsu.parsers.model.MangaState import org.koitharu.kotatsu.parsers.util.attrAsAbsoluteUrlOrNull +import org.koitharu.kotatsu.parsers.util.domain import org.koitharu.kotatsu.parsers.util.mapToSet import org.koitharu.kotatsu.parsers.util.selectFirstOrThrow import java.util.* @MangaSourceParser("PRISMA_SCANS", "Prisma Scans", "pt") internal class PrismaScansParser(context: MangaLoaderContext) : - Madara6Parser(context, MangaSource.PRISMA_SCANS, "prismascans.net") { + Madara6Parser(context, MangaSource.PRISMA_SCANS, "prismascans.net") { - override val tagPrefix = "manga-genre/" - override val datePattern = "MMM dd, yyyy" + override val tagPrefix = "manga-genre/" + override val datePattern = "MMM dd, yyyy" - override fun getFaviconUrl(): String { - return "https://${getDomain()}/wp-content/uploads/2022/07/cropped-branca-1-192x192.png" - } + override fun getFaviconUrl(): String { + return "https://${domain}/wp-content/uploads/2022/07/cropped-branca-1-192x192.png" + } - override fun parseDetails(manga: Manga, body: Element, chapters: List): Manga { - val root = body.selectFirstOrThrow(".site-content") - val postContent = root.selectFirstOrThrow(".post-content") - val tags = postContent.getElementsContainingOwnText("Gênero") - .firstOrNull()?.tableValue() - ?.getElementsByAttributeValueContaining("href", tagPrefix) - ?.mapToSet { a -> a.asMangaTag() } ?: manga.tags - return manga.copy( - largeCoverUrl = root.selectFirst("picture") - ?.selectFirst("img[data-src]") - ?.attrAsAbsoluteUrlOrNull("data-src"), - description = root.selectFirstOrThrow(".manga-excerpt").firstElementChild()?.html(), - author = postContent.getElementsContainingOwnText("Artista") - .firstOrNull()?.tableValue()?.text()?.trim(), - altTitle = postContent.getElementsContainingOwnText("Título Alternativo") - .firstOrNull()?.tableValue()?.text()?.trim(), - state = postContent.getElementsContainingOwnText("Status") - .firstOrNull()?.tableValue()?.text()?.asMangaState(), - tags = tags, - isNsfw = body.hasClass("adult-content"), - chapters = chapters, - ) - } + override fun parseDetails(manga: Manga, body: Element, chapters: List): Manga { + val root = body.selectFirstOrThrow(".site-content") + val postContent = root.selectFirstOrThrow(".post-content") + val tags = postContent.getElementsContainingOwnText("Gênero") + .firstOrNull()?.tableValue() + ?.getElementsByAttributeValueContaining("href", tagPrefix) + ?.mapToSet { a -> a.asMangaTag() } ?: manga.tags + return manga.copy( + largeCoverUrl = root.selectFirst("picture") + ?.selectFirst("img[data-src]") + ?.attrAsAbsoluteUrlOrNull("data-src"), + description = root.selectFirstOrThrow(".manga-excerpt").firstElementChild()?.html(), + author = postContent.getElementsContainingOwnText("Artista") + .firstOrNull()?.tableValue()?.text()?.trim(), + altTitle = postContent.getElementsContainingOwnText("Título Alternativo") + .firstOrNull()?.tableValue()?.text()?.trim(), + state = postContent.getElementsContainingOwnText("Status") + .firstOrNull()?.tableValue()?.text()?.asMangaState(), + tags = tags, + isNsfw = body.hasClass("adult-content"), + chapters = chapters, + ) + } - override fun String.asMangaState() = when (trim().lowercase(sourceLocale ?: Locale.ROOT)) { - "em lançamento" -> MangaState.ONGOING - "completo", - "cancelado", - -> MangaState.FINISHED + override fun String.asMangaState() = when (trim().lowercase(sourceLocale ?: Locale.ROOT)) { + "em lançamento" -> MangaState.ONGOING + "completo", + "cancelado", + -> MangaState.FINISHED - else -> null - } -} \ No newline at end of file + else -> null + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/ReaperScansParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/ReaperScansParser.kt index 7e3a20021..b161995f5 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/ReaperScansParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/ReaperScansParser.kt @@ -9,80 +9,80 @@ import java.util.* @MangaSourceParser("REAPER_SCANS_ID", "ReaperScansID", "in") internal class ReaperScansParser(context: MangaLoaderContext) : - Madara6Parser(context, MangaSource.REAPER_SCANS_ID, "reaperscans.id") { + Madara6Parser(context, MangaSource.REAPER_SCANS_ID, "reaperscans.id") { - override val datePattern = "MMMM dd, yyyy" - override val tagPrefix = "genre/" - override val sourceLocale: Locale = Locale.ENGLISH + override val datePattern = "MMMM dd, yyyy" + override val tagPrefix = "genre/" + override val sourceLocale: Locale = Locale.ENGLISH - override fun String.asMangaState(): MangaState? = when (this) { - "OnGoing", - "Upcoming", - -> MangaState.ONGOING + override fun String.asMangaState(): MangaState? = when (this) { + "OnGoing", + "Upcoming", + -> MangaState.ONGOING - "Completed", - "Dropped", - -> MangaState.FINISHED + "Completed", + "Dropped", + -> MangaState.FINISHED - else -> null - } + else -> null + } - override fun parseDetails(manga: Manga, body: Element, chapters: List): Manga { - val root = body.selectFirstOrThrow(".site-content") - val postContent = root.requireElementById("nav-info") - val tags = postContent.getElementsContainingOwnText("Gênero") - .firstOrNull()?.tableValue() - ?.getElementsByAttributeValueContaining("href", tagPrefix) - ?.mapToSet { a -> a.asMangaTag() } ?: manga.tags - return manga.copy( - rating = postContent.selectFirstOrThrow(".post-rating") - .selectFirstOrThrow(".total_votes").text().toFloat() / 5f, - largeCoverUrl = root.selectFirst(".summary_image") - ?.selectFirst("img[data-src]") - ?.attrAsAbsoluteUrlOrNull("data-src") - .assertNotNull("largeCoverUrl"), - description = root.requireElementById("nav-profile") - .selectFirstOrThrow(".description-summary") - .firstElementChild()?.html(), - author = postContent.getElementsContainingOwnText("Author(s)") - .firstOrNull()?.tableValue()?.text()?.trim(), - altTitle = postContent.getElementsContainingOwnText("Alternative") - .firstOrNull()?.tableValue()?.text()?.trim(), - state = postContent.getElementsContainingOwnText("Status") - .firstOrNull()?.tableValue()?.text()?.asMangaState(), - tags = tags, - isNsfw = body.hasClass("adult-content"), - chapters = chapters, - ) - } + override fun parseDetails(manga: Manga, body: Element, chapters: List): Manga { + val root = body.selectFirstOrThrow(".site-content") + val postContent = root.requireElementById("nav-info") + val tags = postContent.getElementsContainingOwnText("Gênero") + .firstOrNull()?.tableValue() + ?.getElementsByAttributeValueContaining("href", tagPrefix) + ?.mapToSet { a -> a.asMangaTag() } ?: manga.tags + return manga.copy( + rating = postContent.selectFirstOrThrow(".post-rating") + .selectFirstOrThrow(".total_votes").text().toFloat() / 5f, + largeCoverUrl = root.selectFirst(".summary_image") + ?.selectFirst("img[data-src]") + ?.attrAsAbsoluteUrlOrNull("data-src") + .assertNotNull("largeCoverUrl"), + description = root.requireElementById("nav-profile") + .selectFirstOrThrow(".description-summary") + .firstElementChild()?.html(), + author = postContent.getElementsContainingOwnText("Author(s)") + .firstOrNull()?.tableValue()?.text()?.trim(), + altTitle = postContent.getElementsContainingOwnText("Alternative") + .firstOrNull()?.tableValue()?.text()?.trim(), + state = postContent.getElementsContainingOwnText("Status") + .firstOrNull()?.tableValue()?.text()?.asMangaState(), + tags = tags, + isNsfw = body.hasClass("adult-content"), + chapters = chapters, + ) + } - override suspend fun getTags(): Set { - val doc = context.httpGet("https://${getDomain()}/semua-komik/").parseHtml() - val body = doc.body() - val root1 = body.selectFirst("header")?.selectFirst("ul.second-menu") - val root2 = body.selectFirst("div.genres_wrap")?.selectFirst("ul.list-unstyled") - if (root1 == null && root2 == null) { - doc.parseFailed("Root not found") - } - val list = root1?.select("li").orEmpty() + root2?.select("li").orEmpty() - val keySet = HashSet(list.size) - return list.mapNotNullToSet { li -> - val a = li.selectFirst("a") ?: return@mapNotNullToSet null - val href = a.attr("href").removeSuffix("/") - .substringAfterLast(tagPrefix, "") - if (href.isEmpty() || !keySet.add(href)) { - return@mapNotNullToSet null - } - MangaTag( - key = href, - title = a.ownText().trim().ifEmpty { - a.selectFirst(".menu-image-title")?.text()?.trim() ?: return@mapNotNullToSet null - }.toTitleCase(), - source = source, - ) - } - } + override suspend fun getTags(): Set { + val doc = webClient.httpGet("https://${domain}/semua-komik/").parseHtml() + val body = doc.body() + val root1 = body.selectFirst("header")?.selectFirst("ul.second-menu") + val root2 = body.selectFirst("div.genres_wrap")?.selectFirst("ul.list-unstyled") + if (root1 == null && root2 == null) { + doc.parseFailed("Root not found") + } + val list = root1?.select("li").orEmpty() + root2?.select("li").orEmpty() + val keySet = HashSet(list.size) + return list.mapNotNullToSet { li -> + val a = li.selectFirst("a") ?: return@mapNotNullToSet null + val href = a.attr("href").removeSuffix("/") + .substringAfterLast(tagPrefix, "") + if (href.isEmpty() || !keySet.add(href)) { + return@mapNotNullToSet null + } + MangaTag( + key = href, + title = a.ownText().trim().ifEmpty { + a.selectFirst(".menu-image-title")?.text()?.trim() ?: return@mapNotNullToSet null + }.toTitleCase(), + source = source, + ) + } + } - override fun getFaviconUrl(): String = - "https://wuz.${getDomain()}/wp-content/uploads/2022/03/20063242/cropped-ms-icon-310x310-2-192x192.png" -} \ No newline at end of file + override fun getFaviconUrl(): String = + "https://wuz.${domain}/wp-content/uploads/2022/03/20063242/cropped-ms-icon-310x310-2-192x192.png" +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/TatakaeScansParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/TatakaeScansParser.kt index d93664599..4a9a65cb8 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/TatakaeScansParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/TatakaeScansParser.kt @@ -3,49 +3,53 @@ package org.koitharu.kotatsu.parsers.site.madara import org.jsoup.nodes.Element import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.* +import org.koitharu.kotatsu.parsers.model.Manga +import org.koitharu.kotatsu.parsers.model.MangaChapter +import org.koitharu.kotatsu.parsers.model.MangaSource +import org.koitharu.kotatsu.parsers.model.MangaState import org.koitharu.kotatsu.parsers.util.attrAsAbsoluteUrlOrNull +import org.koitharu.kotatsu.parsers.util.domain import org.koitharu.kotatsu.parsers.util.mapToSet import org.koitharu.kotatsu.parsers.util.selectFirstOrThrow import java.util.* @MangaSourceParser("TATAKAE_SCANS", "Tatakae Scans", "pt") internal class TatakaeScansParser(context: MangaLoaderContext) : - Madara6Parser(context, MangaSource.TATAKAE_SCANS, "tatakaescan.com") { - - override val datePattern: String = "dd 'de' MMMMM 'de' yyyy" - - override fun parseDetails(manga: Manga, body: Element, chapters: List): Manga { - val root = body.selectFirstOrThrow(".site-content") - val postContent = root.selectFirstOrThrow(".post-content") - val tags = postContent.getElementsContainingOwnText("Gênero") - .firstOrNull()?.tableValue() - ?.getElementsByAttributeValueContaining("href", tagPrefix) - ?.mapToSet { a -> a.asMangaTag() } ?: manga.tags - return manga.copy( - largeCoverUrl = root.selectFirst("picture") - ?.selectFirst("img[data-src]") - ?.attrAsAbsoluteUrlOrNull("data-src"), - description = (root.selectFirst(".detail-content") - ?: root.selectFirstOrThrow(".manga-excerpt")).html(), - author = postContent.getElementsContainingOwnText("Autor") - .firstOrNull()?.tableValue()?.text()?.trim(), - state = postContent.getElementsContainingOwnText("Status") - .firstOrNull()?.tableValue()?.text()?.asMangaState(), - tags = tags, - isNsfw = body.hasClass("adult-content"), - chapters = chapters, - ) - } - - override fun String.asMangaState() = when (trim().lowercase(Locale.ROOT)) { - "em lançamento" -> MangaState.ONGOING - - else -> null - } - - override fun getFaviconUrl(): String { - return "https://${getDomain()}/wp-content/uploads/2022/07/cropped-favicon-180x180.png" - } - -} \ No newline at end of file + Madara6Parser(context, MangaSource.TATAKAE_SCANS, "tatakaescan.com") { + + override val datePattern: String = "dd 'de' MMMMM 'de' yyyy" + + override fun parseDetails(manga: Manga, body: Element, chapters: List): Manga { + val root = body.selectFirstOrThrow(".site-content") + val postContent = root.selectFirstOrThrow(".post-content") + val tags = postContent.getElementsContainingOwnText("Gênero") + .firstOrNull()?.tableValue() + ?.getElementsByAttributeValueContaining("href", tagPrefix) + ?.mapToSet { a -> a.asMangaTag() } ?: manga.tags + return manga.copy( + largeCoverUrl = root.selectFirst("picture") + ?.selectFirst("img[data-src]") + ?.attrAsAbsoluteUrlOrNull("data-src"), + description = (root.selectFirst(".detail-content") + ?: root.selectFirstOrThrow(".manga-excerpt")).html(), + author = postContent.getElementsContainingOwnText("Autor") + .firstOrNull()?.tableValue()?.text()?.trim(), + state = postContent.getElementsContainingOwnText("Status") + .firstOrNull()?.tableValue()?.text()?.asMangaState(), + tags = tags, + isNsfw = body.hasClass("adult-content"), + chapters = chapters, + ) + } + + override fun String.asMangaState() = when (trim().lowercase(Locale.ROOT)) { + "em lançamento" -> MangaState.ONGOING + + else -> null + } + + override fun getFaviconUrl(): String { + return "https://${domain}/wp-content/uploads/2022/07/cropped-favicon-180x180.png" + } + +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt index 4892f5070..8482aa434 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt @@ -16,509 +16,529 @@ import java.text.SimpleDateFormat import java.util.* internal abstract class MangaReaderParser( - source: MangaSource, - pageSize: Int, - searchPageSize: Int -) : PagedMangaParser(source, pageSize, searchPageSize) { - - override val sortOrders: Set - get() = EnumSet.of(SortOrder.UPDATED, SortOrder.POPULARITY, SortOrder.ALPHABETICAL, SortOrder.NEWEST) - - protected val idLocale - get() = Locale("in", "ID") - - abstract val listUrl: String - abstract val tableMode: Boolean - open val chapterDateFormat = SimpleDateFormat("MMM d, yyyy", idLocale) - - private var tagCache: ArrayMap? = null - private val mutex = Mutex() - private var lastSearchPage = 1 - - override suspend fun getDetails(manga: Manga): Manga { - val docs = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml() - val chapters = docs.select("#chapterlist > ul > li").reversed().mapChapters { index, element -> - val url = element.selectFirst("a")?.attrAsRelativeUrl("href") ?: return@mapChapters null - MangaChapter( - id = generateUid(url), - name = element.selectFirst(".chapternum")?.text() ?: "Chapter ${index + 1}", - url = url, - number = index + 1, - scanlator = null, - uploadDate = chapterDateFormat.tryParse(element.selectFirst(".chapterdate")?.text()), - branch = null, - source = source - ) - } - return if (tableMode) parseInfoTable(docs, manga, chapters) else parseInfoList(docs, manga, chapters) - } - - open suspend fun parseInfoTable(docs: Document, manga: Manga, chapters: List): Manga { - val mangaInfo = docs.selectFirst("div.seriestucontent > div.seriestucontentr") - val mangaState = mangaInfo?.selectFirst(".infotable td:contains(Status)")?.lastElementSibling()?.let { - when (it.text()) { - "Ongoing" -> MangaState.ONGOING - "Completed" -> MangaState.FINISHED - else -> null - } - } - val tagMap = getOrCreateTagMap() - val tags = mangaInfo?.select(".seriestugenre > a")?.mapNotNullToSet { tagMap[it.text()] } - - return manga.copy( - description = mangaInfo?.selectFirst("div.entry-content")?.html(), - state = mangaState, - author = mangaInfo?.selectFirst(".infotable td:contains(Author)")?.lastElementSibling()?.text(), - isNsfw = docs.selectFirst(".restrictcontainer") != null, - tags = tags.orEmpty(), - chapters = chapters, - ) - } - - open suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { - val mangaState = docs.selectFirst(".info-left .tsinfo div:contains(Status)")?.lastElementChild()?.let { - when (it.text()) { - "Ongoing" -> MangaState.ONGOING - "Completed" -> MangaState.FINISHED - else -> null - } - } - val tagMap = getOrCreateTagMap() - val tags = docs.select(".info-right .mgen > a").mapNotNullToSet { tagMap[it.text()] } - - return manga.copy( - description = docs.selectFirst(".info-right div.entry-content > p")?.html(), - state = mangaState, - author = docs.selectFirst(".info-left .tsinfo div:contains(Author)")?.lastElementChild()?.text(), - isNsfw = docs.selectFirst(".info-right .alr") != null, - tags = tags, - chapters = chapters, - ) - } - - override suspend fun getListPage( - page: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder - ): List { - if (!query.isNullOrEmpty()) { - if (page > lastSearchPage) { - return emptyList() - } - - val url = buildString { - append("https://") - append(getDomain()) - append("/page/") - append(page) - append("/?s=") - append(query.urlEncoded()) - } - - val docs = context.httpGet(url).parseHtml() - lastSearchPage = docs.selectFirst(".pagination .next") - ?.previousElementSibling() - ?.text()?.toIntOrNull() ?: 1 - return parseMangaList(docs) - } - - val sortQuery = when (sortOrder) { - SortOrder.ALPHABETICAL -> "title" - SortOrder.NEWEST -> "latest" - SortOrder.POPULARITY -> "popular" - SortOrder.UPDATED -> "update" - else -> "" - } - val tagKey = "genre[]".urlEncoded() - val tagQuery = if (tags.isNullOrEmpty()) "" else tags.joinToString(separator = "&", prefix = "&") { "$tagKey=${it.key}" } - val url = buildString { - append("https://") - append(getDomain()) - append(listUrl) - append("/?order=") - append(sortQuery) - append(tagQuery) - append("&page=") - append(page) - } - - return parseMangaList(context.httpGet(url).parseHtml()) - } - - private fun parseMangaList(docs: Document): List { - return docs.select(".postbody .listupd .bs .bsx").mapNotNull { - val a = it.selectFirst("a") ?: return@mapNotNull null - val relativeUrl = a.attrAsRelativeUrl("href") - val rating = it.selectFirst(".numscore")?.text() - ?.toFloatOrNull()?.div(10) ?: RATING_UNKNOWN - - Manga( - id = generateUid(relativeUrl), - url = relativeUrl, - title = a.attr("title"), - altTitle = null, - publicUrl = a.attrAsAbsoluteUrl("href"), - rating = rating, - isNsfw = false, - coverUrl = it.selectFirst("img.ts-post-image")?.imageUrl().orEmpty(), - tags = emptySet(), - state = null, - author = null, - source = source - ) - } - } - - override suspend fun getPages(chapter: MangaChapter): List { - val chapterUrl = chapter.url.toAbsoluteUrl(getDomain()) - val docs = context.httpGet(chapterUrl).parseHtml() - val script = docs.selectFirstOrThrow("script:containsData(ts_reader)") - val images = JSONObject(script.data().substringAfter('(').substringBeforeLast(')')) - .getJSONArray("sources") - .getJSONObject(0) - .getJSONArray("images") - - val pages = ArrayList(images.length()) - for (i in 0 until images.length()) { - pages.add( - MangaPage( - id = generateUid(images.getString(i)), - url = images.getString(i), - referer = chapterUrl, - preview = null, - source = source - ) - ) - } - - return pages - } - - override suspend fun getTags(): Set { - return getOrCreateTagMap().values.toSet() - } - - protected suspend fun getOrCreateTagMap(): Map = mutex.withLock { - tagCache?.let { return@withLock it } - val tagMap = ArrayMap() - - val url = listUrl.toAbsoluteUrl(getDomain()) - val tagElements = context.httpGet(url).parseHtml().select("ul.genrez > li") - for (el in tagElements) { - if (el.text().isEmpty()) continue - - tagMap[el.text()] = MangaTag( - title = el.text(), - key = el.selectFirst("input")?.attr("value") ?: continue, - source = source - ) - } - - tagCache = tagMap - return@withLock tagMap - } - - private fun Element.imageUrl(): String { - return attrAsAbsoluteUrlOrNull("src") - ?: attrAsAbsoluteUrlOrNull("data-cfsrc") - ?: "" - } - - @MangaSourceParser("MANHWALAND", "Manhwaland", "id") - class ManhwaLandParser(override val context: MangaLoaderContext) : MangaReaderParser(MangaSource.MANHWALAND, pageSize = 20, searchPageSize = 10) { - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("manhwaland.guru", null) - - override val listUrl: String - get() = "/series" - override val tableMode: Boolean - get() = false - } - - @MangaSourceParser("SEKAIKOMIK", "Sekaikomik", "id") - class SekaikomikParser(override val context: MangaLoaderContext) : MangaReaderParser(MangaSource.SEKAIKOMIK, pageSize = 20, searchPageSize = 100) { - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("sekaikomik.pro", null) - - override val listUrl: String - get() = "/manga" - override val tableMode: Boolean - get() = false - override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMMM D, yyyy", idLocale) - } - - @MangaSourceParser("MANHWAINDO", "Manhwaindo", "id") - class ManhwaIndoParser(override val context: MangaLoaderContext) : MangaReaderParser(MangaSource.MANHWAINDO, pageSize = 20, searchPageSize = 10) { - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("manhwaindo.id", null) - - override val listUrl: String get() = "/series" - override val tableMode: Boolean get() = false - } - - @MangaSourceParser("MANHWALIST", "Manhwalist", "id") - class ManhwalistParser(override val context: MangaLoaderContext) : MangaReaderParser(MangaSource.MANHWALIST, pageSize = 24, searchPageSize = 10) { - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("manhwalist.in", null) - - override val listUrl: String = "/manga" - override val tableMode: Boolean get() = false - override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMMM d, yyyy", Locale.ENGLISH) - } - - @MangaSourceParser("KIRYUU", "Kiryuu", "id") - class KiryuuParser(override val context: MangaLoaderContext) : MangaReaderParser(MangaSource.KIRYUU, pageSize = 20, searchPageSize = 10) { - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("kiryuu.id", null) - - override val listUrl: String - get() = "/manga" - override val tableMode: Boolean - get() = true - override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMMM d, yyyy", idLocale) - } - - @MangaSourceParser("TURKTOON", "Turktoon", "tr") - class TurktoonParser(override val context: MangaLoaderContext) : MangaReaderParser(MangaSource.TURKTOON, pageSize = 20, searchPageSize = 10) { - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("turktoon.com", null) - - override val listUrl: String - get() = "/manga" - override val tableMode: Boolean - get() = false - override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMMM d, yyyy", Locale("tr", "TR")) - - override suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { - val mangaState = docs.selectFirst(".info-left .tsinfo div:contains(Durum)")?.lastElementChild()?.let { - when (it.text()) { - "Devam Ediyor" -> MangaState.ONGOING - "Tamamlandı" -> MangaState.FINISHED - else -> null - } - } - - return super.parseInfoList(docs, manga, chapters).copy(state = mangaState) - } - } - - @MangaSourceParser("WESTMANGA", "Westmanga", "id") - class WestmangaParser(override val context: MangaLoaderContext) : MangaReaderParser(MangaSource.WESTMANGA, pageSize = 20, searchPageSize = 10) { - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("westmanga.info", null) - - override val listUrl: String - get() = "/manga" - override val tableMode: Boolean - get() = true - override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMMM d, yyyy", Locale.ENGLISH) - } - - @MangaSourceParser("TEMPESTFANSUB", "Tempestfansub", "tr") - class TempestfansubParser(override val context: MangaLoaderContext) : MangaReaderParser(MangaSource.TEMPESTFANSUB, pageSize = 40, searchPageSize = 40) { - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("manga.tempestfansub.com", null) - - override val listUrl: String get() = "/manga" - override val tableMode: Boolean get() = true - override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMMM d, yyyy", Locale("tr", "TR")) - - override suspend fun parseInfoTable(docs: Document, manga: Manga, chapters: List): Manga { - val infoElement = docs.selectFirst("div.infox") - return manga.copy( - chapters = chapters, - description = infoElement?.selectFirst("div.entry-content")?.html(), - author = infoElement?.selectFirst(".flex-wrap div:contains(Artist)")?.lastElementSibling()?.text(), - tags = infoElement?.select(".wd-full .mgen > a") - ?.mapNotNullToSet { getOrCreateTagMap()[it.text()] } - .orEmpty(), - isNsfw = docs.selectFirst(".postbody .alr") != null, - ) - } - } - - @MangaSourceParser("MANHWADESU", "ManhwaDesu", "id") - class ManhwadesuParser(override val context: MangaLoaderContext) : MangaReaderParser(MangaSource.MANHWADESU, pageSize = 20, searchPageSize = 10) { - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("manhwadesu.org", null) - - override val listUrl: String get() = "/komik" - override val tableMode: Boolean get() = false - } - - @MangaSourceParser("MANGATALE", "MangaTale", "id") - class MangaTaleParser(override val context: MangaLoaderContext) : MangaReaderParser(MangaSource.MANGATALE, pageSize = 20, searchPageSize = 10) { - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("mangatale.co", null) - - override val listUrl: String get() = "/manga" - override val tableMode: Boolean get() = false - - override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMMM d, yyyy", Locale.ENGLISH) - - override suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { - val infoElement = docs.selectFirst("div.infox") - return manga.copy( - chapters = chapters, - description = infoElement?.selectFirst("div.entry-content")?.html(), - author = infoElement?.selectFirst(".flex-wrap div:contains(Author)")?.lastElementSibling()?.text(), - tags = infoElement?.select(".wd-full .mgen > a") - ?.mapNotNullToSet { getOrCreateTagMap()[it.text()] } - .orEmpty(), - isNsfw = docs.selectFirst(".postbody .alr") != null, - ) - } - } - - @MangaSourceParser("DRAGONTRANSLATION", "DragonTranslation", "es") - class DragonTranslationParser(override val context: MangaLoaderContext) : MangaReaderParser(MangaSource.DRAGONTRANSLATION, pageSize = 20, searchPageSize = 10) { - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("dragontranslation.com", null) - - override val listUrl: String get() = "/manga" - override val tableMode: Boolean get() = false - - override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMMM d, yyyy", Locale.ENGLISH) - - override suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { - val infoElement = docs.selectFirst("div.infox") - return manga.copy( - chapters = chapters, - description = infoElement?.selectFirst("div.entry-content")?.html(), - author = infoElement?.selectFirst(".flex-wrap div:contains(Author)")?.lastElementSibling()?.text(), - tags = infoElement?.select(".wd-full .mgen > a") - ?.mapNotNullToSet { getOrCreateTagMap()[it.text()] } - .orEmpty(), - isNsfw = docs.selectFirst(".postbody .alr") != null, - ) - } - } - - @MangaSourceParser("ASURATR", "Asura Scans (tr)", "tr") - class AsuraTRParser(override val context: MangaLoaderContext) : MangaReaderParser(MangaSource.ASURATR, pageSize = 30, searchPageSize = 10) { - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("asurascanstr.com", null) - - override val listUrl: String get() = "/manga" - override val tableMode: Boolean get() = false - - override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", Locale("tr")) - - override suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { - val infoElement = docs.selectFirst("div.infox") - return manga.copy( - chapters = chapters, - description = infoElement?.selectFirst("div.entry-content")?.html(), - author = infoElement?.selectFirst(".flex-wrap div:contains(Yazar)")?.lastElementSibling()?.text(), - tags = infoElement?.select(".wd-full .mgen > a") - ?.mapNotNullToSet { getOrCreateTagMap()[it.text()] } - .orEmpty(), - isNsfw = docs.selectFirst(".postbody .alr") != null, - ) - } - } - - @MangaSourceParser("KOMIKTAP", "KomikTap", "id") - class KomikTapParser(override val context: MangaLoaderContext) : MangaReaderParser(MangaSource.KOMIKTAP, pageSize = 15, searchPageSize = 10) { - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("92.87.6.124", null) - - override val listUrl: String - get() = "/manga" - override val tableMode: Boolean - get() = true - - override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", idLocale) - } - - @MangaSourceParser("ASURASCANS", "Asura Scans", "en") - class AsuraScansParser(override val context: MangaLoaderContext) : MangaReaderParser(MangaSource.ASURASCANS, pageSize = 20, searchPageSize = 10) { - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("asurascans.com", null) - - override val listUrl: String - get() = "/manga" - override val tableMode: Boolean - get() = false - - override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH) - - override suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { - val infoElement = docs.selectFirst("div.infox") - return manga.copy( - chapters = chapters, - description = infoElement?.selectFirst("div.entry-content")?.html(), - author = infoElement?.selectFirst(".flex-wrap div:contains(Author)")?.lastElementSibling()?.text(), - tags = infoElement?.select(".wd-full .mgen > a") - ?.mapNotNullToSet { getOrCreateTagMap()[it.text()] } - .orEmpty(), - isNsfw = docs.selectFirst(".postbody .alr") != null, - ) - } - } - - @MangaSourceParser("COSMICSCANS", "CosmicScans", "en") - class CosmicScansParser(override val context: MangaLoaderContext) : MangaReaderParser(MangaSource.COSMICSCANS, pageSize = 20, searchPageSize = 10) { - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("cosmicscans.com", null) - - override val listUrl: String - get() = "/manga" - override val tableMode: Boolean - get() = false - - override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH) - - override suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { - val infoElement = docs.selectFirst("div.infox") - return manga.copy( - chapters = chapters, - description = infoElement?.selectFirst("div.entry-content")?.html(), - author = infoElement?.selectFirst(".flex-wrap div:contains(Author)")?.lastElementSibling()?.text(), - tags = infoElement?.select(".wd-full .mgen > a") - ?.mapNotNullToSet { getOrCreateTagMap()[it.text()] } - .orEmpty(), - ) - } - } - - @MangaSourceParser("MANGASUSU", "Mangasusu", "id") - class MangasusuParser(override val context: MangaLoaderContext) : MangaReaderParser(MangaSource.MANGASUSU, pageSize = 20, searchPageSize = 10) { - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("92.84.132.251", null) - - override val listUrl: String - get() = "/project" - override val tableMode: Boolean - get() = true - - override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH) - } - - @MangaSourceParser("KOMIKINDO", "KomikIndo", "id") - class KomikIndoParser(override val context: MangaLoaderContext) : MangaReaderParser(MangaSource.KOMIKINDO, pageSize = 20, searchPageSize = 10) { - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("komikindo.co", null) - - override val listUrl: String - get() = "/project" - override val tableMode: Boolean - get() = true - - override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", idLocale) - } - - @MangaSourceParser("KOMIKMANGA", "KomikManga", "id") - class KomikMangaParser(override val context: MangaLoaderContext) : MangaReaderParser(MangaSource.KOMIKMANGA, pageSize = 20, searchPageSize = 10) { - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("komikmanga.xyz", null) - - override val listUrl: String - get() = "/project" - override val tableMode: Boolean - get() = true - - override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH) - } + context: MangaLoaderContext, + source: MangaSource, + pageSize: Int, + searchPageSize: Int +) : PagedMangaParser(context, source, pageSize, searchPageSize) { + + override val sortOrders: Set + get() = EnumSet.of(SortOrder.UPDATED, SortOrder.POPULARITY, SortOrder.ALPHABETICAL, SortOrder.NEWEST) + + protected val idLocale + get() = Locale("in", "ID") + + abstract val listUrl: String + abstract val tableMode: Boolean + open val chapterDateFormat = SimpleDateFormat("MMM d, yyyy", idLocale) + + private var tagCache: ArrayMap? = null + private val mutex = Mutex() + private var lastSearchPage = 1 + + override suspend fun getDetails(manga: Manga): Manga { + val docs = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val chapters = docs.select("#chapterlist > ul > li").reversed().mapChapters { index, element -> + val url = element.selectFirst("a")?.attrAsRelativeUrl("href") ?: return@mapChapters null + MangaChapter( + id = generateUid(url), + name = element.selectFirst(".chapternum")?.text() ?: "Chapter ${index + 1}", + url = url, + number = index + 1, + scanlator = null, + uploadDate = chapterDateFormat.tryParse(element.selectFirst(".chapterdate")?.text()), + branch = null, + source = source + ) + } + return if (tableMode) parseInfoTable(docs, manga, chapters) else parseInfoList(docs, manga, chapters) + } + + open suspend fun parseInfoTable(docs: Document, manga: Manga, chapters: List): Manga { + val mangaInfo = docs.selectFirst("div.seriestucontent > div.seriestucontentr") + val mangaState = mangaInfo?.selectFirst(".infotable td:contains(Status)")?.lastElementSibling()?.let { + when (it.text()) { + "Ongoing" -> MangaState.ONGOING + "Completed" -> MangaState.FINISHED + else -> null + } + } + val tagMap = getOrCreateTagMap() + val tags = mangaInfo?.select(".seriestugenre > a")?.mapNotNullToSet { tagMap[it.text()] } + + return manga.copy( + description = mangaInfo?.selectFirst("div.entry-content")?.html(), + state = mangaState, + author = mangaInfo?.selectFirst(".infotable td:contains(Author)")?.lastElementSibling()?.text(), + isNsfw = docs.selectFirst(".restrictcontainer") != null, + tags = tags.orEmpty(), + chapters = chapters, + ) + } + + open suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { + val mangaState = docs.selectFirst(".info-left .tsinfo div:contains(Status)")?.lastElementChild()?.let { + when (it.text()) { + "Ongoing" -> MangaState.ONGOING + "Completed" -> MangaState.FINISHED + else -> null + } + } + val tagMap = getOrCreateTagMap() + val tags = docs.select(".info-right .mgen > a").mapNotNullToSet { tagMap[it.text()] } + + return manga.copy( + description = docs.selectFirst(".info-right div.entry-content > p")?.html(), + state = mangaState, + author = docs.selectFirst(".info-left .tsinfo div:contains(Author)")?.lastElementChild()?.text(), + isNsfw = docs.selectFirst(".info-right .alr") != null, + tags = tags, + chapters = chapters, + ) + } + + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder + ): List { + if (!query.isNullOrEmpty()) { + if (page > lastSearchPage) { + return emptyList() + } + + val url = buildString { + append("https://") + append(domain) + append("/page/") + append(page) + append("/?s=") + append(query.urlEncoded()) + } + + val docs = webClient.httpGet(url).parseHtml() + lastSearchPage = docs.selectFirst(".pagination .next") + ?.previousElementSibling() + ?.text()?.toIntOrNull() ?: 1 + return parseMangaList(docs) + } + + val sortQuery = when (sortOrder) { + SortOrder.ALPHABETICAL -> "title" + SortOrder.NEWEST -> "latest" + SortOrder.POPULARITY -> "popular" + SortOrder.UPDATED -> "update" + else -> "" + } + val tagKey = "genre[]".urlEncoded() + val tagQuery = + if (tags.isNullOrEmpty()) "" else tags.joinToString(separator = "&", prefix = "&") { "$tagKey=${it.key}" } + val url = buildString { + append("https://") + append(domain) + append(listUrl) + append("/?order=") + append(sortQuery) + append(tagQuery) + append("&page=") + append(page) + } + + return parseMangaList(webClient.httpGet(url).parseHtml()) + } + + private fun parseMangaList(docs: Document): List { + return docs.select(".postbody .listupd .bs .bsx").mapNotNull { + val a = it.selectFirst("a") ?: return@mapNotNull null + val relativeUrl = a.attrAsRelativeUrl("href") + val rating = it.selectFirst(".numscore")?.text() + ?.toFloatOrNull()?.div(10) ?: RATING_UNKNOWN + + Manga( + id = generateUid(relativeUrl), + url = relativeUrl, + title = a.attr("title"), + altTitle = null, + publicUrl = a.attrAsAbsoluteUrl("href"), + rating = rating, + isNsfw = false, + coverUrl = it.selectFirst("img.ts-post-image")?.imageUrl().orEmpty(), + tags = emptySet(), + state = null, + author = null, + source = source + ) + } + } + + override suspend fun getPages(chapter: MangaChapter): List { + val chapterUrl = chapter.url.toAbsoluteUrl(domain) + val docs = webClient.httpGet(chapterUrl).parseHtml() + val script = docs.selectFirstOrThrow("script:containsData(ts_reader)") + val images = JSONObject(script.data().substringAfter('(').substringBeforeLast(')')) + .getJSONArray("sources") + .getJSONObject(0) + .getJSONArray("images") + + val pages = ArrayList(images.length()) + for (i in 0 until images.length()) { + pages.add( + MangaPage( + id = generateUid(images.getString(i)), + url = images.getString(i), + referer = chapterUrl, + preview = null, + source = source + ) + ) + } + + return pages + } + + override suspend fun getTags(): Set { + return getOrCreateTagMap().values.toSet() + } + + protected suspend fun getOrCreateTagMap(): Map = mutex.withLock { + tagCache?.let { return@withLock it } + val tagMap = ArrayMap() + + val url = listUrl.toAbsoluteUrl(domain) + val tagElements = webClient.httpGet(url).parseHtml().select("ul.genrez > li") + for (el in tagElements) { + if (el.text().isEmpty()) continue + + tagMap[el.text()] = MangaTag( + title = el.text(), + key = el.selectFirst("input")?.attr("value") ?: continue, + source = source + ) + } + + tagCache = tagMap + return@withLock tagMap + } + + private fun Element.imageUrl(): String { + return attrAsAbsoluteUrlOrNull("src") + ?: attrAsAbsoluteUrlOrNull("data-cfsrc") + ?: "" + } + + @MangaSourceParser("MANHWALAND", "Manhwaland", "id") + class ManhwaLandParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.MANHWALAND, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("manhwaland.guru", null) + + override val listUrl: String + get() = "/series" + override val tableMode: Boolean + get() = false + } + + @MangaSourceParser("SEKAIKOMIK", "Sekaikomik", "id") + class SekaikomikParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.SEKAIKOMIK, pageSize = 20, searchPageSize = 100) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("sekaikomik.pro", null) + + override val listUrl: String + get() = "/manga" + override val tableMode: Boolean + get() = false + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMMM D, yyyy", idLocale) + } + + @MangaSourceParser("MANHWAINDO", "Manhwaindo", "id") + class ManhwaIndoParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.MANHWAINDO, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("manhwaindo.id", null) + + override val listUrl: String get() = "/series" + override val tableMode: Boolean get() = false + } + + @MangaSourceParser("MANHWALIST", "Manhwalist", "id") + class ManhwalistParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.MANHWALIST, pageSize = 24, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("manhwalist.in", null) + + override val listUrl: String = "/manga" + override val tableMode: Boolean get() = false + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMMM d, yyyy", Locale.ENGLISH) + } + + @MangaSourceParser("KIRYUU", "Kiryuu", "id") + class KiryuuParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.KIRYUU, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("kiryuu.id", null) + + override val listUrl: String + get() = "/manga" + override val tableMode: Boolean + get() = true + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMMM d, yyyy", idLocale) + } + + @MangaSourceParser("TURKTOON", "Turktoon", "tr") + class TurktoonParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.TURKTOON, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("turktoon.com", null) + + override val listUrl: String + get() = "/manga" + override val tableMode: Boolean + get() = false + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMMM d, yyyy", Locale("tr", "TR")) + + override suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { + val mangaState = docs.selectFirst(".info-left .tsinfo div:contains(Durum)")?.lastElementChild()?.let { + when (it.text()) { + "Devam Ediyor" -> MangaState.ONGOING + "Tamamlandı" -> MangaState.FINISHED + else -> null + } + } + + return super.parseInfoList(docs, manga, chapters).copy(state = mangaState) + } + } + + @MangaSourceParser("WESTMANGA", "Westmanga", "id") + class WestmangaParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.WESTMANGA, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("westmanga.info", null) + + override val listUrl: String + get() = "/manga" + override val tableMode: Boolean + get() = true + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMMM d, yyyy", Locale.ENGLISH) + } + + @MangaSourceParser("TEMPESTFANSUB", "Tempestfansub", "tr") + class TempestfansubParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.TEMPESTFANSUB, pageSize = 40, searchPageSize = 40) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("manga.tempestfansub.com", null) + + override val listUrl: String get() = "/manga" + override val tableMode: Boolean get() = true + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMMM d, yyyy", Locale("tr", "TR")) + + override suspend fun parseInfoTable(docs: Document, manga: Manga, chapters: List): Manga { + val infoElement = docs.selectFirst("div.infox") + return manga.copy( + chapters = chapters, + description = infoElement?.selectFirst("div.entry-content")?.html(), + author = infoElement?.selectFirst(".flex-wrap div:contains(Artist)")?.lastElementSibling()?.text(), + tags = infoElement?.select(".wd-full .mgen > a") + ?.mapNotNullToSet { getOrCreateTagMap()[it.text()] } + .orEmpty(), + isNsfw = docs.selectFirst(".postbody .alr") != null, + ) + } + } + + @MangaSourceParser("MANHWADESU", "ManhwaDesu", "id") + class ManhwadesuParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.MANHWADESU, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("manhwadesu.org", null) + + override val listUrl: String get() = "/komik" + override val tableMode: Boolean get() = false + } + + @MangaSourceParser("MANGATALE", "MangaTale", "id") + class MangaTaleParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.MANGATALE, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("mangatale.co", null) + + override val listUrl: String get() = "/manga" + override val tableMode: Boolean get() = false + + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMMM d, yyyy", Locale.ENGLISH) + + override suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { + val infoElement = docs.selectFirst("div.infox") + return manga.copy( + chapters = chapters, + description = infoElement?.selectFirst("div.entry-content")?.html(), + author = infoElement?.selectFirst(".flex-wrap div:contains(Author)")?.lastElementSibling()?.text(), + tags = infoElement?.select(".wd-full .mgen > a") + ?.mapNotNullToSet { getOrCreateTagMap()[it.text()] } + .orEmpty(), + isNsfw = docs.selectFirst(".postbody .alr") != null, + ) + } + } + + @MangaSourceParser("DRAGONTRANSLATION", "DragonTranslation", "es") + class DragonTranslationParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.DRAGONTRANSLATION, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("dragontranslation.com", null) + + override val listUrl: String get() = "/manga" + override val tableMode: Boolean get() = false + + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMMM d, yyyy", Locale.ENGLISH) + + override suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { + val infoElement = docs.selectFirst("div.infox") + return manga.copy( + chapters = chapters, + description = infoElement?.selectFirst("div.entry-content")?.html(), + author = infoElement?.selectFirst(".flex-wrap div:contains(Author)")?.lastElementSibling()?.text(), + tags = infoElement?.select(".wd-full .mgen > a") + ?.mapNotNullToSet { getOrCreateTagMap()[it.text()] } + .orEmpty(), + isNsfw = docs.selectFirst(".postbody .alr") != null, + ) + } + } + + @MangaSourceParser("ASURATR", "Asura Scans (tr)", "tr") + class AsuraTRParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.ASURATR, pageSize = 30, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("asurascanstr.com", null) + + override val listUrl: String get() = "/manga" + override val tableMode: Boolean get() = false + + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", Locale("tr")) + + override suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { + val infoElement = docs.selectFirst("div.infox") + return manga.copy( + chapters = chapters, + description = infoElement?.selectFirst("div.entry-content")?.html(), + author = infoElement?.selectFirst(".flex-wrap div:contains(Yazar)")?.lastElementSibling()?.text(), + tags = infoElement?.select(".wd-full .mgen > a") + ?.mapNotNullToSet { getOrCreateTagMap()[it.text()] } + .orEmpty(), + isNsfw = docs.selectFirst(".postbody .alr") != null, + ) + } + } + + @MangaSourceParser("KOMIKTAP", "KomikTap", "id") + class KomikTapParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.KOMIKTAP, pageSize = 15, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("92.87.6.124", null) + + override val listUrl: String + get() = "/manga" + override val tableMode: Boolean + get() = true + + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", idLocale) + } + + @MangaSourceParser("ASURASCANS", "Asura Scans", "en") + class AsuraScansParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.ASURASCANS, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("asurascans.com", null) + + override val listUrl: String + get() = "/manga" + override val tableMode: Boolean + get() = false + + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH) + + override suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { + val infoElement = docs.selectFirst("div.infox") + return manga.copy( + chapters = chapters, + description = infoElement?.selectFirst("div.entry-content")?.html(), + author = infoElement?.selectFirst(".flex-wrap div:contains(Author)")?.lastElementSibling()?.text(), + tags = infoElement?.select(".wd-full .mgen > a") + ?.mapNotNullToSet { getOrCreateTagMap()[it.text()] } + .orEmpty(), + isNsfw = docs.selectFirst(".postbody .alr") != null, + ) + } + } + + @MangaSourceParser("COSMICSCANS", "CosmicScans", "en") + class CosmicScansParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.COSMICSCANS, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("cosmicscans.com", null) + + override val listUrl: String + get() = "/manga" + override val tableMode: Boolean + get() = false + + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH) + + override suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { + val infoElement = docs.selectFirst("div.infox") + return manga.copy( + chapters = chapters, + description = infoElement?.selectFirst("div.entry-content")?.html(), + author = infoElement?.selectFirst(".flex-wrap div:contains(Author)")?.lastElementSibling()?.text(), + tags = infoElement?.select(".wd-full .mgen > a") + ?.mapNotNullToSet { getOrCreateTagMap()[it.text()] } + .orEmpty(), + ) + } + } + + @MangaSourceParser("MANGASUSU", "Mangasusu", "id") + class MangasusuParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.MANGASUSU, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("92.84.132.251", null) + + override val listUrl: String + get() = "/project" + override val tableMode: Boolean + get() = true + + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH) + } + + @MangaSourceParser("KOMIKINDO", "KomikIndo", "id") + class KomikIndoParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.KOMIKINDO, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("komikindo.co", null) + + override val listUrl: String + get() = "/project" + override val tableMode: Boolean + get() = true + + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", idLocale) + } + + @MangaSourceParser("KOMIKMANGA", "KomikManga", "id") + class KomikMangaParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.KOMIKMANGA, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("komikmanga.xyz", null) + + override val listUrl: String + get() = "/project" + override val tableMode: Boolean + get() = true + + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH) + } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/multichan/ChanParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/multichan/ChanParser.kt index 5792505d7..2a886fca3 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/multichan/ChanParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/multichan/ChanParser.kt @@ -1,5 +1,6 @@ package org.koitharu.kotatsu.parsers.site.multichan +import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaParser import org.koitharu.kotatsu.parsers.MangaParserAuthProvider import org.koitharu.kotatsu.parsers.exception.AuthRequiredException @@ -8,172 +9,175 @@ import org.koitharu.kotatsu.parsers.util.* import java.text.SimpleDateFormat import java.util.* -internal abstract class ChanParser(source: MangaSource) : MangaParser(source), MangaParserAuthProvider { +internal abstract class ChanParser( + context: MangaLoaderContext, + source: MangaSource, +) : MangaParser(context, source), MangaParserAuthProvider { - override val sortOrders: Set = EnumSet.of( - SortOrder.NEWEST, - SortOrder.POPULARITY, - SortOrder.ALPHABETICAL, - ) + override val sortOrders: Set = EnumSet.of( + SortOrder.NEWEST, + SortOrder.POPULARITY, + SortOrder.ALPHABETICAL, + ) - override val authUrl: String - get() = "https://${getDomain()}" + override val authUrl: String + get() = "https://${domain}" - override val isAuthorized: Boolean - get() = context.cookieJar.getCookies(getDomain()).any { it.name == "dle_user_id" } + override val isAuthorized: Boolean + get() = context.cookieJar.getCookies(domain).any { it.name == "dle_user_id" } - override suspend fun getList( - offset: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - val domain = getDomain() - val url = when { - !query.isNullOrEmpty() -> { - if (offset != 0) { - return emptyList() - } - "https://$domain/?do=search&subaction=search&story=${query.urlEncoded()}" - } + override suspend fun getList( + offset: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + val domain = domain + val url = when { + !query.isNullOrEmpty() -> { + if (offset != 0) { + return emptyList() + } + "https://$domain/?do=search&subaction=search&story=${query.urlEncoded()}" + } - !tags.isNullOrEmpty() -> tags.joinToString( - prefix = "https://$domain/tags/", - postfix = "&n=${getSortKey2(sortOrder)}?offset=$offset", - separator = "+", - ) { tag -> tag.key } + !tags.isNullOrEmpty() -> tags.joinToString( + prefix = "https://$domain/tags/", + postfix = "&n=${getSortKey2(sortOrder)}?offset=$offset", + separator = "+", + ) { tag -> tag.key } - else -> "https://$domain/${getSortKey(sortOrder)}?offset=$offset" - } - val doc = context.httpGet(url).parseHtml() - val root = doc.body().selectFirst("div.main_fon")?.getElementById("content") - ?: doc.parseFailed("Cannot find root") - return root.select("div.content_row").mapNotNull { row -> - val a = row.selectFirst("div.manga_row1")?.selectFirst("h2")?.selectFirst("a") - ?: return@mapNotNull null - val href = a.attrAsRelativeUrl("href") - Manga( - id = generateUid(href), - url = href, - publicUrl = href.toAbsoluteUrl(a.host ?: domain), - altTitle = a.attr("title"), - title = a.text().substringAfterLast('(').substringBeforeLast(')'), - author = row.getElementsByAttributeValueStarting( - "href", - "/mangaka", - ).firstOrNull()?.text(), - coverUrl = row.selectFirst("div.manga_images")?.selectFirst("img") - ?.absUrl("src").orEmpty(), - tags = runCatching { - row.selectFirst("div.genre")?.select("a")?.mapToSet { - MangaTag( - title = it.text().toTagName(), - key = it.attr("href").substringAfterLast('/').urlEncoded(), - source = source, - ) - } - }.getOrNull().orEmpty(), - rating = RATING_UNKNOWN, - state = null, - isNsfw = false, - source = source, - ) - } - } + else -> "https://$domain/${getSortKey(sortOrder)}?offset=$offset" + } + val doc = webClient.httpGet(url).parseHtml() + val root = doc.body().selectFirst("div.main_fon")?.getElementById("content") + ?: doc.parseFailed("Cannot find root") + return root.select("div.content_row").mapNotNull { row -> + val a = row.selectFirst("div.manga_row1")?.selectFirst("h2")?.selectFirst("a") + ?: return@mapNotNull null + val href = a.attrAsRelativeUrl("href") + Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(a.host ?: domain), + altTitle = a.attr("title"), + title = a.text().substringAfterLast('(').substringBeforeLast(')'), + author = row.getElementsByAttributeValueStarting( + "href", + "/mangaka", + ).firstOrNull()?.text(), + coverUrl = row.selectFirst("div.manga_images")?.selectFirst("img") + ?.absUrl("src").orEmpty(), + tags = runCatching { + row.selectFirst("div.genre")?.select("a")?.mapToSet { + MangaTag( + title = it.text().toTagName(), + key = it.attr("href").substringAfterLast('/').urlEncoded(), + source = source, + ) + } + }.getOrNull().orEmpty(), + rating = RATING_UNKNOWN, + state = null, + isNsfw = false, + source = source, + ) + } + } - override suspend fun getDetails(manga: Manga): Manga { - val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml() - val root = doc.body().getElementById("dle-content") ?: doc.parseFailed("Cannot find root") - val dateFormat = SimpleDateFormat("yyyy-MM-dd", Locale.US) - return manga.copy( - description = root.getElementById("description")?.html()?.substringBeforeLast(" - val href = tr?.selectFirst("a")?.attrAsRelativeUrlOrNull("href") - ?: return@mapChapters null - MangaChapter( - id = generateUid(href), - name = tr.selectFirst("a")?.text().orEmpty(), - number = i + 1, - url = href, - scanlator = null, - branch = null, - uploadDate = dateFormat.tryParse(tr.selectFirst("div.date")?.text()), - source = source, - ) - }, - ) - } + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val root = doc.body().getElementById("dle-content") ?: doc.parseFailed("Cannot find root") + val dateFormat = SimpleDateFormat("yyyy-MM-dd", Locale.US) + return manga.copy( + description = root.getElementById("description")?.html()?.substringBeforeLast(" + val href = tr?.selectFirst("a")?.attrAsRelativeUrlOrNull("href") + ?: return@mapChapters null + MangaChapter( + id = generateUid(href), + name = tr.selectFirst("a")?.text().orEmpty(), + number = i + 1, + url = href, + scanlator = null, + branch = null, + uploadDate = dateFormat.tryParse(tr.selectFirst("div.date")?.text()), + source = source, + ) + }, + ) + } - override suspend fun getPages(chapter: MangaChapter): List { - val fullUrl = chapter.url.toAbsoluteUrl(getDomain()) - val doc = context.httpGet(fullUrl).parseHtml() - val scripts = doc.select("script") - for (script in scripts) { - val data = script.html() - val pos = data.indexOf("\"fullimg") - if (pos == -1) { - continue - } - val json = data.substring(pos).substringAfter('[').substringBefore(';') - .substringBeforeLast(']') - val domain = getDomain() - return json.split(",").mapNotNull { - it.trim() - .removeSurrounding('"', '\'') - .toRelativeUrl(domain) - .takeUnless(String::isBlank) - }.map { url -> - MangaPage( - id = generateUid(url), - url = url, - preview = null, - referer = fullUrl, - source = source, - ) - } - } - doc.parseFailed("Pages list not found at ${chapter.url}") - } + override suspend fun getPages(chapter: MangaChapter): List { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + val scripts = doc.select("script") + for (script in scripts) { + val data = script.html() + val pos = data.indexOf("\"fullimg") + if (pos == -1) { + continue + } + val json = data.substring(pos).substringAfter('[').substringBefore(';') + .substringBeforeLast(']') + val domain = domain + return json.split(",").mapNotNull { + it.trim() + .removeSurrounding('"', '\'') + .toRelativeUrl(domain) + .takeUnless(String::isBlank) + }.map { url -> + MangaPage( + id = generateUid(url), + url = url, + preview = null, + referer = fullUrl, + source = source, + ) + } + } + doc.parseFailed("Pages list not found at ${chapter.url}") + } - override suspend fun getTags(): Set { - val domain = getDomain() - val doc = context.httpGet("https://$domain/mostfavorites&sort=manga").parseHtml() - val root = doc.body().selectFirst("div.main_fon")?.getElementById("side") - ?.select("ul")?.last() ?: doc.parseFailed("Cannot find root") - return root.select("li.sidetag").mapToSet { li -> - val a = li.children().lastOrNull() ?: li.parseFailed("a is null") - MangaTag( - title = a.text().toTagName(), - key = a.attr("href").substringAfterLast('/'), - source = source, - ) - } - } + override suspend fun getTags(): Set { + val domain = domain + val doc = webClient.httpGet("https://$domain/mostfavorites&sort=manga").parseHtml() + val root = doc.body().selectFirst("div.main_fon")?.getElementById("side") + ?.select("ul")?.last() ?: doc.parseFailed("Cannot find root") + return root.select("li.sidetag").mapToSet { li -> + val a = li.children().lastOrNull() ?: li.parseFailed("a is null") + MangaTag( + title = a.text().toTagName(), + key = a.attr("href").substringAfterLast('/'), + source = source, + ) + } + } - override suspend fun getUsername(): String { - val doc = context.httpGet("https://${getDomain()}").parseHtml().body() - val root = doc.requireElementById("top_user") - val a = root.getElementsByAttributeValueContaining("href", "/user/").firstOrNull() - ?: throw AuthRequiredException(source) - return a.attr("href").removeSuffix('/').substringAfterLast('/') - } + override suspend fun getUsername(): String { + val doc = webClient.httpGet("https://${domain}").parseHtml().body() + val root = doc.requireElementById("top_user") + val a = root.getElementsByAttributeValueContaining("href", "/user/").firstOrNull() + ?: throw AuthRequiredException(source) + return a.attr("href").removeSuffix('/').substringAfterLast('/') + } - private fun getSortKey(sortOrder: SortOrder) = - when (sortOrder) { - SortOrder.ALPHABETICAL -> "catalog" - SortOrder.POPULARITY -> "mostfavorites" - SortOrder.NEWEST -> "manga/new" - else -> "mostfavorites" - } + private fun getSortKey(sortOrder: SortOrder) = + when (sortOrder) { + SortOrder.ALPHABETICAL -> "catalog" + SortOrder.POPULARITY -> "mostfavorites" + SortOrder.NEWEST -> "manga/new" + else -> "mostfavorites" + } - private fun getSortKey2(sortOrder: SortOrder) = - when (sortOrder) { - SortOrder.ALPHABETICAL -> "abcasc" - SortOrder.POPULARITY -> "favdesc" - SortOrder.NEWEST -> "datedesc" - else -> "favdesc" - } + private fun getSortKey2(sortOrder: SortOrder) = + when (sortOrder) { + SortOrder.ALPHABETICAL -> "abcasc" + SortOrder.POPULARITY -> "favdesc" + SortOrder.NEWEST -> "datedesc" + else -> "favdesc" + } - private fun String.toTagName() = replace('_', ' ').toTitleCase() -} \ No newline at end of file + private fun String.toTagName() = replace('_', ' ').toTitleCase() +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/multichan/HenChanParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/multichan/HenChanParser.kt index 97a5f032d..eec7465c5 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/multichan/HenChanParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/multichan/HenChanParser.kt @@ -7,54 +7,54 @@ import org.koitharu.kotatsu.parsers.model.* import org.koitharu.kotatsu.parsers.util.* @MangaSourceParser("HENCHAN", "Хентай-тян", "ru") -internal class HenChanParser(override val context: MangaLoaderContext) : ChanParser(MangaSource.HENCHAN) { +internal class HenChanParser(context: MangaLoaderContext) : ChanParser(context, MangaSource.HENCHAN) { - override val configKeyDomain = ConfigKey.Domain( - "y.hentaichan.live", - arrayOf("y.hentaichan.live", "xxx.hentaichan.live", "xx.hentaichan.live", "hentaichan.live", "hentaichan.pro"), - ) + override val configKeyDomain = ConfigKey.Domain( + "y.hentaichan.live", + arrayOf("y.hentaichan.live", "xxx.hentaichan.live", "xx.hentaichan.live", "hentaichan.live", "hentaichan.pro"), + ) - override suspend fun getList( - offset: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - return super.getList(offset, query, tags, sortOrder).map { - it.copy( - coverUrl = it.coverUrl.replace("_blur", ""), - isNsfw = true, - ) - } - } + override suspend fun getList( + offset: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + return super.getList(offset, query, tags, sortOrder).map { + it.copy( + coverUrl = it.coverUrl.replace("_blur", ""), + isNsfw = true, + ) + } + } - override suspend fun getDetails(manga: Manga): Manga { - val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml() - val root = doc.body().requireElementById("dle-content") - val readLink = manga.url.replace("manga", "online") - return manga.copy( - description = root.getElementById("description")?.html()?.substringBeforeLast(" - table.select("div.manga") - }.mapNotNull { it.selectFirst("a") }.reversed().mapChapters { i, a -> - val href = a.attrAsRelativeUrl("href") - MangaChapter( - id = generateUid(href), - name = a.text().trim(), - number = i + 1, - url = href, - uploadDate = 0L, - source = source, - scanlator = null, - branch = null, - ) - }, - ) - } -} \ No newline at end of file + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val root = doc.body().requireElementById("dle-content") + return manga.copy( + description = root.getElementById("description")?.html()?.substringBeforeLast(" + table.select("div.manga") + }.mapNotNull { it.selectFirst("a") }.reversed().mapChapters { i, a -> + val href = a.attrAsRelativeUrl("href") + MangaChapter( + id = generateUid(href), + name = a.text().trim(), + number = i + 1, + url = href, + uploadDate = 0L, + source = source, + scanlator = null, + branch = null, + ) + }, + ) + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/rulib/MangaLibParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/rulib/MangaLibParser.kt index 27872a768..d00dd2dd1 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/rulib/MangaLibParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/rulib/MangaLibParser.kt @@ -20,284 +20,284 @@ import java.text.SimpleDateFormat import java.util.* internal open class MangaLibParser( - override val context: MangaLoaderContext, - source: MangaSource, -) : PagedMangaParser(source, pageSize = 60), MangaParserAuthProvider { + context: MangaLoaderContext, + source: MangaSource, +) : PagedMangaParser(context, source, pageSize = 60), MangaParserAuthProvider { - override val configKeyDomain = ConfigKey.Domain("mangalib.me", null) + override val configKeyDomain = ConfigKey.Domain("mangalib.me", null) - override val authUrl: String - get() = "https://${getDomain()}/login" + override val authUrl: String + get() = "https://${domain}/login" - override val sortOrders: Set = EnumSet.of( - SortOrder.RATING, - SortOrder.ALPHABETICAL, - SortOrder.POPULARITY, - SortOrder.UPDATED, - SortOrder.NEWEST, - ) + override val sortOrders: Set = EnumSet.of( + SortOrder.RATING, + SortOrder.ALPHABETICAL, + SortOrder.POPULARITY, + SortOrder.UPDATED, + SortOrder.NEWEST, + ) - override suspend fun getListPage( - page: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - if (!query.isNullOrEmpty()) { - return if (page == searchPaginator.firstPage) search(query) else emptyList() - } - val url = buildString { - append("https://") - append(getDomain()) - append("/manga-list?dir=") - append(getSortKey(sortOrder)) - append("&page=") - append(page) - tags?.forEach { tag -> - append("&genres[include][]=") - append(tag.key) - } - } - val doc = context.httpGet(url).parseHtml() - val root = doc.body().getElementById("manga-list") ?: doc.parseFailed("Root not found") - val items = root.selectFirst("div.media-cards-grid")?.select("div.media-card-wrap") - ?: return emptyList() - return items.mapNotNull { card -> - val a = card.selectFirst("a.media-card") ?: return@mapNotNull null - val href = a.attrAsRelativeUrl("href") - Manga( - id = generateUid(href), - title = card.selectFirst("h3")?.text().orEmpty(), - coverUrl = a.absUrl("data-src"), - altTitle = null, - author = null, - rating = RATING_UNKNOWN, - url = href, - publicUrl = href.toAbsoluteUrl(a.host ?: getDomain()), - tags = emptySet(), - state = null, - isNsfw = false, - source = source, - ) - } - } + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + if (!query.isNullOrEmpty()) { + return if (page == searchPaginator.firstPage) search(query) else emptyList() + } + val url = buildString { + append("https://") + append(domain) + append("/manga-list?dir=") + append(getSortKey(sortOrder)) + append("&page=") + append(page) + tags?.forEach { tag -> + append("&genres[include][]=") + append(tag.key) + } + } + val doc = webClient.httpGet(url).parseHtml() + val root = doc.body().getElementById("manga-list") ?: doc.parseFailed("Root not found") + val items = root.selectFirst("div.media-cards-grid")?.select("div.media-card-wrap") + ?: return emptyList() + return items.mapNotNull { card -> + val a = card.selectFirst("a.media-card") ?: return@mapNotNull null + val href = a.attrAsRelativeUrl("href") + Manga( + id = generateUid(href), + title = card.selectFirst("h3")?.text().orEmpty(), + coverUrl = a.absUrl("data-src"), + altTitle = null, + author = null, + rating = RATING_UNKNOWN, + url = href, + publicUrl = href.toAbsoluteUrl(a.host ?: domain), + tags = emptySet(), + state = null, + isNsfw = false, + source = source, + ) + } + } - override suspend fun getDetails(manga: Manga): Manga { - val fullUrl = manga.url.toAbsoluteUrl(getDomain()) - val doc = context.httpGet("$fullUrl?section=info").parseHtml() - val root = doc.body().getElementById("main-page") ?: throw ParseException("Root not found", fullUrl) - val title = root.selectFirst("div.media-header__wrap")?.children() - val info = root.selectFirst("div.media-content") - val chaptersDoc = context.httpGet("$fullUrl?section=chapters").parseHtml() - val scripts = chaptersDoc.select("script") - val dateFormat = SimpleDateFormat("yyy-MM-dd", Locale.US) - var chapters: ChaptersListBuilder? = null - scripts@ for (script in scripts) { - val raw = script.html().lines() - for (line in raw) { - if (line.startsWith("window.__DATA__")) { - val json = JSONObject(line.substringAfter('=').substringBeforeLast(';')) - val list = json.getJSONObject("chapters").getJSONArray("list") - val id = json.optJSONObject("user")?.getLong("id")?.toString() ?: "not" - val total = list.length() - chapters = ChaptersListBuilder(total) - for (i in 0 until total) { - val item = list.getJSONObject(i) - val chapterId = item.getLong("chapter_id") - val scanlator = item.getStringOrNull("username") - val url = buildString { - if (isAuthorized) { - append(manga.url) - append("/v") - append(item.getInt("chapter_volume")) - append("/c") - append(item.getString("chapter_number")) - append("?ui=") - append(id) - } else { - append(manga.url) - append("/v") - append(item.getInt("chapter_volume")) - append("/c") - append(item.getString("chapter_number")) - } - } - val nameChapter = item.getStringOrNull("chapter_name") - val volume = item.getInt("chapter_volume") - val number = item.getString("chapter_number") - val fullNameChapter = "Том $volume. Глава $number" - chapters.add( - MangaChapter( - id = generateUid(chapterId), - url = url, - source = source, - number = total - i, - uploadDate = dateFormat.tryParse( - item.getString("chapter_created_at").substringBefore(" "), - ), - scanlator = scanlator, - branch = null, - name = if (nameChapter.isNullOrBlank()) fullNameChapter else "$fullNameChapter - $nameChapter", - ), - ) - } - chapters.reverse() - break@scripts - } - } - } - return manga.copy( - title = title?.getOrNull(0)?.text()?.takeUnless(String::isBlank) ?: manga.title, - altTitle = title?.getOrNull(1)?.text()?.substringBefore('/')?.trim(), - rating = root.selectFirst("div.media-stats-item__score") - ?.selectFirst("span") - ?.text()?.toFloatOrNull()?.div(5f) ?: manga.rating, - author = info?.getElementsMatchingOwnText("Автор")?.firstOrNull() - ?.nextElementSibling()?.text() ?: manga.author, - tags = info?.selectFirst("div.media-tags") - ?.select("a.media-tag-item")?.mapNotNullToSet { a -> - val href = a.attr("href") - if (href.contains("genres")) { - MangaTag( - title = a.text().toTitleCase(), - key = href.substringAfterLast('='), - source = source, - ) - } else null - } ?: manga.tags, - isNsfw = isNsfw(doc), - description = info?.selectFirst("div.media-description__text")?.html(), - chapters = chapters?.toList(), - ) - } + override suspend fun getDetails(manga: Manga): Manga { + val fullUrl = manga.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet("$fullUrl?section=info").parseHtml() + val root = doc.body().getElementById("main-page") ?: throw ParseException("Root not found", fullUrl) + val title = root.selectFirst("div.media-header__wrap")?.children() + val info = root.selectFirst("div.media-content") + val chaptersDoc = webClient.httpGet("$fullUrl?section=chapters").parseHtml() + val scripts = chaptersDoc.select("script") + val dateFormat = SimpleDateFormat("yyy-MM-dd", Locale.US) + var chapters: ChaptersListBuilder? = null + scripts@ for (script in scripts) { + val raw = script.html().lines() + for (line in raw) { + if (line.startsWith("window.__DATA__")) { + val json = JSONObject(line.substringAfter('=').substringBeforeLast(';')) + val list = json.getJSONObject("chapters").getJSONArray("list") + val id = json.optJSONObject("user")?.getLong("id")?.toString() ?: "not" + val total = list.length() + chapters = ChaptersListBuilder(total) + for (i in 0 until total) { + val item = list.getJSONObject(i) + val chapterId = item.getLong("chapter_id") + val scanlator = item.getStringOrNull("username") + val url = buildString { + if (isAuthorized) { + append(manga.url) + append("/v") + append(item.getInt("chapter_volume")) + append("/c") + append(item.getString("chapter_number")) + append("?ui=") + append(id) + } else { + append(manga.url) + append("/v") + append(item.getInt("chapter_volume")) + append("/c") + append(item.getString("chapter_number")) + } + } + val nameChapter = item.getStringOrNull("chapter_name") + val volume = item.getInt("chapter_volume") + val number = item.getString("chapter_number") + val fullNameChapter = "Том $volume. Глава $number" + chapters.add( + MangaChapter( + id = generateUid(chapterId), + url = url, + source = source, + number = total - i, + uploadDate = dateFormat.tryParse( + item.getString("chapter_created_at").substringBefore(" "), + ), + scanlator = scanlator, + branch = null, + name = if (nameChapter.isNullOrBlank()) fullNameChapter else "$fullNameChapter - $nameChapter", + ), + ) + } + chapters.reverse() + break@scripts + } + } + } + return manga.copy( + title = title?.getOrNull(0)?.text()?.takeUnless(String::isBlank) ?: manga.title, + altTitle = title?.getOrNull(1)?.text()?.substringBefore('/')?.trim(), + rating = root.selectFirst("div.media-stats-item__score") + ?.selectFirst("span") + ?.text()?.toFloatOrNull()?.div(5f) ?: manga.rating, + author = info?.getElementsMatchingOwnText("Автор")?.firstOrNull() + ?.nextElementSibling()?.text() ?: manga.author, + tags = info?.selectFirst("div.media-tags") + ?.select("a.media-tag-item")?.mapNotNullToSet { a -> + val href = a.attr("href") + if (href.contains("genres")) { + MangaTag( + title = a.text().toTitleCase(), + key = href.substringAfterLast('='), + source = source, + ) + } else null + } ?: manga.tags, + isNsfw = isNsfw(doc), + description = info?.selectFirst("div.media-description__text")?.html(), + chapters = chapters?.toList(), + ) + } - override suspend fun getPages(chapter: MangaChapter): List { - val fullUrl = chapter.url.toAbsoluteUrl(getDomain()) - val doc = context.httpGet(fullUrl).parseHtml() - if (doc.location().substringBefore('?').endsWith("/register")) { - throw AuthRequiredException(source) - } - val scripts = doc.head().select("script") - val pg = (doc.body().getElementById("pg")?.html() ?: doc.parseFailed("Element #pg not found")) - .substringAfter('=') - .substringBeforeLast(';') - val pages = JSONArray(pg) - for (script in scripts) { - val raw = script.html().trim() - if (raw.contains("window.__info")) { - val json = JSONObject( - raw.substringAfter("window.__info") - .substringAfter('=') - .substringBeforeLast(';'), - ) - val domain = json.getJSONObject("servers").run { - getStringOrNull("main") ?: getString( - json.getJSONObject("img").getString("server"), - ) - } - val url = json.getJSONObject("img").getString("url") - return pages.mapJSON { x -> - val pageUrl = "$domain/$url${x.getString("u")}" - MangaPage( - id = generateUid(pageUrl), - url = pageUrl, - preview = null, - referer = fullUrl, - source = source, - ) - } - } - } - throw ParseException("Script with info not found", fullUrl) - } + override suspend fun getPages(chapter: MangaChapter): List { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + if (doc.location().substringBefore('?').endsWith("/register")) { + throw AuthRequiredException(source) + } + val scripts = doc.head().select("script") + val pg = (doc.body().getElementById("pg")?.html() ?: doc.parseFailed("Element #pg not found")) + .substringAfter('=') + .substringBeforeLast(';') + val pages = JSONArray(pg) + for (script in scripts) { + val raw = script.html().trim() + if (raw.contains("window.__info")) { + val json = JSONObject( + raw.substringAfter("window.__info") + .substringAfter('=') + .substringBeforeLast(';'), + ) + val domain = json.getJSONObject("servers").run { + getStringOrNull("main") ?: getString( + json.getJSONObject("img").getString("server"), + ) + } + val url = json.getJSONObject("img").getString("url") + return pages.mapJSON { x -> + val pageUrl = "$domain/$url${x.getString("u")}" + MangaPage( + id = generateUid(pageUrl), + url = pageUrl, + preview = null, + referer = fullUrl, + source = source, + ) + } + } + } + throw ParseException("Script with info not found", fullUrl) + } - override suspend fun getTags(): Set { - val url = "https://${getDomain()}/manga-list" - val doc = context.httpGet(url).parseHtml() - val scripts = doc.body().select("script") - for (script in scripts) { - val raw = script.html().trim() - if (raw.startsWith("window.__DATA")) { - val json = JSONObject(raw.substringAfter('=').substringBeforeLast(';')) - val genres = json.getJSONObject("filters").getJSONArray("genres") - val result = ArraySet(genres.length()) - for (x in genres.JSONIterator()) { - result += MangaTag( - source = source, - key = x.getInt("id").toString(), - title = x.getString("name").toTitleCase(), - ) - } - return result - } - } - throw ParseException("Script with genres not found", url) - } + override suspend fun getTags(): Set { + val url = "https://${domain}/manga-list" + val doc = webClient.httpGet(url).parseHtml() + val scripts = doc.body().select("script") + for (script in scripts) { + val raw = script.html().trim() + if (raw.startsWith("window.__DATA")) { + val json = JSONObject(raw.substringAfter('=').substringBeforeLast(';')) + val genres = json.getJSONObject("filters").getJSONArray("genres") + val result = ArraySet(genres.length()) + for (x in genres.JSONIterator()) { + result += MangaTag( + source = source, + key = x.getInt("id").toString(), + title = x.getString("name").toTitleCase(), + ) + } + return result + } + } + throw ParseException("Script with genres not found", url) + } - override val isAuthorized: Boolean - get() { - return context.cookieJar.getCookies(getDomain()).any { - it.name.startsWith("remember_web_") - } - } + override val isAuthorized: Boolean + get() { + return context.cookieJar.getCookies(domain).any { + it.name.startsWith("remember_web_") + } + } - override suspend fun getUsername(): String { - val body = context.httpGet("https://${LibConst.LIB_SOCIAL_LINK}/messages").parseHtml().body() - if (body.baseUri().endsWith("/login")) { - throw AuthRequiredException(source) - } - return body.selectFirst(".profile-user__username")?.text() ?: body.parseFailed("Cannot find username") - } + override suspend fun getUsername(): String { + val body = webClient.httpGet("https://${LibConst.LIB_SOCIAL_LINK}/messages").parseHtml().body() + if (body.baseUri().endsWith("/login")) { + throw AuthRequiredException(source) + } + return body.selectFirst(".profile-user__username")?.text() ?: body.parseFailed("Cannot find username") + } - protected open fun isNsfw(doc: Document): Boolean { - val sidebar = doc.body().run { - selectFirst(".media-sidebar") ?: selectFirst(".media-info") - } ?: doc.parseFailed("Sidebar not found") - return sidebar.getElementsContainingOwnText("18+").isNotEmpty() - } + protected open fun isNsfw(doc: Document): Boolean { + val sidebar = doc.body().run { + selectFirst(".media-sidebar") ?: selectFirst(".media-info") + } ?: doc.parseFailed("Sidebar not found") + return sidebar.getElementsContainingOwnText("18+").isNotEmpty() + } - private fun getSortKey(sortOrder: SortOrder?) = when (sortOrder) { - SortOrder.RATING -> "desc&sort=rate" - SortOrder.ALPHABETICAL -> "asc&sort=name" - SortOrder.POPULARITY -> "desc&sort=views" - SortOrder.UPDATED -> "desc&sort=last_chapter_at" - SortOrder.NEWEST -> "desc&sort=created_at" - else -> "desc&sort=last_chapter_at" - } + private fun getSortKey(sortOrder: SortOrder?) = when (sortOrder) { + SortOrder.RATING -> "desc&sort=rate" + SortOrder.ALPHABETICAL -> "asc&sort=name" + SortOrder.POPULARITY -> "desc&sort=views" + SortOrder.UPDATED -> "desc&sort=last_chapter_at" + SortOrder.NEWEST -> "desc&sort=created_at" + else -> "desc&sort=last_chapter_at" + } - private suspend fun search(query: String): List { - val domain = getDomain() - val json = context.httpGet("https://$domain/search?type=manga&q=$query") - .parseJsonArray() - return json.mapJSON { jo -> - val slug = jo.getString("slug") - val url = "/$slug" - val covers = jo.getJSONObject("covers") - val title = jo.getString("rus_name").ifEmpty { jo.getString("name") } - Manga( - id = generateUid(url), - url = url, - publicUrl = "https://$domain/$slug", - title = title, - altTitle = jo.getString("name").takeUnless { it == title }, - author = null, - tags = emptySet(), - rating = jo.getString("rate_avg").toFloatOrNull()?.div(5f) ?: RATING_UNKNOWN, - state = null, - isNsfw = false, - source = source, - coverUrl = covers.getString("thumbnail").toAbsoluteUrl(domain), - largeCoverUrl = covers.getString("default").toAbsoluteUrl(domain), - ) - } - } + private suspend fun search(query: String): List { + val domain = domain + val json = webClient.httpGet("https://$domain/search?type=manga&q=$query") + .parseJsonArray() + return json.mapJSON { jo -> + val slug = jo.getString("slug") + val url = "/$slug" + val covers = jo.getJSONObject("covers") + val title = jo.getString("rus_name").ifEmpty { jo.getString("name") } + Manga( + id = generateUid(url), + url = url, + publicUrl = "https://$domain/$slug", + title = title, + altTitle = jo.getString("name").takeUnless { it == title }, + author = null, + tags = emptySet(), + rating = jo.getString("rate_avg").toFloatOrNull()?.div(5f) ?: RATING_UNKNOWN, + state = null, + isNsfw = false, + source = source, + coverUrl = covers.getString("thumbnail").toAbsoluteUrl(domain), + largeCoverUrl = covers.getString("default").toAbsoluteUrl(domain), + ) + } + } - @MangaSourceParser("MANGALIB", "MangaLib", "ru") - class Impl(context: MangaLoaderContext) : MangaLibParser(context, MangaSource.MANGALIB) + @MangaSourceParser("MANGALIB", "MangaLib", "ru") + class Impl(context: MangaLoaderContext) : MangaLibParser(context, MangaSource.MANGALIB) - object LibConst { + object LibConst { - val LIB_SOCIAL_LINK = "lib.social" + val LIB_SOCIAL_LINK = "lib.social" - } -} \ No newline at end of file + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/FaviconParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/FaviconParser.kt index fec7f706a..cb8a6a3d7 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/FaviconParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/FaviconParser.kt @@ -1,94 +1,92 @@ package org.koitharu.kotatsu.parsers.util -import okhttp3.Headers import org.jsoup.nodes.Element -import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.model.Favicon import org.koitharu.kotatsu.parsers.model.Favicons +import org.koitharu.kotatsu.parsers.network.WebClient import org.koitharu.kotatsu.parsers.util.json.mapJSON class FaviconParser( - private val context: MangaLoaderContext, - private val domain: String, - private val headers: Headers?, + private val webClient: WebClient, + private val domain: String, ) { - suspend fun parseFavicons(): Favicons { - val url = "https://$domain" - val doc = context.httpGet(url, headers).parseHtml() - val result = HashSet() - val manifestLink = doc.getElementsByAttributeValue("rel", "manifest").firstOrNull() - ?.attrAsAbsoluteUrlOrNull("href") - if (manifestLink != null) { - result += parseManifest(manifestLink) - } - val links = doc.getElementsByAttributeValueContaining("rel", "icon") - links.mapNotNullTo(result) { link -> - parseLink(link) - } - if (result.isEmpty()) { - result.add(createFallback()) - } - return Favicons(result, url) - } + suspend fun parseFavicons(): Favicons { + val url = "https://$domain" + val doc = webClient.httpGet(url).parseHtml() + val result = HashSet() + val manifestLink = doc.getElementsByAttributeValue("rel", "manifest").firstOrNull() + ?.attrAsAbsoluteUrlOrNull("href") + if (manifestLink != null) { + result += parseManifest(manifestLink) + } + val links = doc.getElementsByAttributeValueContaining("rel", "icon") + links.mapNotNullTo(result) { link -> + parseLink(link) + } + if (result.isEmpty()) { + result.add(createFallback()) + } + return Favicons(result, url) + } - private fun parseLink(link: Element): Favicon? { - val href = link.attrAsAbsoluteUrlOrNull("href") - if (href == null || href.endsWith('/')) { - return null - } - val sizes = link.attr("sizes") - return Favicon( - url = href, - size = parseSize(sizes), - rel = link.attrOrNull("rel"), - ) - } + private fun parseLink(link: Element): Favicon? { + val href = link.attrAsAbsoluteUrlOrNull("href") + if (href == null || href.endsWith('/')) { + return null + } + val sizes = link.attr("sizes") + return Favicon( + url = href, + size = parseSize(sizes), + rel = link.attrOrNull("rel"), + ) + } - private fun parseSize(sizes: String): Int { - if (sizes.isEmpty() || sizes == "any") { - return 0 - } - return sizes.substringBefore(' ') - .split('x', 'X', '*') - .firstNotNullOfOrNull { it.toIntOrNull() } - ?: 0 - } + private fun parseSize(sizes: String): Int { + if (sizes.isEmpty() || sizes == "any") { + return 0 + } + return sizes.substringBefore(' ') + .split('x', 'X', '*') + .firstNotNullOfOrNull { it.toIntOrNull() } + ?: 0 + } - private suspend fun parseManifest(url: String): List { - val json = context.httpGet(url, headers).parseJson() - val icons = json.getJSONArray("icons") - return icons.mapJSON { jo -> - Favicon( - url = jo.getString("src").resolveLink(), - size = parseSize(jo.getString("sizes")), - rel = null, - ) - } - } + private suspend fun parseManifest(url: String): List { + val json = webClient.httpGet(url).parseJson() + val icons = json.getJSONArray("icons") + return icons.mapJSON { jo -> + Favicon( + url = jo.getString("src").resolveLink(), + size = parseSize(jo.getString("sizes")), + rel = null, + ) + } + } - private fun createFallback(): Favicon { - val href = "https://$domain/favicon.ico" - return Favicon( - url = href, - size = 0, - rel = null, - ) - } + private fun createFallback(): Favicon { + val href = "https://$domain/favicon.ico" + return Favicon( + url = href, + size = 0, + rel = null, + ) + } - private fun String.resolveLink(): String { - return when { - startsWith("http:") || startsWith("https:") -> { - this - } + private fun String.resolveLink(): String { + return when { + startsWith("http:") || startsWith("https:") -> { + this + } - startsWith('/') -> { - "https://$domain$this" - } + startsWith('/') -> { + "https://$domain$this" + } - else -> { - "https://$domain/$this" - } - } - } -} \ No newline at end of file + else -> { + "https://$domain/$this" + } + } + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/MangaParserEnv.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/MangaParserEnv.kt new file mode 100644 index 000000000..1e20c9e0c --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/MangaParserEnv.kt @@ -0,0 +1,78 @@ +package org.koitharu.kotatsu.parsers.util + +import okhttp3.HttpUrl +import org.jsoup.nodes.Element +import org.koitharu.kotatsu.parsers.InternalParsersApi +import org.koitharu.kotatsu.parsers.MangaParser +import org.koitharu.kotatsu.parsers.exception.ParseException +import org.koitharu.kotatsu.parsers.model.Manga +import org.koitharu.kotatsu.parsers.model.MangaChapter +import org.koitharu.kotatsu.parsers.model.MangaPage +import org.koitharu.kotatsu.parsers.model.MangaTag + + +/** + * Create a unique id for [Manga]/[MangaChapter]/[MangaPage]. + * @param url must be relative url, without a domain + * @see [Manga.id] + * @see [MangaChapter.id] + * @see [MangaPage.id] + */ +@InternalParsersApi +fun MangaParser.generateUid(url: String): Long { + var h = 1125899906842597L + source.name.forEach { c -> + h = 31 * h + c.code + } + url.forEach { c -> + h = 31 * h + c.code + } + return h +} + +/** + * Create a unique id for [Manga]/[MangaChapter]/[MangaPage]. + * @param id an internal identifier + * @see [Manga.id] + * @see [MangaChapter.id] + * @see [MangaPage.id] + */ +@InternalParsersApi +fun MangaParser.generateUid(id: Long): Long { + var h = 1125899906842597L + source.name.forEach { c -> + h = 31 * h + c.code + } + h = 31 * h + id + return h +} + +@InternalParsersApi +fun Element.parseFailed(message: String? = null): Nothing { + throw ParseException(message, ownerDocument()?.location() ?: baseUri(), null) +} + +@InternalParsersApi +fun Set?.oneOrThrowIfMany(): MangaTag? { + return when { + isNullOrEmpty() -> null + size == 1 -> first() + else -> throw IllegalArgumentException("Multiple genres are not supported by this source") + } +} + +val MangaParser.domain: String + get() { + return config[configKeyDomain] + } + +fun MangaParser.getDomain(subdomain: String): String { + val domain = domain + return subdomain + "." + domain.removePrefix("www.") +} + +fun MangaParser.urlBuilder(): HttpUrl.Builder { + return HttpUrl.Builder() + .scheme("https") + .host(domain) +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/OkHttp.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/OkHttp.kt index 08c5d3c08..df363e703 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/OkHttp.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/OkHttp.kt @@ -4,16 +4,26 @@ package org.koitharu.kotatsu.parsers.util import kotlinx.coroutines.suspendCancellableCoroutine import okhttp3.Call +import okhttp3.Headers import okhttp3.Response suspend fun Call.await(): Response = suspendCancellableCoroutine { continuation -> - val callback = ContinuationCallCallback(this, continuation) - enqueue(callback) - continuation.invokeOnCancellation(callback) + val callback = ContinuationCallCallback(this, continuation) + enqueue(callback) + continuation.invokeOnCancellation(callback) } val Response.mimeType: String? - get() = body?.contentType()?.run { "$type/$subtype" } + get() = body?.contentType()?.run { "$type/$subtype" } val Response.contentDisposition: String? - get() = header("Content-Disposition") \ No newline at end of file + get() = header("Content-Disposition") + +fun Headers.Builder.mergeWith(other: Headers, replaceExisting: Boolean): Headers.Builder { + for ((name, value) in other) { + if (replaceExisting || this[name] == null) { + this[name] = value + } + } + return this +} diff --git a/src/test/kotlin/org/koitharu/kotatsu/parsers/AuthCheckExtension.kt b/src/test/kotlin/org/koitharu/kotatsu/parsers/AuthCheckExtension.kt index 0dfbe83f4..05ab3c6f7 100644 --- a/src/test/kotlin/org/koitharu/kotatsu/parsers/AuthCheckExtension.kt +++ b/src/test/kotlin/org/koitharu/kotatsu/parsers/AuthCheckExtension.kt @@ -8,27 +8,27 @@ import org.koitharu.kotatsu.parsers.util.runCatchingCancellable class AuthCheckExtension : BeforeAllCallback { - private val loaderContext: MangaLoaderContext = MangaLoaderContextMock() + private val loaderContext: MangaLoaderContext = MangaLoaderContextMock - override fun beforeAll(context: ExtensionContext) { - for (source in MangaSource.values()) { - if (source == MangaSource.LOCAL || source == MangaSource.DUMMY) { - continue - } - val parser = source.newParser(loaderContext) - if (parser is MangaParserAuthProvider) { - checkAuthorization(source, parser) - } - } - } + override fun beforeAll(context: ExtensionContext) { + for (source in MangaSource.values()) { + if (source == MangaSource.LOCAL || source == MangaSource.DUMMY) { + continue + } + val parser = source.newParser(loaderContext) + if (parser is MangaParserAuthProvider) { + checkAuthorization(source, parser) + } + } + } - private fun checkAuthorization(source: MangaSource, parser: MangaParserAuthProvider) = runTest { - runCatchingCancellable { - parser.getUsername() - }.onSuccess { username -> - println("Signed in to ${source.name} as $username") - }.onFailure { error -> - System.err.println("Auth failed for ${source.name}: ${error.javaClass.name}(${error.message})") - } - } -} \ No newline at end of file + private fun checkAuthorization(source: MangaSource, parser: MangaParserAuthProvider) = runTest { + runCatchingCancellable { + parser.getUsername() + }.onSuccess { username -> + println("Signed in to ${source.name} as $username") + }.onFailure { error -> + System.err.println("Auth failed for ${source.name}: ${error.javaClass.name}(${error.message})") + } + } +} diff --git a/src/test/kotlin/org/koitharu/kotatsu/parsers/CloudFlareInterceptor.kt b/src/test/kotlin/org/koitharu/kotatsu/parsers/CloudFlareInterceptor.kt index 957eb8895..8878779df 100644 --- a/src/test/kotlin/org/koitharu/kotatsu/parsers/CloudFlareInterceptor.kt +++ b/src/test/kotlin/org/koitharu/kotatsu/parsers/CloudFlareInterceptor.kt @@ -4,26 +4,25 @@ import okhttp3.Interceptor import okhttp3.Response import okhttp3.internal.closeQuietly import org.koitharu.kotatsu.parsers.exception.CloudFlareProtectedException -import java.net.HttpURLConnection.HTTP_FORBIDDEN -import java.net.HttpURLConnection.HTTP_UNAVAILABLE +import java.net.HttpURLConnection private const val HEADER_SERVER = "Server" private const val SERVER_CLOUDFLARE = "cloudflare" -class CloudFlareInterceptor : Interceptor { +internal class CloudFlareInterceptor : Interceptor { - override fun intercept(chain: Interceptor.Chain): Response { - val request = chain.request() - val response = chain.proceed(request) - if (response.code == HTTP_FORBIDDEN || response.code == HTTP_UNAVAILABLE) { - if (response.header(HEADER_SERVER)?.startsWith(SERVER_CLOUDFLARE) == true) { - response.closeQuietly() - throw CloudFlareProtectedException( - url = response.request.url.toString(), - headers = request.headers, - ) - } - } - return response - } -} \ No newline at end of file + override fun intercept(chain: Interceptor.Chain): Response { + val request = chain.request() + val response = chain.proceed(request) + if (response.code == HttpURLConnection.HTTP_FORBIDDEN || response.code == HttpURLConnection.HTTP_UNAVAILABLE) { + if (response.header(HEADER_SERVER)?.startsWith(SERVER_CLOUDFLARE) == true) { + response.closeQuietly() + throw CloudFlareProtectedException( + url = response.request.url.toString(), + headers = request.headers, + ) + } + } + return response + } +} diff --git a/src/test/kotlin/org/koitharu/kotatsu/parsers/CommonHeadersInterceptor.kt b/src/test/kotlin/org/koitharu/kotatsu/parsers/CommonHeadersInterceptor.kt new file mode 100644 index 000000000..732364816 --- /dev/null +++ b/src/test/kotlin/org/koitharu/kotatsu/parsers/CommonHeadersInterceptor.kt @@ -0,0 +1,47 @@ +package org.koitharu.kotatsu.parsers + +import okhttp3.Interceptor +import okhttp3.Request +import okhttp3.Response +import org.koitharu.kotatsu.parsers.model.MangaSource +import org.koitharu.kotatsu.parsers.util.domain +import org.koitharu.kotatsu.parsers.util.mergeWith + +private const val HEADER_USER_AGENT = "User-Agent" +private const val HEADER_REFERER = "Referer" + +internal class CommonHeadersInterceptor( + private val userAgent: String, +) : Interceptor { + + override fun intercept(chain: Interceptor.Chain): Response { + val request = chain.request() + val source = request.tag(MangaSource::class.java) + val parser = source?.newParser(MangaLoaderContextMock) + val sourceHeaders = parser?.headers + val headersBuilder = request.headers.newBuilder() + if (sourceHeaders != null) { + headersBuilder.mergeWith(sourceHeaders, replaceExisting = false) + } + if (headersBuilder[HEADER_USER_AGENT] == null) { + headersBuilder[HEADER_USER_AGENT] = userAgent + } + if (headersBuilder[HEADER_REFERER] == null && parser != null) { + headersBuilder[HEADER_REFERER] = "https://${parser.domain}/" + } + val newRequest = request.newBuilder().headers(headersBuilder.build()).build() + return if (parser is Interceptor) { + parser.intercept(ProxyChain(chain, newRequest)) + } else { + return chain.proceed(newRequest) + } + } + + private class ProxyChain( + private val delegate: Interceptor.Chain, + private val request: Request, + ) : Interceptor.Chain by delegate { + + override fun request(): Request = request + } +} diff --git a/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaLoaderContextMock.kt b/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaLoaderContextMock.kt index 95ebb59c3..2c80cb0f3 100644 --- a/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaLoaderContextMock.kt +++ b/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaLoaderContextMock.kt @@ -10,59 +10,59 @@ import org.koitharu.kotatsu.parsers.model.MangaSource import org.koitharu.kotatsu.parsers.util.await import java.util.concurrent.TimeUnit -internal class MangaLoaderContextMock : MangaLoaderContext() { +internal object MangaLoaderContextMock : MangaLoaderContext() { - private val userAgent = "Kotatsu/%s (Android %s; %s; %s %s; %s)".format( - /*BuildConfig.VERSION_NAME*/ "3.0", - /*Build.VERSION.RELEASE*/ "r", - /*Build.MODEL*/ "", - /*Build.BRAND*/ "", - /*Build.DEVICE*/ "", - /*Locale.getDefault().language*/ "en", - ) + private val userAgent = "Kotatsu/%s (Android %s; %s; %s %s; %s)".format( + /*BuildConfig.VERSION_NAME*/ "4.3", + /*Build.VERSION.RELEASE*/ "r", + /*Build.MODEL*/ "", + /*Build.BRAND*/ "", + /*Build.DEVICE*/ "", + /*Locale.getDefault().language*/ "en", + ) - override val cookieJar = InMemoryCookieJar() + override val cookieJar = InMemoryCookieJar() - override val httpClient: OkHttpClient = OkHttpClient.Builder() - .cookieJar(cookieJar) - .addInterceptor(UserAgentInterceptor(userAgent)) - .addInterceptor(CloudFlareInterceptor()) - .connectTimeout(20, TimeUnit.SECONDS) - .readTimeout(60, TimeUnit.SECONDS) - .writeTimeout(20, TimeUnit.SECONDS) - .build() + override val httpClient: OkHttpClient = OkHttpClient.Builder() + .cookieJar(cookieJar) + .addInterceptor(CommonHeadersInterceptor(userAgent)) + .addInterceptor(CloudFlareInterceptor()) + .connectTimeout(20, TimeUnit.SECONDS) + .readTimeout(60, TimeUnit.SECONDS) + .writeTimeout(20, TimeUnit.SECONDS) + .build() - init { - loadTestCookies() - } + init { + loadTestCookies() + } - override suspend fun evaluateJs(script: String): String? { - return QuackContext.create().use { - it.evaluate(script)?.toString() - } - } + override suspend fun evaluateJs(script: String): String? { + return QuackContext.create().use { + it.evaluate(script)?.toString() + } + } - override fun getConfig(source: MangaSource): MangaSourceConfig { - return SourceConfigMock() - } + override fun getConfig(source: MangaSource): MangaSourceConfig { + return SourceConfigMock() + } - suspend fun doRequest(url: String, referer: String? = null, extraHeaders: Headers? = null): Response { - val request = Request.Builder() - .get() - .url(url) - if (extraHeaders != null) { - request.headers(extraHeaders) - } - if (referer != null) { - request.header("Referer", referer) - } - return httpClient.newCall(request.build()).await() - } + suspend fun doRequest(url: String, referer: String? = null, extraHeaders: Headers? = null): Response { + val request = Request.Builder() + .get() + .url(url) + if (extraHeaders != null) { + request.headers(extraHeaders) + } + if (referer != null) { + request.header("Referer", referer) + } + return httpClient.newCall(request.build()).await() + } - private fun loadTestCookies() { - // https://addons.mozilla.org/ru/firefox/addon/cookies-txt/ - javaClass.getResourceAsStream("/cookies.txt")?.use { - cookieJar.loadFromStream(it) - } ?: println("No cookies loaded!") - } -} \ No newline at end of file + private fun loadTestCookies() { + // https://addons.mozilla.org/ru/firefox/addon/cookies-txt/ + javaClass.getResourceAsStream("/cookies.txt")?.use { + cookieJar.loadFromStream(it) + } ?: println("No cookies loaded!") + } +} diff --git a/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaParserTest.kt b/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaParserTest.kt index 4abd15066..ab4503691 100644 --- a/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaParserTest.kt +++ b/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaParserTest.kt @@ -8,6 +8,7 @@ import org.junit.jupiter.params.ParameterizedTest import org.koitharu.kotatsu.parsers.model.Manga import org.koitharu.kotatsu.parsers.model.MangaSource import org.koitharu.kotatsu.parsers.model.SortOrder +import org.koitharu.kotatsu.parsers.util.domain import org.koitharu.kotatsu.parsers.util.medianOrNull import org.koitharu.kotatsu.parsers.util.mimeType import org.koitharu.kotatsu.test_util.isDistinct @@ -19,184 +20,184 @@ import org.koitharu.kotatsu.test_util.maxDuplicates @ExtendWith(AuthCheckExtension::class) internal class MangaParserTest { - private val context = MangaLoaderContextMock() - - @ParameterizedTest(name = "{index}|list|{0}") - @MangaSources - fun list(source: MangaSource) = runTest { - val parser = source.newParser(context) - val list = parser.getList(20, sortOrder = SortOrder.POPULARITY, tags = null) - checkMangaList(list, "list") - assert(list.all { it.source == source }) - } - - @ParameterizedTest(name = "{index}|pagination|{0}") - @MangaSources - fun pagination(source: MangaSource) = runTest { - val parser = source.newParser(context) - val page1 = parser.getList(0, sortOrder = null, tags = null) - val page2 = parser.getList(page1.size, sortOrder = null, tags = null) - assert(page1.isNotEmpty()) { "Page 1 is empty" } - assert(page2.isNotEmpty()) { "Page 2 is empty" } - val intersection = page1.intersect(page2.toSet()) - assert(intersection.isEmpty()) { - "Pages are intersected by " + intersection.size - } - } - - @ParameterizedTest(name = "{index}|search|{0}") - @MangaSources - fun search(source: MangaSource) = runTest { - val parser = source.newParser(context) - val subject = parser.getList(20, sortOrder = SortOrder.POPULARITY, tags = null).minByOrNull { - it.title.length - } ?: error("No manga found") - val query = subject.title - check(query.isNotBlank()) { "Manga title '$query' is blank" } - val list = parser.getList(0, query) - assert(list.isNotEmpty()) { "Empty search results by \"$query\"" } - assert(list.singleOrNull { it.url == subject.url && it.id == subject.id } != null) { - "Single subject '${subject.title} (${subject.publicUrl})' not found in search results" - } - checkMangaList(list, "search('$query')") - assert(list.all { it.source == source }) - } - - @ParameterizedTest(name = "{index}|tags|{0}") - @MangaSources - fun tags(source: MangaSource) = runTest { - val parser = source.newParser(context) - val tags = parser.getTags() - assert(tags.isNotEmpty()) { "No tags found" } - val keys = tags.map { it.key } - assert(keys.isDistinct()) - assert("" !in keys) - val titles = tags.map { it.title } + private val context = MangaLoaderContextMock + + @ParameterizedTest(name = "{index}|list|{0}") + @MangaSources + fun list(source: MangaSource) = runTest { + val parser = source.newParser(context) + val list = parser.getList(20, sortOrder = SortOrder.POPULARITY, tags = null) + checkMangaList(list, "list") + assert(list.all { it.source == source }) + } + + @ParameterizedTest(name = "{index}|pagination|{0}") + @MangaSources + fun pagination(source: MangaSource) = runTest { + val parser = source.newParser(context) + val page1 = parser.getList(0, sortOrder = null, tags = null) + val page2 = parser.getList(page1.size, sortOrder = null, tags = null) + assert(page1.isNotEmpty()) { "Page 1 is empty" } + assert(page2.isNotEmpty()) { "Page 2 is empty" } + val intersection = page1.intersect(page2.toSet()) + assert(intersection.isEmpty()) { + "Pages are intersected by " + intersection.size + } + } + + @ParameterizedTest(name = "{index}|search|{0}") + @MangaSources + fun search(source: MangaSource) = runTest { + val parser = source.newParser(context) + val subject = parser.getList(20, sortOrder = SortOrder.POPULARITY, tags = null).minByOrNull { + it.title.length + } ?: error("No manga found") + val query = subject.title + check(query.isNotBlank()) { "Manga title '$query' is blank" } + val list = parser.getList(0, query) + assert(list.isNotEmpty()) { "Empty search results by \"$query\"" } + assert(list.singleOrNull { it.url == subject.url && it.id == subject.id } != null) { + "Single subject '${subject.title} (${subject.publicUrl})' not found in search results" + } + checkMangaList(list, "search('$query')") + assert(list.all { it.source == source }) + } + + @ParameterizedTest(name = "{index}|tags|{0}") + @MangaSources + fun tags(source: MangaSource) = runTest { + val parser = source.newParser(context) + val tags = parser.getTags() + assert(tags.isNotEmpty()) { "No tags found" } + val keys = tags.map { it.key } + assert(keys.isDistinct()) + assert("" !in keys) + val titles = tags.map { it.title } // assert(titles.isDistinct()) - assert("" !in titles) - assert(tags.all { it.source == source }) - - val tag = tags.last() - val list = parser.getList(offset = 0, tags = setOf(tag), sortOrder = null) - checkMangaList(list, "${tag.title} (${tag.key})") - assert(list.all { it.source == source }) - } - - @ParameterizedTest(name = "{index}|details|{0}") - @MangaSources - fun details(source: MangaSource) = runTest { - val parser = source.newParser(context) - val list = parser.getList(20, sortOrder = SortOrder.POPULARITY, tags = null) - val manga = list[3] - parser.getDetails(manga).apply { - assert(!chapters.isNullOrEmpty()) { "Chapters are null or empty" } - assert(publicUrl.isUrlAbsolute()) { "Manga public url is not absolute: '$publicUrl'" } - assert(description != null) { "Detailed description is null: '$publicUrl'" } - assert(title.startsWith(manga.title)) { - "Titles are mismatch: '$title' and '${manga.title}' for $publicUrl" - } - assert(this.source == source) - val c = checkNotNull(chapters) - assert(c.isDistinctBy { it.id }) { - "Chapters are not distinct by id: ${c.maxDuplicates { it.id }} for $publicUrl" - } - assert(c.isDistinctBy { it.number to it.branch }) { - "Chapters are not distinct by number: ${c.maxDuplicates { it.number to it.branch }} for $publicUrl" - } - assert(c.all { it.source == source }) - checkImageRequest(coverUrl, publicUrl) - largeCoverUrl?.let { - checkImageRequest(it, publicUrl) - } - } - } - - @ParameterizedTest(name = "{index}|pages|{0}") - @MangaSources - fun pages(source: MangaSource) = runTest { - val parser = source.newParser(context) - val list = parser.getList(20, sortOrder = SortOrder.POPULARITY, tags = null) - val manga = list.first() - val chapter = parser.getDetails(manga).chapters?.firstOrNull() ?: error("Chapter is null") - val pages = parser.getPages(chapter) - - assert(pages.isNotEmpty()) - assert(pages.isDistinctBy { it.id }) - assert(pages.all { it.source == source }) - - val page = pages.medianOrNull() ?: error("No page") - val pageUrl = parser.getPageUrl(page) - assert(pageUrl.isNotEmpty()) - assert(pageUrl.isUrlAbsolute()) - checkImageRequest(pageUrl, page.referer) - } - - @ParameterizedTest(name = "{index}|favicon|{0}") - @MangaSources - fun favicon(source: MangaSource) = runTest { - val parser = source.newParser(context) - val favicons = parser.getFavicons() - val types = setOf("png", "svg", "ico", "gif", "jpg", "jpeg") - assert(favicons.isNotEmpty()) - favicons.forEach { - assert(it.url.isUrlAbsolute()) { "Favicon url is not absolute: ${it.url}" } - assert(it.type in types) { "Unknown icon type: ${it.type}" } - } - val favicon = favicons.find(24) - checkNotNull(favicon) - checkImageRequest(favicon.url, favicons.referer) - } - - @ParameterizedTest(name = "{index}|domain|{0}") - @MangaSources - fun domain(source: MangaSource) = runTest { - val parser = source.newParser(context) - val defaultDomain = parser.getDomain() - val url = HttpUrl.Builder() - .host(defaultDomain) - .scheme("https") - .toString() - val response = context.doRequest(url, extraHeaders = parser.headers) - val realUrl = response.request.url - val realDomain = realUrl.topPrivateDomain() - val realHost = realUrl.host - assert(defaultDomain == realHost || defaultDomain == realDomain) { - "Domain mismatch:\nRequired:\t\t\t$defaultDomain\nActual:\t\t\t$realDomain\nHost:\t\t\t$realHost" - } - } - - @ParameterizedTest(name = "{index}|authorization|{0}") - @MangaSources - @Disabled - fun authorization(source: MangaSource) = runTest { - val parser = source.newParser(context) - if (parser is MangaParserAuthProvider) { - val username = parser.getUsername() - assert(username.isNotBlank()) { "Username is blank" } - println("Signed in to ${source.name} as $username") - } - } - - private suspend fun checkMangaList(list: List, cause: String) { - assert(list.isNotEmpty()) { "Manga list for '$cause' is empty" } - assert(list.isDistinctBy { it.id }) { "Manga list for '$cause' contains duplicated ids" } - for (item in list) { - assert(item.url.isNotEmpty()) { "Url is empty" } - assert(!item.url.isUrlAbsolute()) { "Url looks like absolute: ${item.url}" } - assert(item.coverUrl.isUrlAbsolute()) { "Cover url is not absolute: ${item.coverUrl}" } - assert(item.title.isNotEmpty()) { "Title for ${item.publicUrl} is empty" } - assert(item.publicUrl.isUrlAbsolute()) - } - val testItem = list.random() - checkImageRequest(testItem.coverUrl, testItem.publicUrl) - } - - private suspend fun checkImageRequest(url: String, referer: String?) { - context.doRequest(url, referer).use { - assert(it.isSuccessful) { "Request failed: ${it.code}(${it.message}): $url" } - assert(it.mimeType?.startsWith("image/") == true) { - "Wrong response mime type: ${it.mimeType}" - } - } - } -} \ No newline at end of file + assert("" !in titles) + assert(tags.all { it.source == source }) + + val tag = tags.last() + val list = parser.getList(offset = 0, tags = setOf(tag), sortOrder = null) + checkMangaList(list, "${tag.title} (${tag.key})") + assert(list.all { it.source == source }) + } + + @ParameterizedTest(name = "{index}|details|{0}") + @MangaSources + fun details(source: MangaSource) = runTest { + val parser = source.newParser(context) + val list = parser.getList(20, sortOrder = SortOrder.POPULARITY, tags = null) + val manga = list[3] + parser.getDetails(manga).apply { + assert(!chapters.isNullOrEmpty()) { "Chapters are null or empty" } + assert(publicUrl.isUrlAbsolute()) { "Manga public url is not absolute: '$publicUrl'" } + assert(description != null) { "Detailed description is null: '$publicUrl'" } + assert(title.startsWith(manga.title)) { + "Titles are mismatch: '$title' and '${manga.title}' for $publicUrl" + } + assert(this.source == source) + val c = checkNotNull(chapters) + assert(c.isDistinctBy { it.id }) { + "Chapters are not distinct by id: ${c.maxDuplicates { it.id }} for $publicUrl" + } + assert(c.isDistinctBy { it.number to it.branch }) { + "Chapters are not distinct by number: ${c.maxDuplicates { it.number to it.branch }} for $publicUrl" + } + assert(c.all { it.source == source }) + checkImageRequest(coverUrl, publicUrl) + largeCoverUrl?.let { + checkImageRequest(it, publicUrl) + } + } + } + + @ParameterizedTest(name = "{index}|pages|{0}") + @MangaSources + fun pages(source: MangaSource) = runTest { + val parser = source.newParser(context) + val list = parser.getList(20, sortOrder = SortOrder.POPULARITY, tags = null) + val manga = list.first() + val chapter = parser.getDetails(manga).chapters?.firstOrNull() ?: error("Chapter is null") + val pages = parser.getPages(chapter) + + assert(pages.isNotEmpty()) + assert(pages.isDistinctBy { it.id }) + assert(pages.all { it.source == source }) + + val page = pages.medianOrNull() ?: error("No page") + val pageUrl = parser.getPageUrl(page) + assert(pageUrl.isNotEmpty()) + assert(pageUrl.isUrlAbsolute()) + checkImageRequest(pageUrl, page.referer) + } + + @ParameterizedTest(name = "{index}|favicon|{0}") + @MangaSources + fun favicon(source: MangaSource) = runTest { + val parser = source.newParser(context) + val favicons = parser.getFavicons() + val types = setOf("png", "svg", "ico", "gif", "jpg", "jpeg") + assert(favicons.isNotEmpty()) + favicons.forEach { + assert(it.url.isUrlAbsolute()) { "Favicon url is not absolute: ${it.url}" } + assert(it.type in types) { "Unknown icon type: ${it.type}" } + } + val favicon = favicons.find(24) + checkNotNull(favicon) + checkImageRequest(favicon.url, favicons.referer) + } + + @ParameterizedTest(name = "{index}|domain|{0}") + @MangaSources + fun domain(source: MangaSource) = runTest { + val parser = source.newParser(context) + val defaultDomain = parser.domain + val url = HttpUrl.Builder() + .host(defaultDomain) + .scheme("https") + .toString() + val response = context.doRequest(url, extraHeaders = parser.headers) + val realUrl = response.request.url + val realDomain = realUrl.topPrivateDomain() + val realHost = realUrl.host + assert(defaultDomain == realHost || defaultDomain == realDomain) { + "Domain mismatch:\nRequired:\t\t\t$defaultDomain\nActual:\t\t\t$realDomain\nHost:\t\t\t$realHost" + } + } + + @ParameterizedTest(name = "{index}|authorization|{0}") + @MangaSources + @Disabled + fun authorization(source: MangaSource) = runTest { + val parser = source.newParser(context) + if (parser is MangaParserAuthProvider) { + val username = parser.getUsername() + assert(username.isNotBlank()) { "Username is blank" } + println("Signed in to ${source.name} as $username") + } + } + + private suspend fun checkMangaList(list: List, cause: String) { + assert(list.isNotEmpty()) { "Manga list for '$cause' is empty" } + assert(list.isDistinctBy { it.id }) { "Manga list for '$cause' contains duplicated ids" } + for (item in list) { + assert(item.url.isNotEmpty()) { "Url is empty" } + assert(!item.url.isUrlAbsolute()) { "Url looks like absolute: ${item.url}" } + assert(item.coverUrl.isUrlAbsolute()) { "Cover url is not absolute: ${item.coverUrl}" } + assert(item.title.isNotEmpty()) { "Title for ${item.publicUrl} is empty" } + assert(item.publicUrl.isUrlAbsolute()) + } + val testItem = list.random() + checkImageRequest(testItem.coverUrl, testItem.publicUrl) + } + + private suspend fun checkImageRequest(url: String, referer: String?) { + context.doRequest(url, referer).use { + assert(it.isSuccessful) { "Request failed: ${it.code}(${it.message}): $url" } + assert(it.mimeType?.startsWith("image/") == true) { + "Wrong response mime type: ${it.mimeType}" + } + } + } +} diff --git a/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaSources.kt b/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaSources.kt index 5795da966..ef660faa0 100644 --- a/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaSources.kt +++ b/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaSources.kt @@ -4,4 +4,4 @@ import org.junit.jupiter.params.provider.EnumSource import org.koitharu.kotatsu.parsers.model.MangaSource @EnumSource(MangaSource::class, names = ["LOCAL", "DUMMY"], mode = EnumSource.Mode.EXCLUDE) -internal annotation class MangaSources \ No newline at end of file +internal annotation class MangaSources diff --git a/src/test/kotlin/org/koitharu/kotatsu/parsers/UserAgentInterceptor.kt b/src/test/kotlin/org/koitharu/kotatsu/parsers/UserAgentInterceptor.kt deleted file mode 100644 index 103201604..000000000 --- a/src/test/kotlin/org/koitharu/kotatsu/parsers/UserAgentInterceptor.kt +++ /dev/null @@ -1,20 +0,0 @@ -package org.koitharu.kotatsu.parsers - -import okhttp3.Interceptor -import okhttp3.Response - -private const val HEADER_USER_AGENT = "User-Agent" -internal class UserAgentInterceptor( - private val userAgent: String, -) : Interceptor { - - override fun intercept(chain: Interceptor.Chain): Response { - val request = chain.request() - val newRequest = if (request.header(HEADER_USER_AGENT) == null) { - request.newBuilder().header(HEADER_USER_AGENT, userAgent).build() - } else { - request - } - return chain.proceed(newRequest) - } -} \ No newline at end of file From 05d705ac0384a3fb08579e4bc91c728469a05b58 Mon Sep 17 00:00:00 2001 From: Koitharu Date: Wed, 8 Feb 2023 11:08:57 +0200 Subject: [PATCH 2/2] Remove deprecated "getFavicon" method --- CONTRIBUTING.md | 6 ++--- .../koitharu/kotatsu/parsers/MangaParser.kt | 9 -------- .../kotatsu/parsers/site/AnibelParser.kt | 4 ---- .../kotatsu/parsers/site/BatoToParser.kt | 2 -- .../kotatsu/parsers/site/CloneMangaParser.kt | 4 ---- .../kotatsu/parsers/site/NudeMoonParser.kt | 4 ---- .../parsers/site/grouple/MintMangaParser.kt | 3 --- .../parsers/site/grouple/ReadmangaParser.kt | 3 --- .../parsers/site/grouple/SelfMangaParser.kt | 3 --- .../parsers/site/madara/Madara5Parser.kt | 1 - .../parsers/site/madara/MadaraParser.kt | 23 +------------------ .../parsers/site/madara/MangalinkParser.kt | 2 -- .../site/madara/MangasOriginesParser.kt | 5 ---- .../parsers/site/madara/PrismaScansParser.kt | 5 ---- .../parsers/site/madara/ReaperScansParser.kt | 2 -- .../parsers/site/madara/TatakaeScansParser.kt | 5 ---- 16 files changed, 4 insertions(+), 77 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3b38769b8..ab56094df 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -59,13 +59,13 @@ All functions in `MangaParser` class are documented. Pay attention to some pecul - `sortOrders` set should not be empty. If your source is not support sorting, specify one most relevance value. - If you cannot obtain direct links to pages images inside `getPages` method, it is ok to use an intermediate url as `Page.url` and fetch a direct link at `getPageUrl` function. -- `getFaviconUrl` function is deprecated, but Kotatsu application before v4 still use it, so it is recommended to - override it too. - You can use _asserts_ to check some optional fields. For example. `Manga.author` field is not required, but if your source provide such information, add `assert(it != null)`. This will not have any effect on production but help to find issues during unit testing. - If your source website (or it's api) uses pages for pagination instead of offset you should extend `PagedMangaParser` instead of `MangaParser`. +- Your parser may also implement the `Interceptor` interface for additional manipulation of all network requests and/or + responses, including image loading. ## Development process @@ -87,4 +87,4 @@ It is recommended to run unit tests before submitting a PR. ## Help -If you need a help or have some questions, ask a community in our [Discord server](https://discord.gg/NNJ5RgVBC5). \ No newline at end of file +If you need a help or have some questions, ask a community in our [Discord server](https://discord.gg/NNJ5RgVBC5). diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaParser.kt index 435976f16..a99d6d64c 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaParser.kt @@ -112,15 +112,6 @@ abstract class MangaParser @InternalParsersApi constructor( */ abstract suspend fun getTags(): Set - /** - * Returns direct link to the website favicon - */ - @Deprecated( - message = "Use parseFavicons() to get multiple favicons with different size", - replaceWith = ReplaceWith("parseFavicons()"), - ) - open fun getFaviconUrl() = "https://${domain}/favicon.ico" - /** * Parse favicons from the main page of the source`s website */ diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/AnibelParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/AnibelParser.kt index a01f96bee..dddfc47d6 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/AnibelParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/AnibelParser.kt @@ -26,10 +26,6 @@ internal class AnibelParser(context: MangaLoaderContext) : MangaParser(context, SortOrder.NEWEST, ) - override fun getFaviconUrl(): String { - return "https://cdn.${domain}/favicons/favicon.png" - } - override suspend fun getList( offset: Int, query: String?, diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BatoToParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BatoToParser.kt index f162a6804..afbcfbb23 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BatoToParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BatoToParser.kt @@ -158,8 +158,6 @@ internal class BatoToParser(context: MangaLoaderContext) : PagedMangaParser( throw ParseException("Cannot find gernes list", scripts[0].baseUri()) } - override fun getFaviconUrl(): String = "https://styles.amarkcdn.com/img/batoto/favicon.ico?v0" - private suspend fun search(page: Int, query: String): List { val url = buildString { append("https://") diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/CloneMangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/CloneMangaParser.kt index 9bdea84f4..889aa81cf 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/CloneMangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/CloneMangaParser.kt @@ -21,10 +21,6 @@ internal class CloneMangaParser(context: MangaLoaderContext) : PagedMangaParser( override val configKeyDomain = ConfigKey.Domain("manga.clone-army.org", null) - override fun getFaviconUrl(): String { - return "https://pbs.twimg.com/profile_images/458758466346029056/Ys93EANp_400x400.png" - } - override suspend fun getListPage( page: Int, query: String?, diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NudeMoonParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NudeMoonParser.kt index 6341dcd72..823cd37b1 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NudeMoonParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NudeMoonParser.kt @@ -205,10 +205,6 @@ internal class NudeMoonParser( } } - override fun getFaviconUrl(): String { - return "https://${domain}/favicon.jpg" - } - private fun getSortKey(sortOrder: SortOrder) = when (sortOrder) { SortOrder.POPULARITY -> "views" diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/MintMangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/MintMangaParser.kt index ddbbdd3b1..3cc040af0 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/MintMangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/MintMangaParser.kt @@ -15,7 +15,4 @@ internal class MintMangaParser( arrayOf("mintmanga.live", "mintmanga.com"), ) - override fun getFaviconUrl(): String { - return "https://resmm.rmr.rocks/static/apple-touch-icon-8fff291039c140493adb0c7ba81065ad.png" - } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/ReadmangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/ReadmangaParser.kt index f4fe90d48..9ac27e9a5 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/ReadmangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/ReadmangaParser.kt @@ -15,7 +15,4 @@ internal class ReadmangaParser( arrayOf("readmanga.io", "readmanga.live", "readmanga.me"), ) - override fun getFaviconUrl(): String { - return "https://resrm.rmr.rocks/static/apple-touch-icon-3162037c9df9f28dca0f9a4092cb0f65.png" - } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/SelfMangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/SelfMangaParser.kt index 912958086..44fc703b9 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/SelfMangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/SelfMangaParser.kt @@ -12,7 +12,4 @@ internal class SelfMangaParser( override val configKeyDomain = ConfigKey.Domain("selfmanga.live", null) - override fun getFaviconUrl(): String { - return "https://ressm.rmr.rocks/static/apple-touch-icon-a769ea533d811b73ac3eedde658bb1d3.png" - } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Madara5Parser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Madara5Parser.kt index ae2e4110a..3cc261968 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Madara5Parser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Madara5Parser.kt @@ -182,6 +182,5 @@ abstract class Madara5Parser @InternalParsersApi constructor( @MangaSourceParser("MANGAOWLS", "BeautyManga", "en") class BeautyManga(context: MangaLoaderContext) : Madara5Parser(context, MangaSource.MANGAOWLS, "beautymanga.com") { - override fun getFaviconUrl() = "http://${domain}/frontend/images/favico.png" } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt index ad7a6b59e..5e8b95e9c 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt @@ -316,7 +316,7 @@ internal abstract class MadaraParser( override val datePattern = "dd MMMM HH:mm" } - @MangaSourceParser("HachiManga", "HACHIMANGA", "ja") + @MangaSourceParser("HACHIMANGA", "HachiManga", "ja") class HachiManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.HACHIMANGA, "hachimanga.com") { @@ -340,17 +340,11 @@ internal abstract class MadaraParser( @MangaSourceParser("AQUAMANGA", "AquaManga", "en") class AquaManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.AQUAMANGA, "aquamanga.com") { - override fun getFaviconUrl(): String { - return "https://$domain/wp-content/uploads/2021/03/cropped-cropped-favicon-1-32x32.png" - } } @MangaSourceParser("MANGALEK", "MangaLek", "ar") class MangaLek(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGALEK, "mangalek.com") { - override fun getFaviconUrl(): String { - return "https://$domain/wp-content/logo192.png" - } } @MangaSourceParser("HARIMANGA", "HariManga", "en") @@ -384,9 +378,6 @@ internal abstract class MadaraParser( @MangaSourceParser("MANHWAKOOL", "Manhwa Kool", "en") class ManhwaKool(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANHWAKOOL, "manhwakool.com") { override val datePattern: String = "MM/dd" - override fun getFaviconUrl(): String { - return "https://$domain/wp-content/uploads/2021/10/cropped-logo-kool-32x32.jpeg" - } } @MangaSourceParser("TOPMANHUA", "Top Manhua", "en") @@ -403,9 +394,6 @@ internal abstract class MadaraParser( override val isNsfwSource = true - override fun getFaviconUrl(): String { - return "https://$domain/wp-content/uploads/cropped-sky-tv-1-32x32.png" - } } @MangaSourceParser("BAKAMAN", "BakaMan", "th") @@ -461,9 +449,6 @@ internal abstract class MadaraParser( override val isNsfwSource = true - override fun getFaviconUrl(): String { - return "https://cdn.$domain/wp-content/uploads/2019/01/cropped-cropped-pcround-32x32.png" - } } @MangaSourceParser("CAT_300", "Cat300", "th") @@ -483,17 +468,11 @@ internal abstract class MadaraParser( override val datePattern = "MM/dd/yyyy" - override fun getFaviconUrl(): String { - return "https://$domain/wp-content/uploads/2017/10/lgoo-treemanga-2-1.jpg" - } } @MangaSourceParser("MANGACV", "Manga Cv", "en") class MangaCv(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGACV, "mangacv.com") { - override fun getFaviconUrl(): String { - return "https://$domain/wp-content/uploads/2020/10/cropped-mangaCV-32x32.png" - } } @MangaSourceParser("MANGA_MANHUA", "Manga Manhua", "en") diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangalinkParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangalinkParser.kt index ff78aab41..25404f35c 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangalinkParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangalinkParser.kt @@ -43,6 +43,4 @@ internal class MangalinkParser(context: MangaLoaderContext) : ) } - override fun getFaviconUrl(): String = - "https://cdn.${domain}/wp-content/uploads/2020/05/cropped-mangalink-180x180.jpg" } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangasOriginesParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangasOriginesParser.kt index af1c3801a..f2cf3cab1 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangasOriginesParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangasOriginesParser.kt @@ -8,7 +8,6 @@ import org.koitharu.kotatsu.parsers.model.MangaChapter import org.koitharu.kotatsu.parsers.model.MangaSource import org.koitharu.kotatsu.parsers.model.MangaState import org.koitharu.kotatsu.parsers.util.attrAsAbsoluteUrlOrNull -import org.koitharu.kotatsu.parsers.util.domain import org.koitharu.kotatsu.parsers.util.mapToSet import org.koitharu.kotatsu.parsers.util.selectFirstOrThrow import java.util.* @@ -19,10 +18,6 @@ internal class MangasOriginesParser(context: MangaLoaderContext) : override val tagPrefix = "catalogues-genre/" - override fun getFaviconUrl(): String { - return "https://${domain}/wp-content/uploads/2020/11/Mangas-150x150.png" - } - override fun parseDetails(manga: Manga, body: Element, chapters: List): Manga { val root = body.selectFirstOrThrow(".site-content") val postContent = root.selectFirstOrThrow(".post-content") diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/PrismaScansParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/PrismaScansParser.kt index b48770b08..cea3be896 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/PrismaScansParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/PrismaScansParser.kt @@ -8,7 +8,6 @@ import org.koitharu.kotatsu.parsers.model.MangaChapter import org.koitharu.kotatsu.parsers.model.MangaSource import org.koitharu.kotatsu.parsers.model.MangaState import org.koitharu.kotatsu.parsers.util.attrAsAbsoluteUrlOrNull -import org.koitharu.kotatsu.parsers.util.domain import org.koitharu.kotatsu.parsers.util.mapToSet import org.koitharu.kotatsu.parsers.util.selectFirstOrThrow import java.util.* @@ -20,10 +19,6 @@ internal class PrismaScansParser(context: MangaLoaderContext) : override val tagPrefix = "manga-genre/" override val datePattern = "MMM dd, yyyy" - override fun getFaviconUrl(): String { - return "https://${domain}/wp-content/uploads/2022/07/cropped-branca-1-192x192.png" - } - override fun parseDetails(manga: Manga, body: Element, chapters: List): Manga { val root = body.selectFirstOrThrow(".site-content") val postContent = root.selectFirstOrThrow(".post-content") diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/ReaperScansParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/ReaperScansParser.kt index b161995f5..d613607d1 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/ReaperScansParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/ReaperScansParser.kt @@ -83,6 +83,4 @@ internal class ReaperScansParser(context: MangaLoaderContext) : } } - override fun getFaviconUrl(): String = - "https://wuz.${domain}/wp-content/uploads/2022/03/20063242/cropped-ms-icon-310x310-2-192x192.png" } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/TatakaeScansParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/TatakaeScansParser.kt index 4a9a65cb8..661f33c65 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/TatakaeScansParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/TatakaeScansParser.kt @@ -8,7 +8,6 @@ import org.koitharu.kotatsu.parsers.model.MangaChapter import org.koitharu.kotatsu.parsers.model.MangaSource import org.koitharu.kotatsu.parsers.model.MangaState import org.koitharu.kotatsu.parsers.util.attrAsAbsoluteUrlOrNull -import org.koitharu.kotatsu.parsers.util.domain import org.koitharu.kotatsu.parsers.util.mapToSet import org.koitharu.kotatsu.parsers.util.selectFirstOrThrow import java.util.* @@ -48,8 +47,4 @@ internal class TatakaeScansParser(context: MangaLoaderContext) : else -> null } - override fun getFaviconUrl(): String { - return "https://${domain}/wp-content/uploads/2022/07/cropped-favicon-180x180.png" - } - }