diff --git a/.editorconfig b/.editorconfig index 7f2b16d0..a352878e 100644 --- a/.editorconfig +++ b/.editorconfig @@ -6,8 +6,8 @@ end_of_line = lf indent_style = tab max_line_length = 120 tab_width = 4 -insert_final_newline = false -disabled_rules=no-wildcard-imports,no-unused-imports +insert_final_newline = true +disabled_rules = no-wildcard-imports, no-unused-imports [{*.kt,*.kts}] ij_kotlin_allow_trailing_comma = true diff --git a/.github/workflows/test-parsers.yml b/.github/workflows/test-parsers.yml index 4479e758..817a3941 100644 --- a/.github/workflows/test-parsers.yml +++ b/.github/workflows/test-parsers.yml @@ -4,7 +4,7 @@ on: workflow_dispatch: pull_request: paths: - - 'src/main/kotlin/org/koitharu/kotatsu/parsers/site/*' + - 'src/main/kotlin/org/koitharu/kotatsu/parsers/site/**' permissions: contents: read @@ -24,4 +24,4 @@ jobs: - uses: actions/upload-artifact@v3 with: name: Report - path: build/test-results-html/TEST-org.koitharu.kotatsu.parsers.MangaParserTest.htm \ No newline at end of file + path: build/test-results-html/TEST-org.koitharu.kotatsu.parsers.MangaParserTest.htm diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..4ba46f33 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,91 @@ +# Contributing + +The following is guide for creating a Kotatsu parsers. Thanks for taking the time to contribute! + +## Prerequisites + +Before you start, please note that the ability to use following technologies is **required**. + +- Basic [Android development](https://developer.android.com/) +- [Kotlin](https://kotlinlang.org/) +- Web scraping ([JSoup](https://jsoup.org/)) or JSON API + +### Tools + +- [Android Studio](https://developer.android.com/studio) +- [IntelliJ IDEA](https://www.jetbrains.com/idea/) (Community edition is enough) +- Android device (or emulator) + +Kotatsu parsers is not a part of Android application, but you can easily develop and test it directly inside an Android +application project and relocate it to the library project when done. + +### Before you start + +First, take a look at `kotatsu-parsers` project structure. Each parser is a single class that +extends `MangaParser` class and have a `MangaSourceParser` annotation. +Also pay attention on extensions in `util` package. For example, extensions from `Jsoup` file +should be used instead of existing JSoup functions because they have better nullability support +and improved error messages. + +## Writing your parser + +So, you want to create a parser, that will provide access to manga from a website. +First, you should explore a website for API availability. +If it does not contain any documentation about +API, [explore network requests](https://firefox-source-docs.mozilla.org/devtools-user/): +some websites use ajax. + +- [Example](https://github.com/KotatsuApp/kotatsu-parsers/blob/master/src/main/kotlin/org/koitharu/kotatsu/parsers/site/DesuMeParser.kt) + of Json API usage. +- [Example](https://github.com/KotatsuApp/kotatsu-parsers/blob/master/src/main/kotlin/org/koitharu/kotatsu/parsers/site/AnibelParser.kt) + of GraphQL API usage +- [Example](https://github.com/KotatsuApp/kotatsu-parsers/blob/master/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaTownParser.kt) + of pure HTML parsing. + +If website is based on some engine it is rationally to use common base class for this one (for example, Madara wordress +theme +and the `MadaraParser` class) + +### Parser class skeleton + +Parser class must have exactly one primary constructor parameter of type `MangaLoaderContext` and have an +`MangaSourceParser` annotation that provides internal name, title and language of a manga source. + +All functions in `MangaParser` class are documented. Pay attention to some peculiarities: + +- Never hardcode domain. Specify default domain in `configKeyDomain` field and obtain an actual one using `getDomain()`. +- All ids must be unique and domain-independent. Use `generateUid` functions with relative url or some internal id which + is unique across the manga source. +- `sortOrders` set should not be empty. If your source is not support sorting, specify one most relevance value. +- If you cannot obtain direct links to pages images inside `getPages` method, it is ok to use an intermediate url + as `Page.url` and fetch a direct link at `getPageUrl` function. +- You can use _asserts_ to check some optional fields. For example. `Manga.author` field is not required, but if your + source provide such information, add `assert(it != null)`. This will not have any effect on production but help to + find issues during unit testing. +- If your source website (or it's api) uses pages for pagination instead of offset you should extend `PagedMangaParser` + instead of `MangaParser`. +- Your parser may also implement the `Interceptor` interface for additional manipulation of all network requests and/or + responses, including image loading. + +## Development process + +During the development it is recommended (but not necessary) to write it directly +in the Kotatsu android application project. You can use `core.parser.DummyParser` class as a sandbox. `Dummy` manga +source is available in debug Kotatsu build. + +Once parser is ready you can relocate your code into `kotatsu-parsers` library project in a `site` package and create a +Pull Request. + +### Testing + +It is recommended to run unit tests before submitting a PR. + +- Temporary modify the `MangaSources` annotation class: specify your parser(s) name(s) and change mode + to `EnumSource.Mode.INCLUDE` +- Run the `MangaParserTest` (`gradlew :test --tests "org.koitharu.kotatsu.parsers.MangaParserTest"`) +- Optionally, you can run the `generateTestsReport` gradle task to get a pretty readable html report from test results. + +## Help + +If you need a help or have some questions, ask a community in our [Telegram chat](https://t.me/kotatsuapp) +or [Discord server](https://discord.gg/NNJ5RgVBC5). diff --git a/README.md b/README.md index 504c9d0a..4612c1a9 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,12 @@ # Kotatsu parsers -Library that provides manga sources. +This library provides manga sources. -[![](https://jitpack.io/v/KotatsuApp/kotatsu-parsers.svg)](https://jitpack.io/#KotatsuApp/kotatsu-parsers) ![Kotlin](https://img.shields.io/github/languages/top/KotatsuApp/kotatsu-parsers) ![License](https://img.shields.io/github/license/KotatsuApp/Kotatsu) [![Discord](https://img.shields.io/discord/898363402467045416?color=5865f2&label=discord)](https://discord.gg/NNJ5RgVBC5) +[![](https://jitpack.io/v/KotatsuApp/kotatsu-parsers.svg)](https://jitpack.io/#KotatsuApp/kotatsu-parsers) ![Kotlin](https://img.shields.io/github/languages/top/KotatsuApp/kotatsu-parsers) ![License](https://img.shields.io/github/license/KotatsuApp/Kotatsu) [![Telegram](https://img.shields.io/badge/chat-telegram-60ACFF)](https://t.me/kotatsuapp) [![Discord](https://img.shields.io/discord/898363402467045416?color=5865f2&label=discord)](https://discord.gg/NNJ5RgVBC5) -### Usage +## Usage -1. Add it in your root build.gradle at the end of repositories: +1. Add it to your root build.gradle at the end of repositories: ```groovy allprojects { @@ -35,7 +35,7 @@ Library that provides manga sources. } ``` - See for versions at [JitPack](https://jitpack.io/#KotatsuApp/kotatsu-parsers) + Versions are available on [JitPack](https://jitpack.io/#KotatsuApp/kotatsu-parsers) 3. Usage in code @@ -44,8 +44,18 @@ Library that provides manga sources. ``` `mangaLoaderContext` is an implementation of the `MangaLoaderContext` class. - See [Android](https://github.com/KotatsuApp/Kotatsu/blob/devel/app/src/main/java/org/koitharu/kotatsu/core/parser/MangaLoaderContextImpl.kt) + See examples + of [Android](https://github.com/KotatsuApp/Kotatsu/blob/devel/app/src/main/java/org/koitharu/kotatsu/core/parser/MangaLoaderContextImpl.kt) and [Non-Android](https://github.com/KotatsuApp/kotatsu-dl/blob/master/src/main/kotlin/org/koitharu/kotatsu_dl/env/MangaLoaderContextImpl.kt) - implementation examples. + implementation. - Note that the `MangaSource.LOCAL` and `MangaSource.DUMMY` parsers cannot be instantiated. \ No newline at end of file + Note that the `MangaSource.LOCAL` and `MangaSource.DUMMY` parsers cannot be instantiated. + +## Contribution + +See [CONTRIBUTING.md](./CONTRIBUTING.md) for the guidelines. + +## DMCA disclaimer + +The developers of this application have no affiliation with the content available in the app. It is collected from +sources freely available through any web browser. diff --git a/build.gradle b/build.gradle index 9cae532e..cbf87048 100644 --- a/build.gradle +++ b/build.gradle @@ -57,7 +57,7 @@ dependencies { implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-core:1.6.4' implementation 'com.squareup.okhttp3:okhttp:4.10.0' implementation 'com.squareup.okio:okio:3.2.0' - api 'org.jsoup:jsoup:1.15.2' + api 'org.jsoup:jsoup:1.15.3' implementation 'org.json:json:20220320' implementation 'androidx.collection:collection-ktx:1.2.0' @@ -70,4 +70,5 @@ dependencies { testImplementation 'io.webfolder:quickjs:1.1.0' } +//noinspection ConfigurationAvoidance task generateTestsReport(type: ReportGenerateTask) \ No newline at end of file diff --git a/buildSrc/src/main/resources/report.html b/buildSrc/src/main/resources/report.html index 0a6c8dbf..b18a0dad 100644 --- a/buildSrc/src/main/resources/report.html +++ b/buildSrc/src/main/resources/report.html @@ -2,13 +2,15 @@ - + {{ testSuite.name }} - - + + + + @@ -25,67 +27,103 @@ {{ testSuite.errors }} ({{ error_percent }}%) -
- - - - - {% for test in tests %} - - {% endfor %} - - - {% for name, cases in results %} - - - {% for test in tests %} - {% set case = cases[test] %} - {% if case.failure == null %} - - {% else %} - {% if case.failure.type == 'java.lang.AssertionError' %} - - {% else %} - - {% endif %} - -
Source{{ test }}
{{ name }} - - - - - -
+ + + + {% for test in tests %} + + {% endfor %} + + + {% for name, cases in results %} + + + {% for test in tests %} + {% set case = cases[test] %} + {% if case.failure == null %} + + {% else %} + {% if case.failure.type == 'java.lang.AssertionError' %} + + {% elseif case.failure.type == 'java.net.SocketTimeoutException' or case.failure.type == + 'java.net.UnknownHostException' %} + + {% elseif case.failure.type == 'org.koitharu.kotatsu.parsers.exception.CloudFlareProtectedException' %} + + {% elseif case.failure.type == 'org.koitharu.kotatsu.parsers.exception.AuthRequiredException' %} + + {% else %} + + {% endif %} + + + + {% endif %} {% endfor %} -
Source{{ test }}
{{ name }} + + + + + + + + + + + +
-
+ + {% endfor %} + \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaLoaderContext.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaLoaderContext.kt index 04753783..1268be96 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaLoaderContext.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaLoaderContext.kt @@ -1,166 +1,29 @@ package org.koitharu.kotatsu.parsers -import okhttp3.* -import okhttp3.HttpUrl.Companion.toHttpUrl -import okhttp3.MediaType.Companion.toMediaType -import okhttp3.RequestBody.Companion.toRequestBody -import org.json.JSONObject -import org.jsoup.HttpStatusException +import okhttp3.CookieJar +import okhttp3.OkHttpClient import org.koitharu.kotatsu.parsers.config.MangaSourceConfig -import org.koitharu.kotatsu.parsers.exception.GraphQLException -import org.koitharu.kotatsu.parsers.exception.NotFoundException import org.koitharu.kotatsu.parsers.model.MangaSource -import org.koitharu.kotatsu.parsers.util.await -import org.koitharu.kotatsu.parsers.util.parseJson import java.util.* abstract class MangaLoaderContext { - protected abstract val httpClient: OkHttpClient + abstract val httpClient: OkHttpClient - abstract val cookieJar: CookieJar + abstract val cookieJar: CookieJar - /** - * Do a GET http request to specific url - * @param url - * @param headers an additional headers for request, may be null - */ - suspend fun httpGet(url: HttpUrl, headers: Headers? = null): Response { - val request = Request.Builder() - .get() - .url(url) - if (headers != null) { - request.headers(headers) - } - return httpClient.newCall(request.build()).await().ensureSuccess() - } + open fun encodeBase64(data: ByteArray): String = Base64.getEncoder().encodeToString(data) - suspend fun httpGet(url: String, headers: Headers? = null): Response { - return httpGet(url.toHttpUrl(), headers) - } + open fun decodeBase64(data: String): ByteArray = Base64.getDecoder().decode(data) + open fun getPreferredLocales(): List = listOf(Locale.getDefault()) - /** - * Do a HEAD http request to specific url - * @param url - * @param headers an additional headers for request, may be null - */ - suspend fun httpHead(url: String, headers: Headers? = null): Response { - val request = Request.Builder() - .head() - .url(url) - if (headers != null) { - request.headers(headers) - } - return httpClient.newCall(request.build()).await().ensureSuccess() - } + /** + * Execute JavaScript code and return result + * @param script JavaScript source code + * @return execution result as string, may be null + */ + abstract suspend fun evaluateJs(script: String): String? - /** - * Do a POST http request to specific url with `multipart/form-data` payload - * @param url - * @param form payload as key=>value map - * @param headers an additional headers for request, may be null - */ - suspend fun httpPost( - url: String, - form: Map, - headers: Headers? = null, - ): Response { - val body = FormBody.Builder() - form.forEach { (k, v) -> - body.addEncoded(k, v) - } - val request = Request.Builder() - .post(body.build()) - .url(url) - if (headers != null) { - request.headers(headers) - } - return httpClient.newCall(request.build()).await().ensureSuccess() - } - - /** - * Do a POST http request to specific url with `multipart/form-data` payload - * @param url - * @param payload payload as `key=value` string with `&` separator - * @param headers an additional headers for request, may be null - */ - suspend fun httpPost( - url: String, - payload: String, - headers: Headers?, - ): Response { - val body = FormBody.Builder() - payload.split('&').forEach { - val pos = it.indexOf('=') - if (pos != -1) { - val k = it.substring(0, pos) - val v = it.substring(pos + 1) - body.addEncoded(k, v) - } - } - val request = Request.Builder() - .post(body.build()) - .url(url) - if (headers != null) { - request.headers(headers) - } - return httpClient.newCall(request.build()).await().ensureSuccess() - } - - /** - * Do a GraphQL request to specific url - * @param endpoint an url - * @param query GraphQL request payload - */ - suspend fun graphQLQuery(endpoint: String, query: String): JSONObject { - val body = JSONObject() - body.put("operationName", null as Any?) - body.put("variables", JSONObject()) - body.put("query", "{$query}") - val mediaType = "application/json; charset=utf-8".toMediaType() - val requestBody = body.toString().toRequestBody(mediaType) - val request = Request.Builder() - .post(requestBody) - .url(endpoint) - val json = httpClient.newCall(request.build()).await().ensureSuccess().parseJson() - json.optJSONArray("errors")?.let { - if (it.length() != 0) { - throw GraphQLException(it) - } - } - return json - } - - open fun encodeBase64(data: ByteArray): String = Base64.getEncoder().encodeToString(data) - - open fun decodeBase64(data: String): ByteArray = Base64.getDecoder().decode(data) - - open fun getPreferredLocales(): List = listOf(Locale.getDefault()) - - /** - * Execute JavaScript code and return result - * @param script JavaScript source code - * @return execution result as string, may be null - */ - abstract suspend fun evaluateJs(script: String): String? - - abstract fun getConfig(source: MangaSource): MangaSourceConfig - - private fun Response.ensureSuccess(): Response { - val exception: Exception? = when (code) { // Catch some error codes, not all - 404 -> NotFoundException(message, request.url.toString()) - in 500..599 -> HttpStatusException(message, code, request.url.toString()) - else -> null - } - if (exception != null) { - runCatching { - close() - }.onFailure { - exception.addSuppressed(it) - } - throw exception - } - return this - } -} \ No newline at end of file + abstract fun getConfig(source: MangaSource): MangaSourceConfig +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaParser.kt index 2788529c..84ee386a 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaParser.kt @@ -1,20 +1,20 @@ package org.koitharu.kotatsu.parsers import androidx.annotation.CallSuper -import androidx.annotation.VisibleForTesting import okhttp3.Headers -import okhttp3.HttpUrl -import org.jsoup.nodes.Element import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.exception.ParseException import org.koitharu.kotatsu.parsers.model.* +import org.koitharu.kotatsu.parsers.network.OkHttpWebClient +import org.koitharu.kotatsu.parsers.network.WebClient import org.koitharu.kotatsu.parsers.util.FaviconParser +import org.koitharu.kotatsu.parsers.util.domain import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl import java.util.* -abstract class MangaParser @InternalParsersApi constructor(val source: MangaSource) { - - protected abstract val context: MangaLoaderContext +abstract class MangaParser @InternalParsersApi constructor( + @property:InternalParsersApi val context: MangaLoaderContext, + val source: MangaSource, +) { /** * Supported [SortOrder] variants. Must not be empty. @@ -25,18 +25,18 @@ abstract class MangaParser @InternalParsersApi constructor(val source: MangaSour val config by lazy { context.getConfig(source) } - val sourceLocale: Locale? - get() = source.locale?.let { Locale(it) } + open val sourceLocale: Locale + get() = source.locale?.let { Locale(it) } ?: Locale.ROOT /** * Provide default domain and available alternatives, if any. * * Never hardcode domain in requests, use [getDomain] instead. */ - protected abstract val configKeyDomain: ConfigKey.Domain + @InternalParsersApi + abstract val configKeyDomain: ConfigKey.Domain - @VisibleForTesting(otherwise = VisibleForTesting.PROTECTED) - internal open val headers: Headers? = null + open val headers: Headers? = null /** * Used as fallback if value of `sortOrder` passed to [getList] is null @@ -47,6 +47,8 @@ abstract class MangaParser @InternalParsersApi constructor(val source: MangaSour return SortOrder.values().first { it in supported } } + protected val webClient: WebClient = OkHttpWebClient(context.httpClient, source) + /** * Parse list of manga by specified criteria * @@ -103,98 +105,22 @@ abstract class MangaParser @InternalParsersApi constructor(val source: MangaSour /** * Fetch direct link to the page image. */ - open suspend fun getPageUrl(page: MangaPage): String = page.url.toAbsoluteUrl(getDomain()) + open suspend fun getPageUrl(page: MangaPage): String = page.url.toAbsoluteUrl(domain) /** * Fetch available tags (genres) for source */ abstract suspend fun getTags(): Set - /** - * Returns direct link to the website favicon - */ - @Deprecated( - message = "Use parseFavicons() to get multiple favicons with different size", - replaceWith = ReplaceWith("parseFavicons()"), - ) - open fun getFaviconUrl() = "https://${getDomain()}/favicon.ico" - /** * Parse favicons from the main page of the source`s website */ open suspend fun getFavicons(): Favicons { - return FaviconParser(context, getDomain(), headers).parseFavicons() + return FaviconParser(webClient, domain).parseFavicons() } @CallSuper open fun onCreateConfig(keys: MutableCollection>) { keys.add(configKeyDomain) } - - /* Utils */ - - fun getDomain(): String { - return config[configKeyDomain] - } - - fun getDomain(subdomain: String): String { - val domain = getDomain() - return subdomain + "." + domain.removePrefix("www.") - } - - fun urlBuilder(): HttpUrl.Builder { - return HttpUrl.Builder() - .scheme("https") - .host(getDomain()) - } - - /** - * Create a unique id for [Manga]/[MangaChapter]/[MangaPage]. - * @param url must be relative url, without a domain - * @see [Manga.id] - * @see [MangaChapter.id] - * @see [MangaPage.id] - */ - @InternalParsersApi - protected fun generateUid(url: String): Long { - var h = 1125899906842597L - source.name.forEach { c -> - h = 31 * h + c.code - } - url.forEach { c -> - h = 31 * h + c.code - } - return h - } - - /** - * Create a unique id for [Manga]/[MangaChapter]/[MangaPage]. - * @param id an internal identifier - * @see [Manga.id] - * @see [MangaChapter.id] - * @see [MangaPage.id] - */ - @InternalParsersApi - protected fun generateUid(id: Long): Long { - var h = 1125899906842597L - source.name.forEach { c -> - h = 31 * h + c.code - } - h = 31 * h + id - return h - } - - @InternalParsersApi - protected fun Element.parseFailed(message: String? = null): Nothing { - throw ParseException(message, ownerDocument()?.location() ?: baseUri(), null) - } - - @InternalParsersApi - protected fun Set?.oneOrThrowIfMany(): MangaTag? { - return when { - isNullOrEmpty() -> null - size == 1 -> first() - else -> throw IllegalArgumentException("Multiple genres are not supported by this source") - } - } -} \ No newline at end of file +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/PagedMangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/PagedMangaParser.kt index 7040bce4..ea18f848 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/PagedMangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/PagedMangaParser.kt @@ -8,43 +8,44 @@ import org.koitharu.kotatsu.parsers.util.Paginator @InternalParsersApi abstract class PagedMangaParser( - source: MangaSource, - pageSize: Int, - searchPageSize: Int = pageSize, -) : MangaParser(source) { - - protected val paginator = Paginator(pageSize) - protected val searchPaginator = Paginator(searchPageSize) - - override suspend fun getList(offset: Int, query: String): List { - return getList(searchPaginator, offset, query, null, defaultSortOrder) - } - - override suspend fun getList(offset: Int, tags: Set?, sortOrder: SortOrder?): List { - return getList(paginator, offset, null, tags, sortOrder ?: defaultSortOrder) - } - - @InternalParsersApi - @Deprecated("You should use getListPage for PagedMangaParser", level = DeprecationLevel.HIDDEN) - final override suspend fun getList( - offset: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List = throw UnsupportedOperationException("You should use getListPage for PagedMangaParser") - - abstract suspend fun getListPage(page: Int, query: String?, tags: Set?, sortOrder: SortOrder): List - - private suspend fun getList( - paginator: Paginator, - offset: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - val page = paginator.getPage(offset) - val list = getListPage(page, query, tags, sortOrder) - paginator.onListReceived(offset, page, list.size) - return list - } -} \ No newline at end of file + context: MangaLoaderContext, + source: MangaSource, + pageSize: Int, + searchPageSize: Int = pageSize, +) : MangaParser(context, source) { + + protected val paginator = Paginator(pageSize) + protected val searchPaginator = Paginator(searchPageSize) + + override suspend fun getList(offset: Int, query: String): List { + return getList(searchPaginator, offset, query, null, defaultSortOrder) + } + + override suspend fun getList(offset: Int, tags: Set?, sortOrder: SortOrder?): List { + return getList(paginator, offset, null, tags, sortOrder ?: defaultSortOrder) + } + + @InternalParsersApi + @Deprecated("You should use getListPage for PagedMangaParser", level = DeprecationLevel.HIDDEN) + final override suspend fun getList( + offset: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List = throw UnsupportedOperationException("You should use getListPage for PagedMangaParser") + + abstract suspend fun getListPage(page: Int, query: String?, tags: Set?, sortOrder: SortOrder): List + + private suspend fun getList( + paginator: Paginator, + offset: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + val page = paginator.getPage(offset) + val list = getListPage(page, query, tags, sortOrder) + paginator.onListReceived(offset, page, list.size) + return list + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/config/ConfigKey.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/config/ConfigKey.kt index c6c623d1..62b8107b 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/config/ConfigKey.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/config/ConfigKey.kt @@ -1,13 +1,21 @@ package org.koitharu.kotatsu.parsers.config sealed class ConfigKey( - val key: String, + @JvmField val key: String, ) { abstract val defaultValue: T class Domain( override val defaultValue: String, - val presetValues: Array?, + @JvmField val presetValues: Array?, ) : ConfigKey("domain") -} \ No newline at end of file + + class ShowSuspiciousContent( + override val defaultValue: Boolean, + ) : ConfigKey("show_suspicious") + + class UserAgent( + override val defaultValue: String, + ) : ConfigKey("user_agent") +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/exception/CloudFlareProtectedException.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/exception/CloudFlareProtectedException.kt index 75337a12..e06415e3 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/exception/CloudFlareProtectedException.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/exception/CloudFlareProtectedException.kt @@ -1,7 +1,9 @@ package org.koitharu.kotatsu.parsers.exception +import okhttp3.Headers import okio.IOException class CloudFlareProtectedException( val url: String, + val headers: Headers, ) : IOException("Protected by CloudFlare: $url") \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/model/Favicon.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/model/Favicon.kt index 7256524e..41e092ff 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/model/Favicon.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/model/Favicon.kt @@ -3,11 +3,12 @@ package org.koitharu.kotatsu.parsers.model import okhttp3.HttpUrl.Companion.toHttpUrl class Favicon internal constructor( - val url: String, - val size: Int, - internal val rel: String?, + @JvmField val url: String, + @JvmField val size: Int, + @JvmField internal val rel: String?, ) : Comparable { + @JvmField val type: String = url.toHttpUrl().pathSegments.last() .substringAfterLast('.', "").lowercase() @@ -48,4 +49,4 @@ class Favicon internal constructor( "mask-icon" -> -1 else -> 0 } -} \ No newline at end of file +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/model/Favicons.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/model/Favicons.kt index f9c9aab7..6c37fe4f 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/model/Favicons.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/model/Favicons.kt @@ -2,7 +2,7 @@ package org.koitharu.kotatsu.parsers.model class Favicons internal constructor( favicons: Collection, - val referer: String, + @JvmField val referer: String, ) : Collection { private val icons = favicons.sortedDescending() @@ -42,4 +42,4 @@ class Favicons internal constructor( } return result } -} \ No newline at end of file +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/model/Manga.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/model/Manga.kt index 9186be16..afda6dc8 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/model/Manga.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/model/Manga.kt @@ -6,67 +6,67 @@ class Manga( /** * Unique identifier for manga */ - val id: Long, + @JvmField val id: Long, /** * Manga title, human-readable */ - val title: String, + @JvmField val title: String, /** * Alternative title (for example on other language), may be null */ - val altTitle: String?, + @JvmField val altTitle: String?, /** * Relative url to manga (**without** a domain) or any other uri. * Used principally in parsers */ - val url: String, + @JvmField val url: String, /** * Absolute url to manga, must be ready to open in browser */ - val publicUrl: String, + @JvmField val publicUrl: String, /** * Normalized manga rating, must be in range of 0..1 or [RATING_UNKNOWN] if rating s unknown * @see hasRating */ - val rating: Float, + @JvmField val rating: Float, /** * Indicates that manga may contain sensitive information (18+, NSFW) */ - val isNsfw: Boolean, + @JvmField val isNsfw: Boolean, /** * Absolute link to the cover * @see largeCoverUrl */ - val coverUrl: String, + @JvmField val coverUrl: String, /** * Tags (genres) of the manga */ - val tags: Set, + @JvmField val tags: Set, /** * Manga status (ongoing, finished) or null if unknown */ - val state: MangaState?, + @JvmField val state: MangaState?, /** * Author of the manga, may be null */ - val author: String?, + @JvmField val author: String?, /** * Large cover url (absolute), null if is no large cover * @see coverUrl */ - val largeCoverUrl: String? = null, + @JvmField val largeCoverUrl: String? = null, /** * Manga description, may be html or null */ - val description: String? = null, + @JvmField val description: String? = null, /** * List of chapters */ - val chapters: List? = null, + @JvmField val chapters: List? = null, /** * Manga source */ - val source: MangaSource, + @JvmField val source: MangaSource, ) { /** @@ -155,4 +155,8 @@ class Manga( result = 31 * result + source.hashCode() return result } -} \ No newline at end of file + + override fun toString(): String { + return "Manga($id - \"$title\" [$url] - $source)" + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/model/MangaChapter.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/model/MangaChapter.kt index 52b01e6b..21e6c6e6 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/model/MangaChapter.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/model/MangaChapter.kt @@ -4,34 +4,34 @@ class MangaChapter( /** * An unique id of chapter */ - val id: Long, + @JvmField val id: Long, /** * User-readable name of chapter */ - val name: String, + @JvmField val name: String, /** * Chapter number starting from 1 */ - val number: Int, + @JvmField val number: Int, /** * Relative url to chapter (**without** a domain) or any other uri. * Used principally in parsers */ - val url: String, + @JvmField val url: String, /** * User-readable name of scanlator (releaser) or null if unknown */ - val scanlator: String?, + @JvmField val scanlator: String?, /** * Chapter upload date in milliseconds */ - val uploadDate: Long, + @JvmField val uploadDate: Long, /** * User-readable name of branch. * A branch is a group of chapters that overlap (e.g. different languages) */ - val branch: String?, - val source: MangaSource, + @JvmField val branch: String?, + @JvmField val source: MangaSource, ) : Comparable { override fun compareTo(other: MangaChapter): Int { @@ -67,4 +67,8 @@ class MangaChapter( result = 31 * result + source.hashCode() return result } -} \ No newline at end of file + + override fun toString(): String { + return "MangaChapter($id - #$number [$url] - $source)" + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/model/MangaPage.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/model/MangaPage.kt index 4cc66d29..2c463761 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/model/MangaPage.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/model/MangaPage.kt @@ -6,24 +6,19 @@ class MangaPage( /** * Unique identifier for manga */ - val id: Long, + @JvmField val id: Long, /** * Relative url to page (**without** a domain) or any other uri. * Used principally in parsers. * May contain link to image or html page. * @see MangaParser.getPageUrl */ - val url: String, - /** - * Absolute link to the chapter or website home page. - * Used in Referer header - */ - val referer: String, + @JvmField val url: String, /** * Absolute url of the small page image if exists, null otherwise */ - val preview: String?, - val source: MangaSource, + @JvmField val preview: String?, + @JvmField val source: MangaSource, ) { override fun equals(other: Any?): Boolean { @@ -34,19 +29,19 @@ class MangaPage( if (id != other.id) return false if (url != other.url) return false - if (referer != other.referer) return false if (preview != other.preview) return false - if (source != other.source) return false - - return true + return source == other.source } override fun hashCode(): Int { var result = id.hashCode() result = 31 * result + url.hashCode() - result = 31 * result + referer.hashCode() result = 31 * result + (preview?.hashCode() ?: 0) result = 31 * result + source.hashCode() return result } -} \ No newline at end of file + + override fun toString(): String { + return "MangaPage($id [$url] - $source)" + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/model/MangaTag.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/model/MangaTag.kt index 0d9fa04b..184a850e 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/model/MangaTag.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/model/MangaTag.kt @@ -6,13 +6,13 @@ class MangaTag( /** * User-readable tag title, should be in Title case */ - val title: String, + @JvmField val title: String, /** * Identifier of a tag, must be unique among the source. * @see MangaParser.getList */ - val key: String, - val source: MangaSource, + @JvmField val key: String, + @JvmField val source: MangaSource, ) { override fun equals(other: Any?): Boolean { @@ -34,4 +34,8 @@ class MangaTag( result = 31 * result + source.hashCode() return result } -} \ No newline at end of file + + override fun toString(): String { + return "MangaTag($key \"$title\" - $source)" + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/network/OkHttpWebClient.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/network/OkHttpWebClient.kt new file mode 100644 index 00000000..7b5dc961 --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/network/OkHttpWebClient.kt @@ -0,0 +1,118 @@ +package org.koitharu.kotatsu.parsers.network + +import okhttp3.* +import okhttp3.MediaType.Companion.toMediaType +import okhttp3.RequestBody.Companion.toRequestBody +import org.json.JSONObject +import org.jsoup.HttpStatusException +import org.koitharu.kotatsu.parsers.exception.AuthRequiredException +import org.koitharu.kotatsu.parsers.exception.GraphQLException +import org.koitharu.kotatsu.parsers.exception.NotFoundException +import org.koitharu.kotatsu.parsers.model.MangaSource +import org.koitharu.kotatsu.parsers.util.await +import org.koitharu.kotatsu.parsers.util.parseJson +import java.net.HttpURLConnection + +class OkHttpWebClient( + private val httpClient: OkHttpClient, + private val mangaSource: MangaSource, +) : WebClient { + + override suspend fun httpGet(url: HttpUrl, extraHeaders: Headers?): Response { + val request = Request.Builder() + .get() + .url(url) + .addTags() + if (extraHeaders != null) { + request.headers(extraHeaders) + } + return httpClient.newCall(request.build()).await().ensureSuccess() + } + + override suspend fun httpHead(url: HttpUrl): Response { + val request = Request.Builder() + .head() + .url(url) + .addTags() + return httpClient.newCall(request.build()).await().ensureSuccess() + } + + override suspend fun httpPost(url: HttpUrl, form: Map): Response { + val body = FormBody.Builder() + form.forEach { (k, v) -> + body.addEncoded(k, v) + } + val request = Request.Builder() + .post(body.build()) + .url(url) + .addTags() + return httpClient.newCall(request.build()).await().ensureSuccess() + } + + override suspend fun httpPost(url: HttpUrl, payload: String): Response { + val body = FormBody.Builder() + payload.split('&').forEach { + val pos = it.indexOf('=') + if (pos != -1) { + val k = it.substring(0, pos) + val v = it.substring(pos + 1) + body.addEncoded(k, v) + } + } + val request = Request.Builder() + .post(body.build()) + .url(url) + .addTags() + return httpClient.newCall(request.build()).await().ensureSuccess() + } + + override suspend fun httpPost(url: HttpUrl, body: JSONObject): Response { + val mediaType = "application/json; charset=utf-8".toMediaType() + val requestBody = body.toString().toRequestBody(mediaType) + val request = Request.Builder() + .post(requestBody) + .url(url) + .addTags() + return httpClient.newCall(request.build()).await().ensureSuccess() + } + + override suspend fun graphQLQuery(endpoint: String, query: String): JSONObject { + val body = JSONObject() + body.put("operationName", null as Any?) + body.put("variables", JSONObject()) + body.put("query", "{$query}") + val json = httpPost(endpoint, body).parseJson() + json.optJSONArray("errors")?.let { + if (it.length() != 0) { + throw GraphQLException(it) + } + } + return json + } + + private fun Request.Builder.addTags(): Request.Builder { + tag(MangaSource::class.java, mangaSource) + return this + } + + private fun Response.ensureSuccess(): Response { + val exception: Exception? = when (code) { // Catch some error codes, not all + HttpURLConnection.HTTP_NOT_FOUND -> NotFoundException(message, request.url.toString()) + HttpURLConnection.HTTP_UNAUTHORIZED -> request.tag(MangaSource::class.java)?.let { + AuthRequiredException(it) + } ?: HttpStatusException(message, code, request.url.toString()) + + in 400..599 -> HttpStatusException(message, code, request.url.toString()) + else -> null + } + if (exception != null) { + runCatching { + close() + }.onFailure { + exception.addSuppressed(it) + } + throw exception + } + return this + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/network/WebClient.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/network/WebClient.kt new file mode 100644 index 00000000..a486824d --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/network/WebClient.kt @@ -0,0 +1,87 @@ +package org.koitharu.kotatsu.parsers.network + +import okhttp3.Headers +import okhttp3.HttpUrl +import okhttp3.HttpUrl.Companion.toHttpUrl +import okhttp3.Response +import org.json.JSONObject + +interface WebClient { + + /** + * Do a GET http request to specific url + * @param url + */ + suspend fun httpGet(url: String): Response = httpGet(url.toHttpUrl()) + + suspend fun httpGet(url: String, extraHeaders: Headers?): Response = httpGet(url.toHttpUrl(), extraHeaders) + + /** + * Do a GET http request to specific url + * @param url + */ + suspend fun httpGet(url: HttpUrl): Response = httpGet(url, null) + + suspend fun httpGet(url: HttpUrl, extraHeaders: Headers?): Response + + /** + * Do a HEAD http request to specific url + * @param url + */ + suspend fun httpHead(url: String): Response = httpHead(url.toHttpUrl()) + + /** + * Do a HEAD http request to specific url + * @param url + */ + suspend fun httpHead(url: HttpUrl): Response + + /** + * Do a POST http request to specific url with `multipart/form-data` payload + * @param url + * @param form payload as key=>value map + */ + suspend fun httpPost(url: String, form: Map): Response = httpPost(url.toHttpUrl(), form) + + /** + * Do a POST http request to specific url with `multipart/form-data` payload + * @param url + * @param form payload as key=>value map + */ + suspend fun httpPost(url: HttpUrl, form: Map): Response + + /** + * Do a POST http request to specific url with `multipart/form-data` payload + * @param url + * @param payload payload as `key=value` string with `&` separator + */ + suspend fun httpPost(url: String, payload: String): Response = httpPost(url.toHttpUrl(), payload) + + /** + * Do a POST http request to specific url with `multipart/form-data` payload + * @param url + * @param payload payload as `key=value` string with `&` separator + */ + suspend fun httpPost(url: HttpUrl, payload: String): Response + + /** + * Do a POST http request to specific url with json payload + * @param url + * @param body + */ + suspend fun httpPost(url: String, body: JSONObject): Response = httpPost(url.toHttpUrl(), body) + + /** + * Do a POST http request to specific url with json payload + * @param url + * @param body + */ + suspend fun httpPost(url: HttpUrl, body: JSONObject): Response + + /** + * Do a GraphQL request to specific url + * @param endpoint an url + * @param query GraphQL request payload + */ + suspend fun graphQLQuery(endpoint: String, query: String): JSONObject +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/AnibelParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/AnibelParser.kt index 8436ee1d..a50fc5b5 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/AnibelParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/AnibelParser.kt @@ -8,6 +8,9 @@ import org.koitharu.kotatsu.parsers.MangaParser import org.koitharu.kotatsu.parsers.MangaSourceParser import org.koitharu.kotatsu.parsers.config.ConfigKey import org.koitharu.kotatsu.parsers.model.* +import org.koitharu.kotatsu.parsers.util.domain +import org.koitharu.kotatsu.parsers.util.generateUid +import org.koitharu.kotatsu.parsers.util.getDomain import org.koitharu.kotatsu.parsers.util.json.mapJSON import org.koitharu.kotatsu.parsers.util.json.mapJSONIndexed import org.koitharu.kotatsu.parsers.util.json.stringIterator @@ -15,7 +18,7 @@ import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl import java.util.* @MangaSourceParser("ANIBEL", "Anibel", "be") -internal class AnibelParser(override val context: MangaLoaderContext) : MangaParser(MangaSource.ANIBEL) { +internal class AnibelParser(context: MangaLoaderContext) : MangaParser(context, MangaSource.ANIBEL) { override val configKeyDomain = ConfigKey.Domain("anibel.net", null) @@ -23,10 +26,6 @@ internal class AnibelParser(override val context: MangaLoaderContext) : MangaPar SortOrder.NEWEST, ) - override fun getFaviconUrl(): String { - return "https://cdn.${getDomain()}/favicons/favicon.png" - } - override suspend fun getList( offset: Int, query: String?, @@ -78,7 +77,7 @@ internal class AnibelParser(override val context: MangaLoaderContext) : MangaPar isNsfw = false, rating = jo.getDouble("rating").toFloat() / 10f, url = href, - publicUrl = "https://${getDomain()}/$href", + publicUrl = "https://${domain}/$href", tags = jo.getJSONArray("genres").mapToTags(), state = when (jo.getString("status")) { "ongoing" -> MangaState.ONGOING @@ -165,12 +164,11 @@ internal class AnibelParser(override val context: MangaLoaderContext) : MangaPar """.trimIndent(), ).getJSONObject("chapter") val pages = chapterJson.getJSONArray("images") - val chapterUrl = "https://${getDomain()}/${chapter.url}" + val chapterUrl = "https://${domain}/${chapter.url}" return pages.mapJSONIndexed { i, jo -> MangaPage( id = generateUid("${chapter.url}/$i"), url = jo.getString("large"), - referer = chapterUrl, preview = jo.getString("thumbnail"), source = source, ) @@ -219,7 +217,7 @@ internal class AnibelParser(override val context: MangaLoaderContext) : MangaPar isNsfw = false, rating = RATING_UNKNOWN, url = href, - publicUrl = "https://${getDomain()}/$href", + publicUrl = "https://${domain}/$href", tags = emptySet(), state = null, source = source, @@ -228,7 +226,7 @@ internal class AnibelParser(override val context: MangaLoaderContext) : MangaPar } private suspend fun apiCall(request: String): JSONObject { - return context.graphQLQuery("https://api.${getDomain()}/graphql", request) + return webClient.graphQLQuery("https://api.${domain}/graphql", request) .getJSONObject("data") } @@ -242,6 +240,7 @@ internal class AnibelParser(override val context: MangaLoaderContext) : MangaPar c == '-' -> { builder.setCharAt(i, ' ') } + capitalize -> { builder.setCharAt(i, c.uppercaseChar()) capitalize = false @@ -263,4 +262,4 @@ internal class AnibelParser(override val context: MangaLoaderContext) : MangaPar } return result } -} \ No newline at end of file +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BatoToParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BatoToParser.kt index a366c1d1..e4e1f417 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BatoToParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BatoToParser.kt @@ -19,7 +19,8 @@ import javax.crypto.spec.IvParameterSpec import javax.crypto.spec.SecretKeySpec @MangaSourceParser("BATOTO", "Bato.To") -internal class BatoToParser(override val context: MangaLoaderContext) : PagedMangaParser( +internal class BatoToParser(context: MangaLoaderContext) : PagedMangaParser( + context = context, source = MangaSource.BATOTO, pageSize = 60, searchPageSize = 20, @@ -34,7 +35,16 @@ internal class BatoToParser(override val context: MangaLoaderContext) : PagedMan override val configKeyDomain = ConfigKey.Domain( "bato.to", - arrayOf("bato.to", "mto.to", "mangatoto.com", "battwo.com", "batotwo.com", "comiko.net", "batotoo.com"), + arrayOf( + "bato.to", + "mto.to", + "hto.to", + "mangatoto.com", + "battwo.com", + "batotwo.com", + "comiko.net", + "batotoo.com", + ), ) override suspend fun getListPage( @@ -49,7 +59,7 @@ internal class BatoToParser(override val context: MangaLoaderContext) : PagedMan @Suppress("NON_EXHAUSTIVE_WHEN_STATEMENT") val url = buildString { append("https://") - append(getDomain()) + append(domain) append("/browse?sort=") when (sortOrder) { SortOrder.UPDATED, @@ -70,7 +80,7 @@ internal class BatoToParser(override val context: MangaLoaderContext) : PagedMan } override suspend fun getDetails(manga: Manga): Manga { - val root = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml() + val root = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() .requireElementById("mainer") val details = root.selectFirstOrThrow(".detail-set") val attrs = details.selectFirst(".attr-main")?.select(".attr-item")?.associate { @@ -93,16 +103,15 @@ internal class BatoToParser(override val context: MangaLoaderContext) : PagedMan chapters = root.selectFirst(".episode-list") ?.selectFirst(".main") ?.children() - ?.reversed() - ?.mapChapters { i, div -> + ?.mapChapters(reversed = true) { i, div -> div.parseChapter(i) }.orEmpty(), ) } override suspend fun getPages(chapter: MangaChapter): List { - val fullUrl = chapter.url.toAbsoluteUrl(getDomain()) - val scripts = context.httpGet(fullUrl).parseHtml().select("script") + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val scripts = webClient.httpGet(fullUrl).parseHtml().select("script") for (script in scripts) { val scriptSrc = script.html() val p = scriptSrc.indexOf("const imgHttpLis =") @@ -126,7 +135,6 @@ internal class BatoToParser(override val context: MangaLoaderContext) : PagedMan result += MangaPage( id = generateUid(url), url = url + "?" + args.getString(i), - referer = fullUrl, preview = null, source = source, ) @@ -137,8 +145,8 @@ internal class BatoToParser(override val context: MangaLoaderContext) : PagedMan } override suspend fun getTags(): Set { - val scripts = context.httpGet( - "https://${getDomain()}/browse", + val scripts = webClient.httpGet( + "https://${domain}/browse", ).parseHtml().selectOrThrow("script") for (script in scripts) { val genres = script.html().substringBetweenFirst("const _genres =", ";") ?: continue @@ -157,12 +165,10 @@ internal class BatoToParser(override val context: MangaLoaderContext) : PagedMan throw ParseException("Cannot find gernes list", scripts[0].baseUri()) } - override fun getFaviconUrl(): String = "https://styles.amarkcdn.com/img/batoto/favicon.ico?v0" - private suspend fun search(page: Int, query: String): List { val url = buildString { append("https://") - append(getDomain()) + append(domain) append("/search?word=") append(query.replace(' ', '+')) append("&page=") @@ -177,7 +183,7 @@ internal class BatoToParser(override val context: MangaLoaderContext) : PagedMan ?.toIntOrNull() ?: body.parseFailed("Cannot determine current page") private suspend fun parseList(url: String, page: Int): List { - val body = context.httpGet(url).parseHtml().body() + val body = webClient.httpGet(url).parseHtml().body() if (body.selectFirst(".browse-no-matches") != null) { return emptyList() } @@ -307,4 +313,4 @@ internal class BatoToParser(override val context: MangaLoaderContext) : PagedMan } else byteArrayOf(), ) } -} \ No newline at end of file +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BentomangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BentomangaParser.kt new file mode 100644 index 00000000..f435c60c --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BentomangaParser.kt @@ -0,0 +1,240 @@ +package org.koitharu.kotatsu.parsers.site + +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.coroutineScope +import okhttp3.Headers +import org.jsoup.nodes.Element +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.PagedMangaParser +import org.koitharu.kotatsu.parsers.config.ConfigKey +import org.koitharu.kotatsu.parsers.exception.ParseException +import org.koitharu.kotatsu.parsers.model.* +import org.koitharu.kotatsu.parsers.util.* +import org.koitharu.kotatsu.parsers.util.json.getIntOrDefault +import java.util.* + +@MangaSourceParser("BENTOMANGA", "Bentomanga", "fr") +internal class BentomangaParser(context: MangaLoaderContext) : PagedMangaParser(context, MangaSource.BENTOMANGA, 10) { + + override val sortOrders: Set = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.RATING, + SortOrder.NEWEST, + SortOrder.ALPHABETICAL, + ) + + override val configKeyDomain = ConfigKey.Domain("www.bentomanga.com", null) + + override val headers: Headers = Headers.Builder() + .add("User-Agent", "Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/113.0") + .build() + + init { + paginator.firstPage = 0 + searchPaginator.firstPage = 0 + } + + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + val url = urlBuilder() + .addPathSegment("manga_list") + .addQueryParameter("limit", page.toString()) + .addQueryParameter( + "order_by", + when (sortOrder) { + SortOrder.UPDATED -> "update" + SortOrder.POPULARITY -> "views" + SortOrder.RATING -> "top" + SortOrder.NEWEST -> "create" + SortOrder.ALPHABETICAL -> "name" + }, + ) + if (!tags.isNullOrEmpty()) { + url.addQueryParameter("withCategories", tags.joinToString(",") { it.key }) + } + if (!query.isNullOrEmpty()) { + url.addQueryParameter("search", query) + } + val root = webClient.httpGet(url.build()).parseHtml().requireElementById("mangas_content") + return root.select(".manga[data-manga]").map { div -> + val header = div.selectFirstOrThrow(".manga_header") + val href = header.selectFirstOrThrow("a").attrAsRelativeUrl("href") + Manga( + id = generateUid(href), + title = div.selectFirstOrThrow("h1").text(), + altTitle = null, + url = href, + publicUrl = href.toAbsoluteUrl(domain), + rating = div.getElementsByAttributeValue("data-icon", "avg_rate") + .firstOrNull() + ?.textOrNull() + ?.toFloatOrNull() + ?.div(10f) + ?: RATING_UNKNOWN, + isNsfw = div.selectFirst(".badge-adult_content") != null, + coverUrl = div.selectFirstOrThrow("img").src(), + tags = div.selectFirst(".component-manga-categories") + .assertNotNull("tags") + ?.select("a") + ?.mapToSet { a -> + MangaTag( + title = a.text().toTitleCase(sourceLocale), + key = a.attr("href").substringAfterLast('='), + source = source, + ) + }.orEmpty(), + state = null, + author = null, + description = div.selectFirst(".manga_synopsis")?.html().assertNotNull("description"), + source = source, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val mangaUrl = manga.url.toAbsoluteUrl(domain) + val root = webClient.httpGet(mangaUrl).parseHtml() + .requireElementById("container_manga_show") + return manga.copy( + altTitle = root.selectFirst(".component-manga-title_alt")?.textOrNull().assertNotNull("altTitle"), + description = root.selectFirst(".datas_synopsis")?.html().assertNotNull("description") + ?: manga.description, + state = when (root.selectFirst(".datas_more-status-data")?.textOrNull().assertNotNull("status")) { + "En cours" -> MangaState.ONGOING + else -> null + }, + author = root.selectFirst(".datas_more-authors-people")?.textOrNull().assertNotNull("author"), + chapters = run { + val input = root.selectFirst("input[name=\"limit\"]") ?: return@run parseChapters(root) + val max = input.attr("max").toInt() + if (max <= 1) { + parseChapters(root) + } else { + coroutineScope { + val result = ArrayList(parseChapters(root)) + result.ensureCapacity(result.size * max) + (2..max).map { i -> + async { + loadChapters(mangaUrl, i) + } + }.awaitAll() + .flattenTo(result) + result + } + } + }.reversed(), + ) + } + + override suspend fun getPages(chapter: MangaChapter): List { + val chapterUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(chapterUrl).parseHtml() + val chapterId = doc.head().getElementsByAttribute("data-chapter-id").first()!!.attr("data-chapter-id") + val json = webClient.httpGet( + "https://$domain/api/?id=$chapterId&type=chapter", + Headers.headersOf( + "Referer", chapterUrl, + "x-requested-with", "XMLHttpRequest", + ), + ).parseJson() + if (json.getIntOrDefault("type", 1) == 2) { + throw ParseException("Light Novels are not supported", chapterUrl) + } + val baseUrl = json.getString("baseImagesUrl") + val pages = json.getJSONArray("page_array") + return (0 until pages.length()).map { i -> + val url = concatUrl(baseUrl, pages.getString(i)) + MangaPage( + id = generateUid(url), + url = url.toAbsoluteUrl(domain), + preview = null, + source = source, + ) + } + } + + override suspend fun getTags(): Set { + val root = webClient.httpGet(urlBuilder().addPathSegment("manga_list").build()) + .parseHtml() + .requireElementById("search_options-form") + return root.getElementsByAttributeValue("name", "categories[]") + .mapToSet { input -> + val div = input.parents().first()!! + MangaTag( + title = div.text().toTitleCase(sourceLocale), + key = input.attr("value"), + source = source, + ) + } + } + + private suspend fun loadChapters(baseUrl: String, page: Int): List { + return parseChapters(webClient.httpGet("$baseUrl?limit=$page").parseHtml().body()) + } + + private fun parseChapters(root: Element): List { + return root.requireElementById("chapters_content") + .select(".component-chapter").map { div -> + val a = div.selectFirstOrThrow("a") + val href = a.attrAsRelativeUrl("href") + val title = div.selectFirstOrThrow(".chapter_volume").text() + val name = div.selectFirst(".chapter_title")?.textOrNull() + MangaChapter( + id = generateUid(href), + name = if (name != null && name != title) "$title: $name" else title, + number = href.substringAfterLast('/').toIntOrNull() ?: 0, + url = href, + scanlator = div.selectFirst(".team_link-name")?.textOrNull(), + uploadDate = div.selectFirst(".component-chapter-date") + ?.ownTextOrNull() + .parseDate(), + branch = null, + source = source, + ) + } + } + + private fun String?.parseDate(): Long { + if (this == null) { + assert(false) { "Date is null" } + return 0L + } + val parts = split(' ') + assert(parts.size == 2) { "Wrong date $this" } + val count = parts.getOrNull(0)?.toIntOrNull() ?: return 0L + val unit = parts.getOrNull(1) ?: return 0L + val calendarUnit = when (unit) { + "s" -> Calendar.SECOND + "min" -> Calendar.MINUTE + "h" -> Calendar.HOUR + "j" -> Calendar.DAY_OF_YEAR + "sem." -> Calendar.WEEK_OF_YEAR + "mois" -> Calendar.MONTH + "ans", "an" -> Calendar.YEAR + else -> { + assert(false) { "Unknown time unit $unit" } + return 0L + } + } + val calendar = Calendar.getInstance() + calendar.add(calendarUnit, -count) + return calendar.timeInMillis + } + + private fun Element.src(): String { + return attrAsAbsoluteUrlOrNull("data-cfsrc") + ?: attrAsAbsoluteUrlOrNull("src") + ?: attrAsAbsoluteUrlOrNull("data-src") + ?: run { + assert(false) { "Image src not found" } + "" + } + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BlogTruyenParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BlogTruyenParser.kt index 08c1514d..2167845f 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BlogTruyenParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BlogTruyenParser.kt @@ -14,11 +14,10 @@ import org.koitharu.kotatsu.parsers.model.* import org.koitharu.kotatsu.parsers.util.* import java.text.SimpleDateFormat import java.util.* -import kotlin.collections.HashSet @MangaSourceParser("BLOGTRUYEN", "BlogTruyen", "vi") -class BlogTruyenParser(override val context: MangaLoaderContext) : - PagedMangaParser(MangaSource.BLOGTRUYEN, pageSize = 20) { +class BlogTruyenParser(context: MangaLoaderContext) : + PagedMangaParser(context, MangaSource.BLOGTRUYEN, pageSize = 20) { override val configKeyDomain: ConfigKey.Domain get() = ConfigKey.Domain("blogtruyen.vn", null) @@ -31,7 +30,7 @@ class BlogTruyenParser(override val context: MangaLoaderContext) : private var cacheTags: ArrayMap? = null override suspend fun getDetails(manga: Manga): Manga { - val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml() + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() val descriptionElement = doc.selectFirstOrThrow("div.description") val statusText = descriptionElement .selectFirst("p:contains(Trạng thái) > span.color-red") @@ -67,16 +66,16 @@ class BlogTruyenParser(override val context: MangaLoaderContext) : author = descriptionElement.selectFirst("p:contains(Tác giả) > a")?.text(), description = doc.selectFirst(".detail .content")?.html(), chapters = parseChapterList(doc), - largeCoverUrl = doc.selectLast("div.thumbnail > img")?.attrAsAbsoluteUrlOrNull("src"), + largeCoverUrl = doc.selectLast("div.thumbnail > img")?.imageUrl(), state = state, rating = rating ?: RATING_UNKNOWN, - isNsfw = doc.getElementById("warningCategory") != null + isNsfw = doc.getElementById("warningCategory") != null, ) } private fun parseChapterList(doc: Document): List { val chapterList = doc.select("#list-chapters > p") - return chapterList.asReversed().mapChapters { index, element -> + return chapterList.mapChapters(reversed = true) { index, element -> val titleElement = element.selectFirst("span.title > a") ?: return@mapChapters null val name = titleElement.text() val relativeUrl = titleElement.attrAsRelativeUrl("href") @@ -90,7 +89,7 @@ class BlogTruyenParser(override val context: MangaLoaderContext) : scanlator = null, uploadDate = uploadDate, branch = null, - source = source + source = source, ) } } @@ -103,25 +102,27 @@ class BlogTruyenParser(override val context: MangaLoaderContext) : ): List { return when { !query.isNullOrEmpty() -> { - val searchUrl = "https://${getDomain()}/timkiem/nangcao/1/0/-1/-1?txt=${query.urlEncoded()}&p=$page" - val searchContent = context.httpGet(searchUrl).parseHtml() + val searchUrl = "https://${domain}/timkiem/nangcao/1/0/-1/-1?txt=${query.urlEncoded()}&p=$page" + val searchContent = webClient.httpGet(searchUrl).parseHtml() .selectFirst("section.list-manga-bycate > div.list") parseMangaList(searchContent) } !tags.isNullOrEmpty() -> { val tag = tags.oneOrThrowIfMany()!! - val categoryAjax = "https://${getDomain()}/ajax/Category/AjaxLoadMangaByCategory?id=${tag.key}&orderBy=5&p=$page" - val listContent = context.httpGet(categoryAjax).parseHtml().selectFirst("div.list") + val categoryAjax = + "https://${domain}/ajax/Category/AjaxLoadMangaByCategory?id=${tag.key}&orderBy=5&p=$page" + val listContent = webClient.httpGet(categoryAjax).parseHtml().selectFirst("div.list") parseMangaList(listContent) } + else -> getNormalList(page) } } private suspend fun getNormalList(page: Int): List { - val pageLink = "https://${getDomain()}/page-$page" - val doc = context.httpGet(pageLink).parseHtml() + val pageLink = "https://${domain}/page-$page" + val doc = webClient.httpGet(pageLink).parseHtml() val listElements = doc.selectFirstOrThrow("section.list-mainpage.listview") .select("div.bg-white.storyitem") @@ -139,8 +140,8 @@ class BlogTruyenParser(override val context: MangaLoaderContext) : altTitle = null, description = it.selectFirst("p.al-j.break.line-height-15")?.text(), url = relativeUrl, - publicUrl = relativeUrl.toAbsoluteUrl(getDomain()), - coverUrl = linkTag.selectLast("img")?.attr("src").orEmpty(), + publicUrl = relativeUrl.toAbsoluteUrl(domain), + coverUrl = linkTag.selectLast("img")?.imageUrl().orEmpty(), source = source, tags = tags, isNsfw = false, @@ -164,8 +165,8 @@ class BlogTruyenParser(override val context: MangaLoaderContext) : altTitle = null, description = mangaInfo.select("div.al-j.fs-12").text(), url = relativeUrl, - publicUrl = relativeUrl.toAbsoluteUrl(getDomain()), - coverUrl = mangaInfo.selectFirst("div > img.img")?.absUrl("src").orEmpty(), + publicUrl = relativeUrl.toAbsoluteUrl(domain), + coverUrl = mangaInfo.selectFirst("div > img.img")?.imageUrl().orEmpty(), isNsfw = false, rating = RATING_UNKNOWN, tags = emptySet(), @@ -179,19 +180,17 @@ class BlogTruyenParser(override val context: MangaLoaderContext) : override suspend fun getPages(chapter: MangaChapter): List { fun generateImageId(index: Int) = generateUid("${chapter.url}/$index") - val doc = context.httpGet(chapter.url.toAbsoluteUrl(getDomain())).parseHtml() + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() val pages = ArrayList() - val referer = chapter.url.toAbsoluteUrl(getDomain()) + val referer = chapter.url.toAbsoluteUrl(domain) doc.select("#content > img").forEach { img -> - val url = img.attrAsRelativeUrl("src") pages.add( MangaPage( - id = generateImageId(pages.lastIndex), - url = url, - referer = referer, + id = generateImageId(pages.size), + url = img.imageUrl(), preview = null, source = source, - ) + ), ) } @@ -204,12 +203,11 @@ class BlogTruyenParser(override val context: MangaLoaderContext) : val imageUrl = imageArr.getJSONObject(i).getString("url") pages.add( MangaPage( - id = generateImageId(pages.lastIndex), + id = generateImageId(pages.size), url = imageUrl, - referer = referer, preview = null, - source = source - ) + source = source, + ), ) } } @@ -230,7 +228,7 @@ class BlogTruyenParser(override val context: MangaLoaderContext) : private suspend fun getOrCreateTagMap(): ArrayMap = mutex.withLock { cacheTags?.let { return@withLock it } - val doc = context.httpGet("/timkiem/nangcao".toAbsoluteUrl(getDomain())).parseHtml() + val doc = webClient.httpGet("/timkiem/nangcao".toAbsoluteUrl(domain)).parseHtml() val tagItems = doc.select("li[data-id]") val tagMap = ArrayMap(tagItems.size) for (tag in tagItems) { @@ -238,11 +236,17 @@ class BlogTruyenParser(override val context: MangaLoaderContext) : tagMap[tag.text().trim()] = MangaTag( title = title, key = tag.attr("data-id"), - source = source + source = source, ) } cacheTags = tagMap tagMap } + + private fun Element.imageUrl(): String { + return attrAsAbsoluteUrlOrNull("src") + ?: attrAsAbsoluteUrlOrNull("data-cfsrc") + ?: "" + } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/CloneMangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/CloneMangaParser.kt index a4d0f797..aa42dfd0 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/CloneMangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/CloneMangaParser.kt @@ -1,13 +1,16 @@ package org.koitharu.kotatsu.parsers.site -import org.koitharu.kotatsu.parsers.* +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.PagedMangaParser import org.koitharu.kotatsu.parsers.config.ConfigKey import org.koitharu.kotatsu.parsers.model.* import org.koitharu.kotatsu.parsers.util.* import java.util.* @MangaSourceParser("CLONEMANGA", "CloneManga", "en") -internal class CloneMangaParser(override val context: MangaLoaderContext) : PagedMangaParser( +internal class CloneMangaParser(context: MangaLoaderContext) : PagedMangaParser( + context, MangaSource.CLONEMANGA, pageSize = 1, ) { @@ -18,10 +21,6 @@ internal class CloneMangaParser(override val context: MangaLoaderContext) : Page override val configKeyDomain = ConfigKey.Domain("manga.clone-army.org", null) - override fun getFaviconUrl(): String { - return "https://pbs.twimg.com/profile_images/458758466346029056/Ys93EANp_400x400.png" - } - override suspend fun getListPage( page: Int, query: String?, @@ -31,17 +30,17 @@ internal class CloneMangaParser(override val context: MangaLoaderContext) : Page if (query != null || page > 1) { return emptyList() } - val link = "https://${getDomain()}/viewer_landing.php" - val doc = context.httpGet(link).parseHtml() + val link = "https://${domain}/viewer_landing.php" + val doc = webClient.httpGet(link).parseHtml() val mangas = doc.getElementsByClass("comicPreviewContainer") return mangas.mapNotNull { item -> - val attr = item.getElementsByClass("comicPreview").attr("style") + val background = item.selectFirstOrThrow(".comicPreview").styleValueOrNull("background") val href = item.selectFirst("a")?.attrAsAbsoluteUrl("href") ?: return@mapNotNull null - val cover = attr.substring(attr.indexOf("site/themes"), attr.indexOf(")")) + val cover = background?.substring(background.indexOf("site/themes"), background.indexOf(")")) Manga( id = generateUid(href), title = item.selectFirst("h3")?.text() ?: return@mapNotNull null, - coverUrl = "https://${getDomain()}/$cover", + coverUrl = "https://${domain}/$cover", altTitle = null, author = "Dan Kim", rating = RATING_UNKNOWN, @@ -49,14 +48,14 @@ internal class CloneMangaParser(override val context: MangaLoaderContext) : Page isNsfw = false, tags = emptySet(), state = null, - publicUrl = href.toAbsoluteUrl(getDomain()), + publicUrl = href.toAbsoluteUrl(domain), source = source, ) } } override suspend fun getDetails(manga: Manga): Manga { - val doc = context.httpGet(manga.publicUrl).parseHtml() + val doc = webClient.httpGet(manga.publicUrl).parseHtml() val series = doc.location() val numChapters = Regex( pattern = "&page=(.*)&lang=", @@ -75,7 +74,7 @@ internal class CloneMangaParser(override val context: MangaLoaderContext) : Page scanlator = null, branch = null, uploadDate = 0L, - source = MangaSource.DUMMY, + source = source, ) chapters.add(chapter) } @@ -83,7 +82,7 @@ internal class CloneMangaParser(override val context: MangaLoaderContext) : Page } override suspend fun getPages(chapter: MangaChapter): List { - val doc = context.httpGet(chapter.url.toAbsoluteUrl(getDomain())).parseHtml() + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() val imgUrl = doc.getElementsByClass("subsectionContainer")[0] .selectFirst("img") ?.attrAsAbsoluteUrlOrNull("src") ?: doc.parseFailed("Something broken") @@ -91,14 +90,11 @@ internal class CloneMangaParser(override val context: MangaLoaderContext) : Page MangaPage( id = generateUid(imgUrl), url = imgUrl, - referer = imgUrl, preview = null, - source = MangaSource.DUMMY, + source = source, ), ) } - override suspend fun getTags(): Set { - return emptySet() - } -} \ No newline at end of file + override suspend fun getTags(): Set = emptySet() +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ComickFunParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ComickFunParser.kt index 06acadeb..aae85a03 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ComickFunParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ComickFunParser.kt @@ -22,9 +22,9 @@ private const val PAGE_SIZE = 20 private const val CHAPTERS_LIMIT = 99999 @MangaSourceParser("COMICK_FUN", "ComicK") -internal class ComickFunParser(override val context: MangaLoaderContext) : MangaParser(MangaSource.COMICK_FUN) { +internal class ComickFunParser(context: MangaLoaderContext) : MangaParser(context, MangaSource.COMICK_FUN) { - override val configKeyDomain = ConfigKey.Domain("comick.fun", null) + override val configKeyDomain = ConfigKey.Domain("comick.app", null) override val sortOrders: Set = EnumSet.of( SortOrder.POPULARITY, @@ -41,11 +41,11 @@ internal class ComickFunParser(override val context: MangaLoaderContext) : Manga tags: Set?, sortOrder: SortOrder, ): List { - val domain = getDomain() + val domain = domain val url = buildString { append("https://api.") append(domain) - append("/search?tachiyomi=true") + append("/v1.0/search?tachiyomi=true") if (!query.isNullOrEmpty()) { if (offset > 0) { return emptyList() @@ -71,7 +71,7 @@ internal class ComickFunParser(override val context: MangaLoaderContext) : Manga ) } } - val ja = context.httpGet(url).parseJsonArray() + val ja = webClient.httpGet(url).parseJsonArray() val tagsMap = cachedTags ?: loadTags() return ja.mapJSON { jo -> val slug = jo.getString("slug") @@ -101,9 +101,9 @@ internal class ComickFunParser(override val context: MangaLoaderContext) : Manga } override suspend fun getDetails(manga: Manga): Manga { - val domain = getDomain() + val domain = domain val url = "https://api.$domain/comic/${manga.url}?tachiyomi=true" - val jo = context.httpGet(url).parseJson() + val jo = webClient.httpGet(url).parseJson() val comic = jo.getJSONObject("comic") return manga.copy( title = comic.getString("title"), @@ -118,21 +118,20 @@ internal class ComickFunParser(override val context: MangaLoaderContext) : Manga ) }, author = jo.getJSONArray("artists").optJSONObject(0)?.getString("name"), - chapters = getChapters(comic.getLong("id")), + chapters = getChapters(comic.getString("hid")), ) } override suspend fun getPages(chapter: MangaChapter): List { - val jo = context.httpGet( - "https://api.${getDomain()}/chapter/${chapter.url}?tachiyomi=true", + val jo = webClient.httpGet( + "https://api.${domain}/chapter/${chapter.url}?tachiyomi=true", ).parseJson().getJSONObject("chapter") - val referer = "https://${getDomain()}/" + val referer = "https://${domain}/" return jo.getJSONArray("images").mapJSON { val url = it.getString("url") MangaPage( id = generateUid(url), url = url, - referer = referer, preview = null, source = source, ) @@ -149,7 +148,7 @@ internal class ComickFunParser(override val context: MangaLoaderContext) : Manga } private suspend fun loadTags(): SparseArrayCompat { - val ja = context.httpGet("https://api.${getDomain()}/genre").parseJsonArray() + val ja = webClient.httpGet("https://api.${domain}/genre").parseJsonArray() val tags = SparseArrayCompat(ja.length()) for (jo in ja.JSONIterator()) { tags.append( @@ -165,9 +164,9 @@ internal class ComickFunParser(override val context: MangaLoaderContext) : Manga return tags } - private suspend fun getChapters(id: Long): List { - val ja = context.httpGet( - url = "https://api.${getDomain()}/comic/$id/chapter?tachiyomi=true&limit=$CHAPTERS_LIMIT", + private suspend fun getChapters(hid: String): List { + val ja = webClient.httpGet( + url = "https://api.${domain}/comic/$hid/chapters?limit=$CHAPTERS_LIMIT", ).parseJson().getJSONArray("chapters") val dateFormat = SimpleDateFormat("yyyy-MM-dd") val counters = HashMap() @@ -213,4 +212,4 @@ internal class ComickFunParser(override val context: MangaLoaderContext) : Manga } return res } -} \ No newline at end of file +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/DesuMeParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/DesuMeParser.kt index 3efe8716..d974ffed 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/DesuMeParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/DesuMeParser.kt @@ -1,5 +1,6 @@ package org.koitharu.kotatsu.parsers.site +import androidx.collection.ArrayMap import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaSourceParser import org.koitharu.kotatsu.parsers.PagedMangaParser @@ -13,7 +14,7 @@ import org.koitharu.kotatsu.parsers.util.json.mapJSONToSet import java.util.* @MangaSourceParser("DESUME", "Desu.me", "ru") -internal class DesuMeParser(override val context: MangaLoaderContext) : PagedMangaParser(MangaSource.DESUME, 20) { +internal class DesuMeParser(context: MangaLoaderContext) : PagedMangaParser(context, MangaSource.DESUME, 20) { override val configKeyDomain = ConfigKey.Domain("desu.me", null) @@ -24,6 +25,8 @@ internal class DesuMeParser(override val context: MangaLoaderContext) : PagedMan SortOrder.ALPHABETICAL, ) + private val tagsCache = SuspendLazy(::fetchTags) + override suspend fun getListPage( page: Int, query: String?, @@ -33,7 +36,7 @@ internal class DesuMeParser(override val context: MangaLoaderContext) : PagedMan if (query != null && page != searchPaginator.firstPage) { return emptyList() } - val domain = getDomain() + val domain = domain val url = buildString { append("https://") append(domain) @@ -50,14 +53,16 @@ internal class DesuMeParser(override val context: MangaLoaderContext) : PagedMan append(query) } } - val json = context.httpGet(url).parseJson().getJSONArray("response") + val json = webClient.httpGet(url).parseJson().getJSONArray("response") ?: throw ParseException("Invalid response", url) val total = json.length() val list = ArrayList(total) + val tagsMap = tagsCache.tryGet().getOrNull() for (i in 0 until total) { val jo = json.getJSONObject(i) val cover = jo.getJSONObject("image") val id = jo.getLong("id") + val genres = jo.getString("genres").split(',') list += Manga( url = "/manga/api/$id", publicUrl = jo.getString("url"), @@ -73,7 +78,13 @@ internal class DesuMeParser(override val context: MangaLoaderContext) : PagedMan rating = jo.getDouble("score").toFloat().coerceIn(0f, 1f), id = generateUid(id), isNsfw = false, - tags = emptySet(), + tags = if (!tagsMap.isNullOrEmpty()) { + genres.mapNotNullToSet { g -> + tagsMap[g.trim().toTitleCase()] + } + } else { + emptySet() + }, author = null, description = jo.getString("description"), ) @@ -82,8 +93,8 @@ internal class DesuMeParser(override val context: MangaLoaderContext) : PagedMan } override suspend fun getDetails(manga: Manga): Manga { - val url = manga.url.toAbsoluteUrl(getDomain()) - val json = context.httpGet(url).parseJson().getJSONObject("response") + val url = manga.url.toAbsoluteUrl(domain) + val json = webClient.httpGet(url).parseJson().getJSONObject("response") ?: throw ParseException("Invalid response", url) val baseChapterUrl = manga.url + "/chapter/" val chaptersList = json.getJSONObject("chapters").getJSONArray("list") @@ -117,14 +128,13 @@ internal class DesuMeParser(override val context: MangaLoaderContext) : PagedMan } override suspend fun getPages(chapter: MangaChapter): List { - val fullUrl = chapter.url.toAbsoluteUrl(getDomain()) - val json = context.httpGet(fullUrl) + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val json = webClient.httpGet(fullUrl) .parseJson() .getJSONObject("response") ?: throw ParseException("Invalid response", fullUrl) return json.getJSONObject("pages").getJSONArray("list").mapJSON { jo -> MangaPage( id = generateUid(jo.getLong("id")), - referer = fullUrl, preview = null, source = chapter.source, url = jo.getString("img"), @@ -133,12 +143,27 @@ internal class DesuMeParser(override val context: MangaLoaderContext) : PagedMan } override suspend fun getTags(): Set { - val doc = context.httpGet("https://${getDomain()}/manga/").parseHtml() + return tagsCache.get().values.toSet() + } + + private fun getSortKey(sortOrder: SortOrder) = + when (sortOrder) { + SortOrder.ALPHABETICAL -> "name" + SortOrder.POPULARITY -> "popular" + SortOrder.UPDATED -> "updated" + SortOrder.NEWEST -> "id" + else -> "updated" + } + + private suspend fun fetchTags(): Map { + val doc = webClient.httpGet("https://${domain}/manga/").parseHtml() val root = doc.body().requireElementById("animeFilter") .selectFirstOrThrow(".catalog-genres") - return root.select("li").mapToSet { + val li = root.select("li") + val result = ArrayMap(li.size) + li.forEach { val input = it.selectFirstOrThrow("input") - MangaTag( + val tag = MangaTag( source = source, key = input.attr("data-genre-slug").ifEmpty { it.parseFailed("data-genre-slug is empty") @@ -147,15 +172,8 @@ internal class DesuMeParser(override val context: MangaLoaderContext) : PagedMan it.parseFailed("data-genre-name is empty") }, ) + result[tag.title] = tag } + return result } - - private fun getSortKey(sortOrder: SortOrder) = - when (sortOrder) { - SortOrder.ALPHABETICAL -> "name" - SortOrder.POPULARITY -> "popular" - SortOrder.UPDATED -> "updated" - SortOrder.NEWEST -> "id" - else -> "updated" - } -} \ No newline at end of file +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/DoujinDesuParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/DoujinDesuParser.kt new file mode 100644 index 00000000..9e9faa7a --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/DoujinDesuParser.kt @@ -0,0 +1,141 @@ +package org.koitharu.kotatsu.parsers.site + +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.PagedMangaParser +import org.koitharu.kotatsu.parsers.config.ConfigKey +import org.koitharu.kotatsu.parsers.model.* +import org.koitharu.kotatsu.parsers.util.* +import java.text.SimpleDateFormat +import java.util.* + +@MangaSourceParser("DOUJINDESU", "DoujinDesu", "id") +class DoujinDesuParser(context: MangaLoaderContext) : PagedMangaParser(context, MangaSource.DOUJINDESU, pageSize = 18) { + + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("212.32.226.234", null) + + override val sortOrders: Set + get() = EnumSet.of(SortOrder.UPDATED, SortOrder.NEWEST, SortOrder.ALPHABETICAL, SortOrder.POPULARITY) + + override suspend fun getDetails(manga: Manga): Manga { + val docs = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml().selectFirstOrThrow("#archive") + val chapterDateFormat = SimpleDateFormat("EEEE, dd MMMM yyyy", sourceLocale) + val metadataEl = docs.selectFirst(".wrapper > .metadata tbody") + val state = when (metadataEl?.selectFirst("tr:contains(Status)")?.selectLast("td")?.text()) { + "Finished" -> MangaState.FINISHED + "Publishing" -> MangaState.ONGOING + else -> null + } + return manga.copy( + author = metadataEl?.selectFirst("tr:contains(Author)")?.selectLast("td")?.text(), + description = docs.selectFirst(".wrapper > .metadata > .pb-2")?.selectFirst("p")?.html(), + state = state, + rating = metadataEl?.selectFirst(".rating-prc")?.ownText()?.toFloatOrNull()?.div(10f) ?: RATING_UNKNOWN, + tags = docs.select(".tags > a").mapToSet { + MangaTag( + key = it.attr("title"), + title = it.text(), + source = source, + ) + }, + chapters = docs.requireElementById("chapter_list") + .select("ul > li") + .mapChapters(reversed = true) { index, element -> + val titleTag = element.selectFirstOrThrow(".epsleft > .lchx > a") + val url = titleTag.attrAsRelativeUrl("href") + MangaChapter( + id = generateUid(url), + name = titleTag.text(), + number = index + 1, + url = url, + scanlator = null, + uploadDate = chapterDateFormat.tryParse(element.select(".epsleft > .date").text()), + branch = null, + source = source, + ) + }, + ) + } + + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + val url = urlBuilder().apply { + addPathSegment("manga") + addPathSegment("page") + addPathSegment("$page/") + val order = when (sortOrder) { + SortOrder.UPDATED -> "update" + SortOrder.POPULARITY -> "popular" + SortOrder.ALPHABETICAL -> "title" + SortOrder.NEWEST -> "latest" + else -> throw IllegalArgumentException("Sort order not supported") + } + addQueryParameter("order", order) + addQueryParameter("title", query.orEmpty()) + tags?.forEach { + addEncodedQueryParameter("genre[]".urlEncoded(), it.key.urlEncoded()) + } + }.build() + + return webClient.httpGet(url).parseHtml() + .requireElementById("archives") + .selectFirstOrThrow("div.entries") + .select(".entry") + .map { + val titleTag = it.selectFirstOrThrow(".metadata > a") + val relativeUrl = titleTag.attrAsRelativeUrl("href") + Manga( + id = generateUid(relativeUrl), + title = titleTag.attr("title"), + altTitle = null, + url = relativeUrl, + publicUrl = relativeUrl.toAbsoluteUrl(domain), + rating = RATING_UNKNOWN, + isNsfw = true, + coverUrl = it.selectFirst(".thumbnail > img")?.attrAsAbsoluteUrl("src").orEmpty(), + tags = emptySet(), + state = null, + author = null, + largeCoverUrl = null, + description = null, + source = source, + ) + } + } + + override suspend fun getPages(chapter: MangaChapter): List { + val id = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + .requireElementById("reader") + .attr("data-id") + return webClient.httpPost("/themes/ajax/ch.php".toAbsoluteUrl(domain), "id=$id").parseHtml() + .select("img") + .map { + val url = it.attrAsRelativeUrl("src") + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } + + override suspend fun getTags(): Set { + return webClient.httpGet("/genre/".toAbsoluteUrl(domain)).parseHtml() + .requireElementById("taxonomy") + .selectFirstOrThrow(".entries") + .select(".entry > a") + .mapToSet { + MangaTag( + key = it.attr("title"), + title = it.attr("title"), + source = source, + ) + } + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ExHentaiParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ExHentaiParser.kt index f53a20f7..0f0664bd 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ExHentaiParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ExHentaiParser.kt @@ -1,5 +1,8 @@ package org.koitharu.kotatsu.parsers.site +import androidx.collection.SparseArrayCompat +import androidx.collection.set +import okhttp3.HttpUrl.Companion.toHttpUrlOrNull import org.jsoup.nodes.Element import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaParserAuthProvider @@ -17,8 +20,8 @@ private const val DOMAIN_AUTHORIZED = "exhentai.org" @MangaSourceParser("EXHENTAI", "ExHentai") internal class ExHentaiParser( - override val context: MangaLoaderContext, -) : PagedMangaParser(MangaSource.EXHENTAI, pageSize = 25), MangaParserAuthProvider { + context: MangaLoaderContext, +) : PagedMangaParser(context, MangaSource.EXHENTAI, pageSize = 25), MangaParserAuthProvider { override val sortOrders: Set = Collections.singleton( SortOrder.NEWEST, @@ -28,11 +31,13 @@ internal class ExHentaiParser( get() = ConfigKey.Domain(if (isAuthorized) DOMAIN_AUTHORIZED else DOMAIN_UNAUTHORIZED, null) override val authUrl: String - get() = "https://${getDomain()}/bounce_login.php" + get() = "https://${domain}/bounce_login.php" private val ratingPattern = Regex("-?[0-9]+px") private val authCookies = arrayOf("ipb_member_id", "ipb_pass_hash") private var updateDm = false + private val nextPages = SparseArrayCompat() + private val suspiciousContentKey = ConfigKey.ShowSuspiciousContent(true) override val isAuthorized: Boolean get() { @@ -54,6 +59,7 @@ internal class ExHentaiParser( init { context.cookieJar.insertCookies(DOMAIN_AUTHORIZED, "nw=1", "sl=dm_2") context.cookieJar.insertCookies(DOMAIN_UNAUTHORIZED, "nw=1", "sl=dm_2") + paginator.firstPage = 0 } override suspend fun getListPage( @@ -63,11 +69,16 @@ internal class ExHentaiParser( sortOrder: SortOrder, ): List { var search = query?.urlEncoded().orEmpty() + val next = nextPages.get(page, 0L) + if (page > 0 && next == 0L) { + assert(false) { "Page timestamp not found" } + return emptyList() + } val url = buildString { append("https://") - append(getDomain()) - append("/?page=") - append(page) + append(domain) + append("/?next=") + append(next) if (!tags.isNullOrEmpty()) { var fCats = 0 for (tag in tags) { @@ -88,8 +99,12 @@ internal class ExHentaiParser( if (updateDm) { append("&inline_set=dm_e") } + append("&advsearch=1") + if (config[suspiciousContentKey]) { + append("&f_sh=on") + } } - val body = context.httpGet(url).parseHtml().body() + val body = webClient.httpGet(url).parseHtml().body() val root = body.selectFirst("table.itg") ?.selectFirst("tbody") ?: if (updateDm) { @@ -99,6 +114,7 @@ internal class ExHentaiParser( return getListPage(page, query, tags, sortOrder) } updateDm = false + nextPages[page + 1] = getNextTimestamp(body) return root.children().mapNotNull { tr -> if (tr.childrenSize() != 2) return@mapNotNull null val (td1, td2) = tr.children() @@ -132,7 +148,7 @@ internal class ExHentaiParser( } override suspend fun getDetails(manga: Manga): Manga { - val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml() + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() val root = doc.body().selectFirstOrThrow("div.gm") val cover = root.getElementById("gd1")?.children()?.first() val title = root.getElementById("gd2") @@ -176,14 +192,13 @@ internal class ExHentaiParser( } override suspend fun getPages(chapter: MangaChapter): List { - val doc = context.httpGet(chapter.url.toAbsoluteUrl(getDomain())).parseHtml() + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() val root = doc.body().requireElementById("gdt") return root.select("a").map { a -> val url = a.attrAsRelativeUrl("href") MangaPage( id = generateUid(url), url = url, - referer = a.absUrl("href"), preview = null, source = source, ) @@ -191,12 +206,12 @@ internal class ExHentaiParser( } override suspend fun getPageUrl(page: MangaPage): String { - val doc = context.httpGet(page.url.toAbsoluteUrl(getDomain())).parseHtml() + val doc = webClient.httpGet(page.url.toAbsoluteUrl(domain)).parseHtml() return doc.body().requireElementById("img").attrAsAbsoluteUrl("src") } override suspend fun getTags(): Set { - val doc = context.httpGet("https://${getDomain()}").parseHtml() + val doc = webClient.httpGet("https://${domain}").parseHtml() val root = doc.body().requireElementById("searchbox").selectFirstOrThrow("table") return root.select("div.cs").mapNotNullToSet { div -> val id = div.id().substringAfterLast('_').toIntOrNull() @@ -210,7 +225,7 @@ internal class ExHentaiParser( } override suspend fun getUsername(): String { - val doc = context.httpGet("https://forums.$DOMAIN_UNAUTHORIZED/").parseHtml().body() + val doc = webClient.httpGet("https://forums.$DOMAIN_UNAUTHORIZED/").parseHtml().body() val username = doc.getElementById("userlinks") ?.getElementsByAttributeValueContaining("href", "showuser=") ?.firstOrNull() @@ -223,6 +238,11 @@ internal class ExHentaiParser( return username } + override fun onCreateConfig(keys: MutableCollection>) { + super.onCreateConfig(keys) + keys.add(suspiciousContentKey) + } + private fun isAuthorized(domain: String): Boolean { val cookies = context.cookieJar.getCookies(domain).mapToSet { x -> x.name } return authCookies.all { it in cookies } @@ -276,4 +296,12 @@ internal class ExHentaiParser( val num = className.drop(2).toIntOrNull(16) ?: return null return 2.0.pow(num).toInt().toString() } -} \ No newline at end of file + + private fun getNextTimestamp(root: Element): Long { + return root.getElementById("unext") + ?.attrAsAbsoluteUrlOrNull("href") + ?.toHttpUrlOrNull() + ?.queryParameter("next") + ?.toLongOrNull() ?: 1 + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/HoneyMangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/HoneyMangaParser.kt new file mode 100644 index 00000000..eea9c81c --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/HoneyMangaParser.kt @@ -0,0 +1,223 @@ +package org.koitharu.kotatsu.parsers.site + +import androidx.collection.ArraySet +import okhttp3.Headers +import okhttp3.Interceptor +import okhttp3.Response +import org.json.JSONArray +import org.json.JSONObject +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.PagedMangaParser +import org.koitharu.kotatsu.parsers.config.ConfigKey +import org.koitharu.kotatsu.parsers.model.* +import org.koitharu.kotatsu.parsers.util.* +import org.koitharu.kotatsu.parsers.util.json.getStringOrNull +import org.koitharu.kotatsu.parsers.util.json.mapJSON +import org.koitharu.kotatsu.parsers.util.json.mapJSONIndexed +import java.text.SimpleDateFormat +import java.util.* + + +private const val PAGE_SIZE = 20 +private const val INFINITE = 999999 +private const val HEADER_ENCODING = "Content-Encoding" + +@MangaSourceParser("HONEYMANGA", "Honey Manga", "uk") +class HoneyMangaParser(context: MangaLoaderContext) : PagedMangaParser(context, MangaSource.HONEYMANGA, PAGE_SIZE), + Interceptor { + + private val urlApi get() = "https://data.api.$domain" + private val mangaApi get() = "$urlApi/v2/manga/cursor-list" + private val chapterApi get() = "$urlApi/v2/chapter/cursor-list" + private val genresListApi get() = "$urlApi/genres-tags/genres-list" + private val framesApi get() = "$urlApi/chapter/frames" + private val searchApi get() = "https://search.api.$domain/api/v1/title/search-matching?query=" + + private val imageStorageUrl = "https://manga-storage.fra1.digitaloceanspaces.com/public-resources" + + override val headers + get() = Headers.Builder() + .add("User-Agent", "Mozilla/5.0 (Android 13; Mobile; rv:68.0) Gecko/68.0 Firefox/109.0") + .build() + + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("honey-manga.com.ua", null) + + override val sortOrders: Set = EnumSet.of( + SortOrder.POPULARITY, + SortOrder.NEWEST, + ) + + override suspend fun getDetails(manga: Manga): Manga { + val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.US) + val body = JSONObject() + body.put("mangaId", manga.url) + body.put("pageSize", INFINITE) // Hack lol (no) + body.put("page", 1) + body.put("sortOrder", "ASC") + val chapterRequest = webClient.httpPost(chapterApi, body).parseJson() + return manga.copy( + chapters = chapterRequest.getJSONArray("data").mapJSONIndexed() { i, jo -> + MangaChapter( + id = generateUid(jo.getString("id")), + name = buildString { + append("Том ") + append(jo.optString("volume", "0")) + append(". ") + append("Розділ ") + append(jo.optString("chapterNum", "0")) + if (jo.optString("title") != "Title") { + append(" - ") + append(jo.optString("title")) + } + }, + number = i + 1, + url = jo.optString("chapterResourcesId"), + scanlator = null, + uploadDate = dateFormat.tryParse(jo.getString("lastUpdated")), + branch = null, + source = source, + ) + }, + ) + } + + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + val body = JSONObject() + body.put("page", page) + body.put("pageSize", PAGE_SIZE) + val sort = JSONObject() + sort.put("sortBy", getSortKey(sortOrder)) + sort.put("sortOrder", "DESC") + body.put("sort", sort) + + val content = when { + !tags.isNullOrEmpty() -> { + // Tags + val filters = JSONArray() + val tagFilter = JSONObject() + tagFilter.put("filterBy", "genres") + tagFilter.put("filterOperator", "ALL") + val tag = JSONArray() + tags.forEach { + tag.put(it.title) + } + tagFilter.put("filterValue", tag) + filters.put(tagFilter) + body.put("filters", filters) + webClient.httpPost(mangaApi, body).parseJson().getJSONArray("data") + + } + + !query.isNullOrEmpty() -> { + // Search + when { + query.length < 3 -> throw IllegalArgumentException( + "The query must contain at least 3 characters (Запит має містити щонайменше 3 символи)", + ) + + page == searchPaginator.firstPage -> webClient + .httpGet(searchApi + query.urlEncoded()) + .parseJsonArray() + + else -> JSONArray() + } + } + + else -> { + // Popular/Newest + body.put("filters", JSONArray()) + webClient.httpPost(mangaApi, body).parseJson().getJSONArray("data") + } + } + return content.mapJSON { jo -> + val id = jo.getString("id") + val posterUrl = jo.getString("posterUrl") + Manga( + id = generateUid(id), + title = jo.getString("title"), + altTitle = jo.getStringOrNull("alternativeTitle"), + url = id, + publicUrl = "https://$domain/book/$id", + rating = RATING_UNKNOWN, + isNsfw = isNsfw(jo.getStringOrNull("adult")), + coverUrl = getCoverUrl(posterUrl, 256), + tags = getTitleTags(jo.optJSONArray("genresAndTags")), + state = when (jo.getStringOrNull("titleStatus")) { + "Онгоінг" -> MangaState.ONGOING + "Завершено" -> MangaState.FINISHED + else -> null + }, + author = null, + largeCoverUrl = getCoverUrl(posterUrl, 1080), + description = jo.getStringOrNull("description"), + chapters = null, + source = source, + ) + } + } + + override suspend fun getPages(chapter: MangaChapter): List { + val body = JSONObject() + body.put("chapterId", chapter.url) + val content = webClient.httpPost(framesApi, body).parseJson().getJSONObject("resourceIds") + return List(content.length()) { i -> + val item = content.getString(i.toString()) + MangaPage(id = generateUid(item), "$imageStorageUrl/$item", getCoverUrl(item, 256), source) + } + } + + override suspend fun getTags(): Set { + // https://data.api.honey-manga.com.ua/genres-tags/genres-list + val content = webClient.httpGet(genresListApi).parseJsonArray() + val tagsSet = ArraySet(content.length()) + repeat(content.length()) { i -> + val item = content.getString(i) + tagsSet.add(MangaTag(item, item, source)) + } + return tagsSet + } + + // Need for disable encoding (with encoding not working) + override fun intercept(chain: Interceptor.Chain): Response { + val request = chain.request() + val newRequest = if (request.header(HEADER_ENCODING) != null) { + request.newBuilder().removeHeader(HEADER_ENCODING).build() + } else { + request + } + return chain.proceed(newRequest) + } + + private fun isNsfw(adultValue: String?): Boolean { + val intValue = adultValue?.removeSuffix('+')?.toIntOrNull() + return intValue != null && intValue >= 18 + } + + private fun getCoverUrl(id: String, w: Int): String { + // https://honey-manga.com.ua/_next/image?url=https%3A%2F%2Fhoneymangastorage.b-cdn.net%2Fpublic-resources%2F1c4613c2-ffe3-405a-b26a-2cab59ddd223%3Foptimizer%3Dimage%26width%3D512%26height%3D512&w=3840&q=75 + return "https://$domain/_next/image?url=https%3A%2F%2Fhoneymangastorage.b-cdn.net%2Fpublic-resources%2F$id&w=$w&q=75" + } + + private fun getSortKey(order: SortOrder?) = when (order) { + SortOrder.POPULARITY -> "likes" + SortOrder.NEWEST -> "lastUpdated" + else -> "likes" + } + + private fun getTitleTags(jsonTags: JSONArray): Set { + val tagsSet = ArraySet(jsonTags.length()) + repeat(jsonTags.length()) { i -> + val item = jsonTags.getString(i) + + tagsSet.add(MangaTag(title = item.toTitleCase(sourceLocale), key = item, source = source)) + } + return tagsSet + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/JapScanParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/JapScanParser.kt new file mode 100644 index 00000000..8af433f5 --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/JapScanParser.kt @@ -0,0 +1,193 @@ +package org.koitharu.kotatsu.parsers.site + +import okhttp3.Headers +import org.json.JSONObject +import org.jsoup.nodes.Element +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.PagedMangaParser +import org.koitharu.kotatsu.parsers.config.ConfigKey +import org.koitharu.kotatsu.parsers.exception.ParseException +import org.koitharu.kotatsu.parsers.model.* +import org.koitharu.kotatsu.parsers.util.* +import org.koitharu.kotatsu.parsers.util.json.getStringOrNull +import org.koitharu.kotatsu.parsers.util.json.mapJSON +import java.text.SimpleDateFormat +import java.util.* + +@MangaSourceParser("JAPSCAN", "JapScan", "fr") +internal class JapScanParser(context: MangaLoaderContext) : PagedMangaParser(context, MangaSource.JAPSCAN, 30) { + + override val sortOrders: Set = EnumSet.of(SortOrder.ALPHABETICAL) + + override val configKeyDomain = ConfigKey.Domain("www.japscan.lol", arrayOf("www.japscan.lol", "japscan.ws")) + + override val headers: Headers = Headers.Builder() + .add("User-Agent", "Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/113.0") + .build() + + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + require(tags.isNullOrEmpty()) { "Tags are not supported by this source" } + if (!query.isNullOrEmpty()) { + return if (page == paginator.firstPage) getListPageSearch(query) else emptyList() + } + val url = urlBuilder() + .addPathSegment("mangas") + .addPathSegment(page.toString()) + .build() + val root = webClient.httpGet(url).parseHtml() + .requireElementById("main") + .selectFirstOrThrow(".flex-wrap") + return root.select(".mainTitle") + .map { p -> + val div = checkNotNull(p.parent()) + val a = div.selectFirstOrThrow("a") + val href = a.attrAsRelativeUrl("href") + Manga( + id = generateUid(href), + title = p.text(), + altTitle = null, + url = href, + publicUrl = href.toAbsoluteUrl(domain), + rating = RATING_UNKNOWN, + isNsfw = false, + coverUrl = div.selectFirstOrThrow("img").attrAsAbsoluteUrl("src"), + tags = setOf(), + state = null, + author = null, + source = source, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val root = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml().requireElementById("main") + val dateFormat = SimpleDateFormat("dd MMM yyyy", Locale.ENGLISH) + return manga.copy( + altTitle = root.tableValue("Nom Original:"), + isNsfw = root.tableValue("Âge conseillé :")?.extractIntOrNull().let { it != null && it >= 18 }, + tags = root.tableValue("Type(s):")?.split(", ")?.mapNotNullToSet { + it.toTag() + }.orEmpty() + root.tableValue("Genre(s):")?.split(", ")?.mapNotNullToSet { + it.toTag() + }.orEmpty(), + state = when (root.tableValue("Statut:")) { + "En Cours" -> MangaState.ONGOING + "Terminé", "Abondonné" -> MangaState.FINISHED + else -> null + }, + author = root.tableValue("Artiste(s):")?.substringBefore(','), + description = root.selectFirst("p.list-group-item-primary")?.html(), + chapters = root.requireElementById("chapters_list") + .select("div.chapters_list") + .mapChapters(reversed = true) { i, div -> + val a = div.selectFirst("a") ?: return@mapChapters null + val href = a.attrAsRelativeUrl("href") + MangaChapter( + id = generateUid(href), + name = a.text(), + number = i, + url = href, + scanlator = null, + uploadDate = dateFormat.tryParse(div.selectFirst("span.float-right")?.text()), + branch = null, + source = source, + ) + }, + ) + } + + override suspend fun getPages(chapter: MangaChapter): List { + val chapterUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(chapterUrl).parseHtml() + val scriptUrl = doc.getElementsByTag("script").firstNotNullOf { script -> + script.attrAsAbsoluteUrlOrNull("src")?.takeIf { it.contains("/zjs/") } + } + val embeddedData = doc.requireElementById("data").attr("data-data") + val script = webClient.httpGet(scriptUrl).parseRaw() + + val sample = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz".toList() + val keyRegex = Regex("""'([\dA-Z]{62})'""", RegexOption.IGNORE_CASE) + + val keyTables = keyRegex.findAll(script) + .mapNotNullTo(ArrayList(2)) { match -> + match.groupValues[1].takeIf { + it.toList().sorted() == sample + } + } + check(keyTables.size == 2) + + var error: Exception? = null + repeat(2) { i -> + val key = keyTables[i].zip(keyTables[1 - i]).toMap() + try { + val unscrambledData = embeddedData.map { key[it] ?: it }.joinToString("") + if (unscrambledData.startsWith("ey")) { + val array = JSONObject(context.decodeBase64(unscrambledData).toString(Charsets.UTF_8)) + .getJSONArray("imagesLink") + val result = ArrayList(array.length()) + repeat(array.length()) { index -> + val url = array.getString(index) + result += MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + return result + } + } catch (e: Exception) { + error = e + } + } + throw (error ?: ParseException("Cannot decode pages list", chapterUrl)) + } + + override suspend fun getTags(): Set { + return emptySet() // not supported + } + + private suspend fun getListPageSearch( + query: String, + ): List { + val json = webClient.httpPost( + "https://$domain/live-search/", + mapOf("search" to query.urlEncoded()), + ).parseJsonArray() + return json.mapJSON { jo -> + val url = jo.getString("url") + Manga( + id = generateUid(url), + title = jo.getString("name"), + altTitle = jo.getStringOrNull("alternate_names")?.substringBefore(','), + url = url, + publicUrl = url.toAbsoluteUrl(domain), + rating = RATING_UNKNOWN, + isNsfw = false, + coverUrl = jo.getString("image").toAbsoluteUrl(domain), + tags = emptySet(), + state = null, + author = null, + source = source, + ) + } + } + + private fun Element.tableValue(label: String): String? { + return getElementsMatchingOwnText(label).firstOrNull()?.parent()?.ownTextOrNull() + } + + private fun String.extractIntOrNull(): Int? = this.filter(Char::isDigit).toIntOrNull() + + private fun String.toTag() = MangaTag( + title = this.toTitleCase(sourceLocale), + key = this.replace(' ', '-').lowercase(sourceLocale), + source = source, + ) +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaDexParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaDexParser.kt index db06ace9..db033c80 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaDexParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaDexParser.kt @@ -20,12 +20,13 @@ private const val PAGE_SIZE = 20 private const val CHAPTERS_FIRST_PAGE_SIZE = 120 private const val CHAPTERS_MAX_PAGE_SIZE = 500 private const val CHAPTERS_PARALLELISM = 3 +private const val CHAPTERS_MAX_COUNT = 10_000 // strange api behavior, looks like a bug private const val CONTENT_RATING = "contentRating[]=safe&contentRating[]=suggestive&contentRating[]=erotica&contentRating[]=pornographic" private const val LOCALE_FALLBACK = "en" @MangaSourceParser("MANGADEX", "MangaDex") -internal class MangaDexParser(override val context: MangaLoaderContext) : MangaParser(MangaSource.MANGADEX) { +internal class MangaDexParser(context: MangaLoaderContext) : MangaParser(context, MangaSource.MANGADEX) { override val configKeyDomain = ConfigKey.Domain("mangadex.org", null) @@ -42,7 +43,7 @@ internal class MangaDexParser(override val context: MangaLoaderContext) : MangaP tags: Set?, sortOrder: SortOrder, ): List { - val domain = getDomain() + val domain = domain val url = buildString { append("https://api.") append(domain) @@ -75,7 +76,7 @@ internal class MangaDexParser(override val context: MangaLoaderContext) : MangaP }, ) } - val json = context.httpGet(url).parseJson().getJSONArray("data") + val json = webClient.httpGet(url).parseJson().getJSONArray("data") return json.mapJSON { jo -> val id = jo.getString("id") val attrs = jo.getJSONObject("attributes") @@ -123,10 +124,10 @@ internal class MangaDexParser(override val context: MangaLoaderContext) : MangaP } override suspend fun getDetails(manga: Manga): Manga = coroutineScope { - val domain = getDomain() + val domain = domain val mangaId = manga.url.removePrefix("/") val attrsDeferred = async { - context.httpGet( + webClient.httpGet( "https://api.$domain/manga/${mangaId}?includes[]=artist&includes[]=author&includes[]=cover_art", ).parseJson().getJSONObject("data").getJSONObject("attributes") } @@ -139,7 +140,7 @@ internal class MangaDexParser(override val context: MangaLoaderContext) : MangaP Locale.ROOT, ) manga.copy( - description = mangaAttrs.getJSONObject("description").selectByLocale() + description = mangaAttrs.optJSONObject("description")?.selectByLocale() ?: manga.description, chapters = feed.mapChapters { _, jo -> val id = jo.getString("id") @@ -149,12 +150,12 @@ internal class MangaDexParser(override val context: MangaLoaderContext) : MangaP } val locale = attrs.getStringOrNull("translatedLanguage")?.let { Locale.forLanguageTag(it) } val relations = jo.getJSONArray("relationships").associateByKey("type") - val number = attrs.getIntOrDefault("chapter", 0) + val number = attrs.getFloatOrDefault("chapter", 0f) MangaChapter( id = generateUid(id), name = attrs.getStringOrNull("title")?.takeUnless(String::isEmpty) ?: "Chapter #$number", - number = number, + number = number.toInt(), url = id, scanlator = relations["scanlation_group"]?.getStringOrNull("name"), uploadDate = dateFormat.tryParse(attrs.getString("publishAt")), @@ -166,19 +167,17 @@ internal class MangaDexParser(override val context: MangaLoaderContext) : MangaP } override suspend fun getPages(chapter: MangaChapter): List { - val domain = getDomain() - val chapterJson = context.httpGet("https://api.$domain/at-home/server/${chapter.url}?forcePort443=false") + val domain = domain + val chapterJson = webClient.httpGet("https://api.$domain/at-home/server/${chapter.url}?forcePort443=false") .parseJson() .getJSONObject("chapter") val pages = chapterJson.getJSONArray("data") val prefix = "https://uploads.$domain/data/${chapterJson.getString("hash")}/" - val referer = "https://$domain/" return List(pages.length()) { i -> val url = prefix + pages.getString(i) MangaPage( id = generateUid(url), url = url, - referer = referer, preview = null, // TODO prefix + dataSaver.getString(i), source = source, ) @@ -186,7 +185,7 @@ internal class MangaDexParser(override val context: MangaLoaderContext) : MangaP } override suspend fun getTags(): Set { - val tags = context.httpGet("https://api.${getDomain()}/manga/tag").parseJson() + val tags = webClient.httpGet("https://api.${domain}/manga/tag").parseJson() .getJSONArray("data") return tags.mapJSONToSet { jo -> MangaTag( @@ -214,7 +213,7 @@ internal class MangaDexParser(override val context: MangaLoaderContext) : MangaP return firstPage.data } val tail = coroutineScope { - val leftCount = firstPage.total - firstPage.size + val leftCount = firstPage.total.coerceAtMost(CHAPTERS_MAX_COUNT) - firstPage.size val pages = (leftCount / CHAPTERS_MAX_PAGE_SIZE.toFloat()).toIntUp() val dispatcher = Dispatchers.Default.limitedParallelism(CHAPTERS_PARALLELISM) List(pages) { page -> @@ -231,20 +230,25 @@ internal class MangaDexParser(override val context: MangaLoaderContext) : MangaP } private suspend fun loadChapters(mangaId: String, offset: Int, limit: Int): Chapters { + val limitedLimit = when { + offset >= CHAPTERS_MAX_COUNT -> return Chapters(emptyList(), CHAPTERS_MAX_COUNT) + offset + limit > CHAPTERS_MAX_COUNT -> CHAPTERS_MAX_COUNT - offset + else -> limit + } val url = buildString { append("https://api.") - append(getDomain()) + append(domain) append("/manga/") append(mangaId) append("/feed") append("?limit=") - append(limit) + append(limitedLimit) append("&includes[]=scanlation_group&order[volume]=asc&order[chapter]=asc&offset=") append(offset) append('&') append(CONTENT_RATING) } - val json = context.httpGet(url).parseJson() + val json = webClient.httpGet(url).parseJson() if (json.getString("result") == "ok") { return Chapters( data = json.optJSONArray("data")?.toJSONList().orEmpty(), @@ -266,4 +270,4 @@ internal class MangaDexParser(override val context: MangaLoaderContext) : MangaP val size: Int get() = data.size } -} \ No newline at end of file +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaInUaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaInUaParser.kt index 928d85c9..549fa3f1 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaInUaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaInUaParser.kt @@ -12,7 +12,8 @@ import java.util.* private const val DEF_BRANCH_NAME = "Основний переклад" @MangaSourceParser("MANGAINUA", "MANGA/in/UA", "uk") -class MangaInUaParser(override val context: MangaLoaderContext) : PagedMangaParser( +class MangaInUaParser(context: MangaLoaderContext) : PagedMangaParser( + context = context, source = MangaSource.MANGAINUA, pageSize = 24, searchPageSize = 10, @@ -31,21 +32,21 @@ class MangaInUaParser(override val context: MangaLoaderContext) : PagedMangaPars ): List { val url = when { !query.isNullOrEmpty() -> ( - "/index.php?do=search" + - "&subaction=search" + - "&search_start=$page" + - "&full_search=1" + - "&story=$query" + - "&titleonly=3" - ).toAbsoluteUrl(getDomain()) + "/index.php?do=search" + + "&subaction=search" + + "&search_start=$page" + + "&full_search=1" + + "&story=$query" + + "&titleonly=3" + ).toAbsoluteUrl(domain) - tags.isNullOrEmpty() -> "/mangas/page/$page".toAbsoluteUrl(getDomain()) + tags.isNullOrEmpty() -> "/mangas/page/$page".toAbsoluteUrl(domain) tags.size == 1 -> "${tags.first().key}/page/$page" tags.size > 1 -> throw IllegalArgumentException("This source supports only 1 genre") - else -> "/mangas/page/$page".toAbsoluteUrl(getDomain()) + else -> "/mangas/page/$page".toAbsoluteUrl(domain) } - val doc = context.httpGet(url).parseHtml() - val container = doc.body().requireElementById("dle-content") + val doc = webClient.httpGet(url).parseHtml() + val container = doc.body().requireElementById("site-content") val items = container.select("div.col-6") return items.mapNotNull { item -> val href = item.selectFirst("a")?.attrAsRelativeUrl("href") ?: return@mapNotNull null @@ -73,15 +74,15 @@ class MangaInUaParser(override val context: MangaLoaderContext) : PagedMangaPars } }.getOrNull().orEmpty(), state = null, - publicUrl = href.toAbsoluteUrl(container.host ?: getDomain()), + publicUrl = href.toAbsoluteUrl(container.host ?: domain), source = source, ) } } override suspend fun getDetails(manga: Manga): Manga { - val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml() - val root = doc.body().requireElementById("dle-content") + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val root = doc.body().requireElementById("site-content") val dateFormat = SimpleDateFormat("dd.MM.yyyy", Locale.US) val chapterNodes = root.selectFirstOrThrow(".linkstocomics").select(".ltcitems") var prevChapterName: String? = null @@ -120,8 +121,8 @@ class MangaInUaParser(override val context: MangaLoaderContext) : PagedMangaPars } override suspend fun getPages(chapter: MangaChapter): List { - val fullUrl = chapter.url.toAbsoluteUrl(getDomain()) - val doc = context.httpGet(fullUrl).parseHtml() + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() val root = doc.body().requireElementById("comics").selectFirstOrThrow("ul.xfieldimagegallery") return root.select("li").map { ul -> val img = ul.selectFirstOrThrow("img") @@ -130,15 +131,14 @@ class MangaInUaParser(override val context: MangaLoaderContext) : PagedMangaPars id = generateUid(url), url = url, preview = null, - referer = fullUrl, source = source, ) } } override suspend fun getTags(): Set { - val domain = getDomain() - val doc = context.httpGet("https://$domain/mangas").parseHtml() + val domain = domain + val doc = webClient.httpGet("https://$domain/mangas").parseHtml() val root = doc.body().requireElementById("menu_1").selectFirstOrThrow("div.menu__wrapper") return root.select("li").mapNotNullToSet { li -> val a = li.selectFirst("a") ?: return@mapNotNullToSet null @@ -149,4 +149,4 @@ class MangaInUaParser(override val context: MangaLoaderContext) : PagedMangaPars ) } } -} \ No newline at end of file +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaOwlParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaOwlParser.kt deleted file mode 100644 index 06dbf9fb..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaOwlParser.kt +++ /dev/null @@ -1,187 +0,0 @@ -package org.koitharu.kotatsu.parsers.site - -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaParser -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.util.* -import java.text.SimpleDateFormat -import java.util.* - -@Deprecated("Permanently shutdown") -@MangaSourceParser("MANGAOWL", "MangaOwl", "en") -internal class MangaOwlParser(override val context: MangaLoaderContext) : MangaParser(MangaSource.MANGAOWL) { - - override val configKeyDomain = ConfigKey.Domain("mangaowls.com", null) - - override val sortOrders: Set = EnumSet.of( - SortOrder.POPULARITY, - SortOrder.NEWEST, - SortOrder.UPDATED, - ) - - private val regexNsfw = Regex("(yaoi)|(yuri)|(smut)|(mature)|(adult)", RegexOption.IGNORE_CASE) - - override suspend fun getList( - offset: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - val page = (offset / 36f).toIntUp().inc() - val link = buildString { - append("https://") - append(getDomain()) - when { - !query.isNullOrEmpty() -> { - append("/search/$page?search=") - append(query.urlEncoded()) - } - - !tags.isNullOrEmpty() -> { - for (tag in tags) { - append(tag.key) - } - append("/$page?type=${getAlternativeSortKey(sortOrder)}") - } - - else -> { - append("/${getSortKey(sortOrder)}/$page") - } - } - } - val doc = context.httpGet(link).parseHtml() - val slides = doc.body().selectOrThrow("ul.slides") - val items = slides.select("div.col-md-2") - return items.mapNotNull { item -> - val href = item.selectFirst("h6 a")?.attrAsRelativeUrlOrNull("href") ?: return@mapNotNull null - Manga( - id = generateUid(href), - title = item.selectFirst("h6 a")?.text() ?: return@mapNotNull null, - coverUrl = item.select("div.img-responsive").attr("abs:data-background-image"), - altTitle = null, - author = null, - rating = runCatching { - item.selectFirst("div.block-stars") - ?.text() - ?.toFloatOrNull() - ?.div(10f) - }.getOrNull() ?: RATING_UNKNOWN, - url = href, - isNsfw = false, - tags = emptySet(), - state = null, - publicUrl = href.toAbsoluteUrl(getDomain()), - source = source, - ) - } - } - - override suspend fun getDetails(manga: Manga): Manga { - val doc = context.httpGet(manga.publicUrl).parseHtml() - val info = doc.body().selectFirstOrThrow("div.single_detail") - val table = doc.body().selectFirstOrThrow("div.single-grid-right") - val dateFormat = SimpleDateFormat("MM/dd/yyyy", Locale.US) - val trRegex = "window\\['tr'] = '([^']*)';".toRegex(RegexOption.IGNORE_CASE) - val trElement = doc.getElementsByTag("script").find { trRegex.find(it.data()) != null } - ?: doc.parseFailed("Oops, tr not found") - val tr = trRegex.find(trElement.data())!!.groups[1]!!.value - val s = context.encodeBase64(getDomain().toByteArray()) - var isNsfw = manga.isNsfw - val parsedTags = info.select("div.col-xs-12.col-md-8.single-right-grid-right > p > a[href*=genres]") - .mapNotNullToSet { - val a = it.selectFirst("a") ?: return@mapNotNullToSet null - val name = a.text() - if (!isNsfw && isNsfwGenre(name)) { - isNsfw = true - } - MangaTag( - title = name.toTitleCase(), - key = a.attr("href"), - source = source, - ) - } - return manga.copy( - description = info.selectFirst(".description")?.html(), - largeCoverUrl = info.select("img").first()?.let { img -> - if (img.hasAttr("data-src")) img.attr("abs:data-src") else img.attr("abs:src") - }, - isNsfw = isNsfw, - author = info.selectFirst("p.fexi_header_para a.author_link")?.text(), - state = parseStatus(info.select("p.fexi_header_para:contains(status)").first()?.ownText()), - tags = manga.tags + parsedTags, - chapters = table.select("div.table.table-chapter-list").select("li.list-group-item.chapter_list") - .asReversed().mapChapters { i, li -> - val a = li.select("a") - val href = a.attr("data-href").ifEmpty { - li.parseFailed("Link is missing") - } - MangaChapter( - id = generateUid(href), - name = a.select("label").text(), - number = i + 1, - url = "$href?tr=$tr&s=$s", - scanlator = null, - branch = null, - uploadDate = dateFormat.tryParse(li.selectFirst("small:last-of-type")?.text()), - source = MangaSource.MANGAOWL, - ) - }, - ) - } - - override suspend fun getPages(chapter: MangaChapter): List { - val fullUrl = chapter.url.toAbsoluteUrl(getDomain()) - val doc = context.httpGet(fullUrl).parseHtml() - val root = doc.body().selectOrThrow("div.item img.owl-lazy") - return root.map { div -> - val url = div?.attrAsRelativeUrlOrNull("data-src") ?: doc.parseFailed("Page image not found") - MangaPage( - id = generateUid(url), - url = url, - preview = null, - referer = url, - source = MangaSource.MANGAOWL, - ) - } - } - - private fun parseStatus(status: String?) = when { - status == null -> null - status.contains("Ongoing") -> MangaState.ONGOING - status.contains("Completed") -> MangaState.FINISHED - else -> null - } - - override suspend fun getTags(): Set { - val doc = context.httpGet("https://${getDomain()}/").parseHtml() - val root = doc.body().select("ul.dropdown-menu.multi-column.columns-3").select("li") - return root.mapToSet { p -> - val a = p.selectFirstOrThrow("a") - MangaTag( - title = a.text().toTitleCase(), - key = a.attr("href"), - source = source, - ) - } - } - - private fun getSortKey(sortOrder: SortOrder) = - when (sortOrder) { - SortOrder.POPULARITY -> "popular" - SortOrder.NEWEST -> "new_release" - SortOrder.UPDATED -> "lastest" - else -> "lastest" - } - - private fun getAlternativeSortKey(sortOrder: SortOrder) = - when (sortOrder) { - SortOrder.POPULARITY -> "0" - SortOrder.NEWEST -> "2" - SortOrder.UPDATED -> "3" - else -> "3" - } - - private fun isNsfwGenre(name: String): Boolean = regexNsfw.containsMatchIn(name) -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaTownParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaTownParser.kt index 95623121..60f8de5e 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaTownParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaTownParser.kt @@ -11,7 +11,7 @@ import java.text.SimpleDateFormat import java.util.* @MangaSourceParser("MANGATOWN", "MangaTown", "en") -internal class MangaTownParser(override val context: MangaLoaderContext) : MangaParser(MangaSource.MANGATOWN) { +internal class MangaTownParser(context: MangaLoaderContext) : MangaParser(context, MangaSource.MANGATOWN) { override val configKeyDomain = ConfigKey.Domain("www.mangatown.com", null) @@ -42,18 +42,18 @@ internal class MangaTownParser(override val context: MangaLoaderContext) : Manga if (offset != 0) { return emptyList() } - "/search?name=${query.urlEncoded()}".toAbsoluteUrl(getDomain()) + "/search?name=${query.urlEncoded()}".toAbsoluteUrl(domain) } - tags.isNullOrEmpty() -> "/directory/$page.htm$sortKey".toAbsoluteUrl(getDomain()) - tags.size == 1 -> "/directory/${tags.first().key}/$page.htm$sortKey".toAbsoluteUrl(getDomain()) + tags.isNullOrEmpty() -> "/directory/$page.htm$sortKey".toAbsoluteUrl(domain) + tags.size == 1 -> "/directory/${tags.first().key}/$page.htm$sortKey".toAbsoluteUrl(domain) else -> tags.joinToString( - prefix = "/search?page=$page".toAbsoluteUrl(getDomain()), + prefix = "/search?page=$page".toAbsoluteUrl(domain), ) { tag -> "&genres[${tag.key}]=1" } } - val doc = context.httpGet(url).parseHtml() + val doc = webClient.httpGet(url).parseHtml() val root = doc.body().selectFirstOrThrow("ul.manga_pic_list") return root.select("li").mapNotNull { li -> val a = li.selectFirst("a.manga_cover") @@ -87,13 +87,13 @@ internal class MangaTownParser(override val context: MangaLoaderContext) : Manga }.orEmpty(), url = href, isNsfw = false, - publicUrl = href.toAbsoluteUrl(a.host ?: getDomain()), + publicUrl = href.toAbsoluteUrl(a.host ?: domain), ) } } override suspend fun getDetails(manga: Manga): Manga { - val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml() + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() val root = doc.body().selectFirstOrThrow("section.main") .selectFirstOrThrow("div.article_content") val info = root.selectFirst("div.detail_info")?.selectFirst("ul") @@ -135,8 +135,8 @@ internal class MangaTownParser(override val context: MangaLoaderContext) : Manga } override suspend fun getPages(chapter: MangaChapter): List { - val fullUrl = chapter.url.toAbsoluteUrl(getDomain()) - val doc = context.httpGet(fullUrl).parseHtml() + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() val root = doc.body().selectFirstOrThrow("div.page_select") return root.selectFirstOrThrow("select").selectOrThrow("option").mapNotNull { val href = it.attrAsRelativeUrlOrNull("value") @@ -147,19 +147,18 @@ internal class MangaTownParser(override val context: MangaLoaderContext) : Manga id = generateUid(href), url = href, preview = null, - referer = fullUrl, source = MangaSource.MANGATOWN, ) } } override suspend fun getPageUrl(page: MangaPage): String { - val doc = context.httpGet(page.url.toAbsoluteUrl(getDomain())).parseHtml() + val doc = webClient.httpGet(page.url.toAbsoluteUrl(domain)).parseHtml() return doc.requireElementById("image").attrAsAbsoluteUrl("src") } override suspend fun getTags(): Set { - val doc = context.httpGet("/directory/".toAbsoluteUrl(getDomain())).parseHtml() + val doc = webClient.httpGet("/directory/".toAbsoluteUrl(domain)).parseHtml() val root = doc.body().selectFirst("aside.right") ?.getElementsContainingOwnText("Genres") ?.first() @@ -188,7 +187,7 @@ internal class MangaTownParser(override val context: MangaLoaderContext) : Manga } private suspend fun bypassLicensedChapters(manga: Manga): List { - val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain("m"))).parseHtml() + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(getDomain("m"))).parseHtml() val list = doc.body().selectFirst("ul.detail-ch-list") ?: return emptyList() val dateFormat = SimpleDateFormat("MMM dd,yyyy", Locale.US) return list.select("li").asReversed().mapIndexedNotNull { i, li -> @@ -214,4 +213,4 @@ internal class MangaTownParser(override val context: MangaLoaderContext) : Manga } private fun String.parseTagKey() = split('/').findLast { regexTag matches it } -} \ No newline at end of file +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/Manhwa18Parser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/Manhwa18Parser.kt new file mode 100644 index 00000000..51673a03 --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/Manhwa18Parser.kt @@ -0,0 +1,196 @@ +package org.koitharu.kotatsu.parsers.site + +import androidx.collection.ArrayMap +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.PagedMangaParser +import org.koitharu.kotatsu.parsers.config.ConfigKey +import org.koitharu.kotatsu.parsers.model.* +import org.koitharu.kotatsu.parsers.util.* +import java.util.* + +@MangaSourceParser("MANHWA18", "Manhwa18", "en") +class Manhwa18Parser(context: MangaLoaderContext) : + PagedMangaParser(context, MangaSource.MANHWA18, pageSize = 20, searchPageSize = 20) { + + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("manhwa18.net", null) + + override val sortOrders: Set + get() = EnumSet.of(SortOrder.UPDATED, SortOrder.POPULARITY, SortOrder.ALPHABETICAL, SortOrder.NEWEST) + + private val tagsMap = SuspendLazy(::parseTags) + + override suspend fun getFavicons(): Favicons { + return Favicons( + listOf( + Favicon("https://${domain}/uploads/logos/logo-mini.png", 92, null), + ), + domain, + ) + } + + override suspend fun getDetails(manga: Manga): Manga { + val docs = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val cardInfoElement = docs.selectFirst(".card .manga-info") + val author = cardInfoElement?.selectFirst("b:contains(Author(s))")?.parent() + ?.select("a.btn") + ?.joinToString(", ") { it.text() } + val availableTags = tagsMap.get() + val tags = cardInfoElement?.selectFirst("b:contains(Genre(s))")?.parent() + ?.select("a.btn") + ?.mapNotNullToSet { availableTags[it.text().lowercase(Locale.ENGLISH)] } + val state = cardInfoElement?.selectFirst("b:contains(Status)")?.parent() + ?.selectFirst("a.btn") + ?.let { + when (it.text()) { + "On going" -> MangaState.ONGOING + "Completed" -> MangaState.FINISHED + else -> null + } + } + + return manga.copy( + altTitle = cardInfoElement?.selectFirst("b:contains(Other names)")?.parent()?.ownText()?.removePrefix(": "), + author = author, + description = docs.selectFirst(".series-summary .summary-content")?.html(), + tags = tags.orEmpty(), + state = state, + chapters = docs.select(".card-body > .list-chapters > a").mapChapters(reversed = true) { index, element -> + // attrAsRelativeUrl only return page url without the '/' + val chapterUrl = element.attrAsAbsoluteUrlOrNull("href")?.toRelativeUrl(domain) + ?: return@mapChapters null + val uploadDate = parseUploadDate(element.selectFirst(".chapter-time")?.text()) + MangaChapter( + id = generateUid(chapterUrl), + name = element.selectFirst(".chapter-name")?.text().orEmpty(), + number = index + 1, + url = chapterUrl, + scanlator = null, + uploadDate = uploadDate, + branch = null, + source = MangaSource.MANHWA18, + ) + }, + ) + } + + // 7 minutes ago + // 5 hours ago + // 2 days ago + // 2 weeks ago + // 4 years ago + private fun parseUploadDate(timeStr: String?): Long { + timeStr ?: return 0 + + val timeWords = timeStr.split(' ') + if (timeWords.size != 3) return 0 + val timeWord = timeWords[1] + val timeAmount = timeWords[0].toIntOrNull() ?: return 0 + val timeUnit = when (timeWord) { + "minute", "minutes" -> Calendar.MINUTE + "hour", "hours" -> Calendar.HOUR + "day", "days" -> Calendar.DAY_OF_YEAR + "week", "weeks" -> Calendar.WEEK_OF_YEAR + "month", "months" -> Calendar.MONTH + "year", "years" -> Calendar.YEAR + else -> return 0 + } + val cal = Calendar.getInstance() + cal.add(timeUnit, -timeAmount) + return cal.time.time + } + + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + val sortQuery = when (sortOrder) { + SortOrder.ALPHABETICAL -> "az" + SortOrder.POPULARITY -> "top" + SortOrder.UPDATED -> "update" + SortOrder.NEWEST -> "new" + else -> "" + } + + val tagQuery = tags?.joinToString(",") { it.key }.orEmpty() + val url = buildString { + append("https://") + append(domain) + append("/tim-kiem?page=") + append(page) + if (!query.isNullOrEmpty()) { + append("&q=") + append(query.urlEncoded()) + } + append("&accept_genres=$tagQuery") + append("&sort=") + append(sortQuery) + } + + val docs = webClient.httpGet(url).parseHtml() + + return docs.select(".card-body .thumb-item-flow") + .map { + val titleElement = it.selectFirstOrThrow(".thumb_attr.series-title > a") + val absUrl = titleElement.attrAsAbsoluteUrl("href") + Manga( + id = generateUid(absUrl.toRelativeUrl(domain)), + title = titleElement.text(), + altTitle = null, + url = absUrl.toRelativeUrl(domain), + publicUrl = absUrl, + rating = RATING_UNKNOWN, + isNsfw = true, + coverUrl = it.selectFirst("div.img-in-ratio")?.attrAsAbsoluteUrl("data-bg").orEmpty(), + tags = emptySet(), + state = null, + author = null, + largeCoverUrl = null, + description = null, + source = MangaSource.MANHWA18, + ) + } + } + + override suspend fun getPages(chapter: MangaChapter): List { + val chapterUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(chapterUrl).parseHtml() + return doc.requireElementById("chapter-content").select("img").mapNotNull { + val url = it.attrAsRelativeUrlOrNull("data-src") + ?: it.attrAsRelativeUrlOrNull("src") + ?: return@mapNotNull null + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = MangaSource.MANHWA18, + ) + } + } + + override suspend fun getTags(): Set { + return tagsMap.get().values.toSet() + } + + private suspend fun parseTags(): Map { + val doc = webClient.httpGet("https://$domain/tim-kiem?q=").parseHtml() + val list = doc.getElementsByAttribute("data-genre-id") + if (list.isEmpty()) { + return emptyMap() + } + val result = ArrayMap(list.size) + for (item in list) { + val id = item.attr("data-genre-id") + val name = item.text() + result[name.lowercase(Locale.ENGLISH)] = MangaTag( + title = name.toTitleCase(Locale.ENGLISH), + key = id, + source = source, + ) + } + return result + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NHentaiParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NHentaiParser.kt index 42170f49..8c1c53be 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NHentaiParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NHentaiParser.kt @@ -15,7 +15,7 @@ import java.text.SimpleDateFormat import java.util.* @MangaSourceParser("NHENTAI", "N-Hentai") -class NHentaiParser(override val context: MangaLoaderContext) : PagedMangaParser(MangaSource.NHENTAI, pageSize = 25) { +class NHentaiParser(context: MangaLoaderContext) : PagedMangaParser(context, MangaSource.NHENTAI, pageSize = 25) { override val configKeyDomain: ConfigKey.Domain get() = ConfigKey.Domain("nhentai.net", null) @@ -32,7 +32,7 @@ class NHentaiParser(override val context: MangaLoaderContext) : PagedMangaParser if (query.isNullOrEmpty() && tags != null && tags.size > 1) { return getListPage(page, buildQuery(tags), emptySet(), sortOrder) } - val domain = getDomain() + val domain = domain val url = buildString { append("https://") append(domain) @@ -66,7 +66,7 @@ class NHentaiParser(override val context: MangaLoaderContext) : PagedMangaParser } } } - val root = context.httpGet(url).parseHtml().body().requireElementById("content") + val root = webClient.httpGet(url).parseHtml().body().requireElementById("content") .selectLastOrThrow("div.index-container") val regexBrackets = Regex("\\[[^]]+]|\\([^)]+\\)") val regexSpaces = Regex("\\s+") @@ -99,8 +99,8 @@ class NHentaiParser(override val context: MangaLoaderContext) : PagedMangaParser } override suspend fun getDetails(manga: Manga): Manga { - val root = context.httpGet( - url = manga.url.toAbsoluteUrl(getDomain()), + val root = webClient.httpGet( + url = manga.url.toAbsoluteUrl(domain), ).parseHtml().body().requireElementById("bigcontainer") val img = root.requireElementById("cover").selectFirstOrThrow("img") val tagContainers = root.requireElementById("tags").select(".tag-container") @@ -135,8 +135,8 @@ class NHentaiParser(override val context: MangaLoaderContext) : PagedMangaParser } override suspend fun getPages(chapter: MangaChapter): List { - val url = chapter.url.toAbsoluteUrl(getDomain()) - val root = context.httpGet(url).parseHtml().requireElementById("thumbnail-container") + val url = chapter.url.toAbsoluteUrl(domain) + val root = webClient.httpGet(url).parseHtml().requireElementById("thumbnail-container") return root.select(".thumb-container").map { div -> val a = div.selectFirstOrThrow("a") val img = div.selectFirstOrThrow("img") @@ -144,7 +144,6 @@ class NHentaiParser(override val context: MangaLoaderContext) : PagedMangaParser MangaPage( id = generateUid(href), url = href, - referer = url, preview = img.attrAsAbsoluteUrlOrNull("data-src") ?: img.attrAsAbsoluteUrl("src"), source = source, @@ -153,7 +152,7 @@ class NHentaiParser(override val context: MangaLoaderContext) : PagedMangaParser } override suspend fun getPageUrl(page: MangaPage): String { - val root = context.httpGet(page.url.toAbsoluteUrl(getDomain())).parseHtml().body() + val root = webClient.httpGet(page.url.toAbsoluteUrl(domain)).parseHtml().body() .requireElementById("image-container") return root.selectFirstOrThrow("img").attrAsAbsoluteUrl("src") } @@ -168,7 +167,7 @@ class NHentaiParser(override val context: MangaLoaderContext) : PagedMangaParser } private suspend fun getTags(page: Int): Set { - val root = context.httpGet("https://${getDomain()}/tags/popular?page=$page").parseHtml().body() + val root = webClient.httpGet("https://${domain}/tags/popular?page=$page").parseHtml().body() .getElementById("tag-container") return root?.parseTags().orEmpty() } @@ -185,4 +184,4 @@ class NHentaiParser(override val context: MangaLoaderContext) : PagedMangaParser private fun buildQuery(tags: Collection) = tags.joinToString(separator = " ") { tag -> "tag:\"${tag.key}\"" } -} \ No newline at end of file +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NetTruyenParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NetTruyenParser.kt index d05b9bf9..a994989f 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NetTruyenParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NetTruyenParser.kt @@ -4,7 +4,9 @@ import androidx.collection.ArrayMap import androidx.collection.ArraySet import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock -import org.koitharu.kotatsu.parsers.* +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.PagedMangaParser import org.koitharu.kotatsu.parsers.config.ConfigKey import org.koitharu.kotatsu.parsers.exception.NotFoundException import org.koitharu.kotatsu.parsers.model.* @@ -13,11 +15,14 @@ import java.text.SimpleDateFormat import java.util.* @MangaSourceParser("NETTRUYEN", "NetTruyen", "vi") -class NetTruyenParser(override val context: MangaLoaderContext) : - PagedMangaParser(MangaSource.NETTRUYEN, pageSize = 36) { +class NetTruyenParser(context: MangaLoaderContext) : + PagedMangaParser(context, MangaSource.NETTRUYEN, pageSize = 36) { override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("www.nettruyenme.com", null) + get() = ConfigKey.Domain( + "www.nettruyento.com", + arrayOf("www.nettruyento.com", "nettruyento.com", "nettruyenin.com"), + ) override val sortOrders: Set get() = EnumSet.of(SortOrder.UPDATED, SortOrder.POPULARITY, SortOrder.NEWEST, SortOrder.RATING) @@ -27,13 +32,13 @@ class NetTruyenParser(override val context: MangaLoaderContext) : private var tagCache: ArrayMap? = null override suspend fun getDetails(manga: Manga): Manga { - val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml() + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() val rating = doc.selectFirst("span[itemprop=ratingValue]") ?.ownText() ?.toFloatOrNull() ?: 0f val chapterElements = doc.getElementById("nt_listchapter")?.select("ul > li") ?: doc.parseFailed() - val chapters = chapterElements.asReversed().mapChapters { index, element -> + val chapters = chapterElements.mapChapters(reversed = true) { index, element -> val a = element.selectFirst("div.chapter > a") ?: return@mapChapters null val relativeUrl = a.attrAsRelativeUrlOrNull("href") ?: return@mapChapters null val timeText = element.selectFirst("div.col-xs-4.text-center.no-wrap.small")?.text() @@ -89,6 +94,7 @@ class NetTruyenParser(override val context: MangaLoaderContext) : val currentYear = calendar.get(Calendar.YEAR).toString().takeLast(2) "$relativeDate/$currentYear" } + 3 -> relativeDate else -> return 0L } @@ -109,7 +115,7 @@ class NetTruyenParser(override val context: MangaLoaderContext) : val isSearching = !query.isNullOrEmpty() val url = buildString { append("https://") - append(getDomain()) + append(domain) if (isSearching) { append("/tim-truyen?keyword=") append(query!!.urlEncoded()) @@ -124,7 +130,7 @@ class NetTruyenParser(override val context: MangaLoaderContext) : } val response = if (isSearching) { - val result = runCatching { context.httpGet(url) } + val result = runCatchingCancellable { webClient.httpGet(url) } val exception = result.exceptionOrNull() if (exception is NotFoundException) { return emptyList() @@ -132,7 +138,7 @@ class NetTruyenParser(override val context: MangaLoaderContext) : result.getOrThrow() } else { - context.httpGet(url) + webClient.httpGet(url) } val itemsElements = response.parseHtml() @@ -155,7 +161,7 @@ class NetTruyenParser(override val context: MangaLoaderContext) : id = generateUid(slug), title = tooltipElement.selectFirst("div.title")?.text().orEmpty(), altTitle = null, - url = absUrl.toRelativeUrl(getDomain()), + url = absUrl.toRelativeUrl(domain), publicUrl = absUrl, rating = RATING_UNKNOWN, isNsfw = false, @@ -173,14 +179,13 @@ class NetTruyenParser(override val context: MangaLoaderContext) : override suspend fun getPages(chapter: MangaChapter): List { - val pageElements = context.httpGet(chapter.url.toAbsoluteUrl(getDomain())).parseHtml() + val pageElements = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() .select("div.reading-detail.box_doc > div img") return pageElements.map { element -> val url = element.attrAsAbsoluteUrl("data-original") MangaPage( id = generateUid(url), url = url, - referer = getDomain(), preview = null, source = source, ) @@ -199,7 +204,7 @@ class NetTruyenParser(override val context: MangaLoaderContext) : private suspend fun getOrCreateTagMap(): ArrayMap = mutex.withLock { tagCache?.let { return@withLock it } - val doc = context.httpGet("/tim-truyen-nang-cao".toAbsoluteUrl(getDomain())).parseHtml() + val doc = webClient.httpGet("/tim-truyen-nang-cao".toAbsoluteUrl(domain)).parseHtml() val tagItems = doc.select("div.genre-item") val result = ArrayMap(tagItems.size) for (item in tagItems) { diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NicovideoSeigaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NicovideoSeigaParser.kt index de8953e4..7be9b20a 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NicovideoSeigaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NicovideoSeigaParser.kt @@ -11,8 +11,8 @@ private const val STATUS_ONGOING = "連載" private const val STATUS_FINISHED = "完結" @MangaSourceParser("NICOVIDEO_SEIGA", "Nicovideo Seiga", "ja") -class NicovideoSeigaParser(override val context: MangaLoaderContext) : - MangaParser(MangaSource.NICOVIDEO_SEIGA), +class NicovideoSeigaParser(context: MangaLoaderContext) : + MangaParser(context, MangaSource.NICOVIDEO_SEIGA), MangaParserAuthProvider { override val authUrl: String @@ -24,7 +24,7 @@ class NicovideoSeigaParser(override val context: MangaLoaderContext) : } override suspend fun getUsername(): String { - val body = context.httpGet("https://${getDomain("app")}/my/apps").parseHtml().body() + val body = webClient.httpGet("https://${getDomain("app")}/my/apps").parseHtml().body() return body.selectFirst("#userinfo > div > div > strong")?.text() ?: throw AuthRequiredException(source) } @@ -48,12 +48,12 @@ class NicovideoSeigaParser(override val context: MangaLoaderContext) : !query.isNullOrEmpty() -> return if (offset == 0) getSearchList(query, page) else emptyList() tags.isNullOrEmpty() -> "https://$domain/manga/list?page=$page&sort=${getSortKey(sortOrder)}" tags.size == 1 -> "https://$domain/manga/list?category=${tags.first().key}&page=$page" + - "&sort=${getSortKey(sortOrder)}" + "&sort=${getSortKey(sortOrder)}" tags.size > 1 -> throw IllegalArgumentException("This source supports only 1 category") else -> "https://$domain/manga/list?page=$page&sort=${getSortKey(sortOrder)}" } - val doc = context.httpGet(url).parseHtml() + val doc = webClient.httpGet(url).parseHtml() val comicList = doc.body().select("#comic_list > ul > li") ?: doc.parseFailed("Container not found") val items = comicList.select("div > .description > div > div") return items.mapNotNull { item -> @@ -88,7 +88,7 @@ class NicovideoSeigaParser(override val context: MangaLoaderContext) : } override suspend fun getDetails(manga: Manga): Manga { - val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain("seiga"))).parseHtml() + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(getDomain("seiga"))).parseHtml() val contents = doc.body().selectFirstOrThrow("#contents") val statusText = contents .select("div.mg_work_detail > div > div:nth-child(2) > div.tip.content_status.status_series > span") @@ -122,7 +122,7 @@ class NicovideoSeigaParser(override val context: MangaLoaderContext) : override suspend fun getPages(chapter: MangaChapter): List { val fullUrl = chapter.url.toAbsoluteUrl(getDomain("seiga")) - val doc = context.httpGet(fullUrl).parseHtml() + val doc = webClient.httpGet(fullUrl).parseHtml() if (!doc.select("#login_manga").isEmpty()) throw AuthRequiredException(source) val root = doc.body().select("#page_contents > li") @@ -132,14 +132,13 @@ class NicovideoSeigaParser(override val context: MangaLoaderContext) : id = generateUid(url), url = url, preview = null, - referer = fullUrl, source = source, ) } } override suspend fun getTags(): Set { - val doc = context.httpGet("https://${getDomain("seiga")}/manga/list").parseHtml() + val doc = webClient.httpGet("https://${getDomain("seiga")}/manga/list").parseHtml() val root = doc.body().selectOrThrow("#mg_category_list > ul > li") return root.mapToSet { li -> val a = li.selectFirstOrThrow("a") @@ -153,7 +152,7 @@ class NicovideoSeigaParser(override val context: MangaLoaderContext) : private suspend fun getSearchList(query: String, page: Int): List { val domain = getDomain("seiga") - val doc = context.httpGet("https://$domain/manga/search/?q=$query&page=$page&sort=score").parseHtml() + val doc = webClient.httpGet("https://$domain/manga/search/?q=$query&page=$page&sort=score").parseHtml() val root = doc.body().select(".search_result__item") return root.mapNotNull { item -> val href = item.selectFirst(".search_result__item__thumbnail > a") @@ -182,4 +181,4 @@ class NicovideoSeigaParser(override val context: MangaLoaderContext) : SortOrder.UPDATED -> "manga_updated" else -> "manga_view" } -} \ No newline at end of file +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NineMangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NineMangaParser.kt index 0918ac86..cfffccbf 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NineMangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NineMangaParser.kt @@ -2,6 +2,9 @@ package org.koitharu.kotatsu.parsers.site import okhttp3.Headers import okhttp3.HttpUrl.Companion.toHttpUrl +import okhttp3.Interceptor +import okhttp3.Response +import okio.IOException import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaSourceParser import org.koitharu.kotatsu.parsers.PagedMangaParser @@ -12,25 +15,39 @@ import java.text.SimpleDateFormat import java.util.* internal abstract class NineMangaParser( - final override val context: MangaLoaderContext, + context: MangaLoaderContext, source: MangaSource, defaultDomain: String, -) : PagedMangaParser(source, pageSize = 26) { +) : PagedMangaParser(context, source, pageSize = 26), Interceptor { override val configKeyDomain = ConfigKey.Domain(defaultDomain, null) init { - context.cookieJar.insertCookies(getDomain(), "ninemanga_template_desk=yes") + context.cookieJar.insertCookies(domain, "ninemanga_template_desk=yes") } override val headers = Headers.Builder() .add("Accept-Language", "en-US;q=0.7,en;q=0.3") + .add("User-Agent", "Mozilla/5.0 (Android 13; Mobile; rv:68.0) Gecko/68.0 Firefox/109.0") .build() override val sortOrders: Set = Collections.singleton( SortOrder.POPULARITY, ) + override fun intercept(chain: Interceptor.Chain): Response { + val request = chain.request() + .newBuilder() + .removeHeader("Referer") + .build() + val response = chain.proceed(request) + val responseUrl = response.request.url + if (responseUrl.host != request.url.host) { + throw IOException("Unexpected redirect to ${responseUrl.host}") + } + return response + } + override suspend fun getListPage( page: Int, query: String?, @@ -39,7 +56,7 @@ internal abstract class NineMangaParser( ): List { val url = buildString { append("https://") - append(getDomain()) + append(domain) when { !query.isNullOrEmpty() -> { append("/search/?name_sel=&wd=") @@ -63,7 +80,7 @@ internal abstract class NineMangaParser( append(page) append(".html") } - val doc = context.httpGet(url, headers).parseHtml() + val doc = webClient.httpGet(url).parseHtml() val root = doc.body().selectFirst("ul.direlist") ?: doc.parseFailed("Cannot find root") val baseHost = root.baseUri().toHttpUrl().host @@ -91,9 +108,8 @@ internal abstract class NineMangaParser( } override suspend fun getDetails(manga: Manga): Manga { - val doc = context.httpGet( - manga.url.toAbsoluteUrl(getDomain()) + "?waring=1", - headers, + val doc = webClient.httpGet( + manga.url.toAbsoluteUrl(domain) + "?waring=1", ).parseHtml() val root = doc.body().selectFirstOrThrow("div.manga") val infoRoot = root.selectFirstOrThrow("div.bookintro") @@ -111,7 +127,7 @@ internal abstract class NineMangaParser( description = infoRoot.getElementsByAttributeValue("itemprop", "description").first() ?.html()?.substringAfter(""), chapters = root.selectFirst("div.chapterbox")?.select("ul.sub_vol_ul > li") - ?.asReversed()?.mapChapters { i, li -> + ?.mapChapters(reversed = true) { i, li -> val a = li.selectFirst("a.chapter_list_a") val href = a?.attrAsRelativeUrlOrNull("href") ?.replace("%20", " ") ?: li.parseFailed("Link not found") @@ -130,13 +146,12 @@ internal abstract class NineMangaParser( } override suspend fun getPages(chapter: MangaChapter): List { - val doc = context.httpGet(chapter.url.toAbsoluteUrl(getDomain()), headers).parseHtml() + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() return doc.body().getElementById("page")?.select("option")?.map { option -> val url = option.attr("value") MangaPage( id = generateUid(url), url = url, - referer = chapter.url.toAbsoluteUrl(getDomain()), preview = null, source = source, ) @@ -144,14 +159,14 @@ internal abstract class NineMangaParser( } override suspend fun getPageUrl(page: MangaPage): String { - val doc = context.httpGet(page.url.toAbsoluteUrl(getDomain()), headers).parseHtml() + val doc = webClient.httpGet(page.url.toAbsoluteUrl(domain)).parseHtml() val root = doc.body() return root.selectFirst("a.pic_download")?.absUrl("href") ?: doc.parseFailed("Page image not found") } override suspend fun getTags(): Set { - val doc = context.httpGet("https://${getDomain()}/search/?type=high", headers) + val doc = webClient.httpGet("https://${domain}/search/?type=high") .parseHtml() val root = doc.body().getElementById("search_form") return root?.select("li.cate_list")?.mapNotNullToSet { li -> @@ -257,4 +272,4 @@ internal abstract class NineMangaParser( MangaSource.NINEMANGA_FR, "fr.ninemanga.com", ) -} \ No newline at end of file +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NudeMoonParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NudeMoonParser.kt index 67520445..3b8a38d3 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NudeMoonParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NudeMoonParser.kt @@ -15,16 +15,19 @@ private const val MAX_THUMB_INDEX = 19 @MangaSourceParser("NUDEMOON", "Nude-Moon", "ru") internal class NudeMoonParser( - override val context: MangaLoaderContext, -) : MangaParser(MangaSource.NUDEMOON), MangaParserAuthProvider { + context: MangaLoaderContext, +) : MangaParser(context, MangaSource.NUDEMOON), MangaParserAuthProvider { - override val configKeyDomain = ConfigKey.Domain("nude-moon.net", null) + override val configKeyDomain = ConfigKey.Domain( + defaultValue = "nude-moon.org", + presetValues = arrayOf("nude-moon.org", "nude-moon.net"), + ) override val authUrl: String - get() = "https://${getDomain()}/index.php" + get() = "https://${domain}/index.php" override val isAuthorized: Boolean get() { - return context.cookieJar.getCookies(getDomain()).any { + return context.cookieJar.getCookies(domain).any { it.name == "fusion_user" } } @@ -37,7 +40,7 @@ internal class NudeMoonParser( init { context.cookieJar.insertCookies( - getDomain(), + domain, "NMfYa=1;", "nm_mobile=0;", ) @@ -49,7 +52,7 @@ internal class NudeMoonParser( tags: Set?, sortOrder: SortOrder, ): List { - val domain = getDomain() + val domain = domain val url = when { !query.isNullOrEmpty() -> "https://$domain/search?stext=${query.urlEncoded()}&rowstart=$offset" !tags.isNullOrEmpty() -> tags.joinToString( @@ -61,7 +64,7 @@ internal class NudeMoonParser( else -> "https://$domain/all_manga?${getSortKey(sortOrder)}&rowstart=$offset" } - val doc = context.httpGet(url).parseHtml() + val doc = webClient.httpGet(url).parseHtml() val root = doc.body().run { selectFirst("td.main-bg") ?: selectFirst("td.main-body") } ?: doc.parseFailed("Cannot find root") @@ -100,7 +103,7 @@ internal class NudeMoonParser( } override suspend fun getDetails(manga: Manga): Manga { - val body = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml().body() + val body = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml().body() val root = body.selectFirst("table.shoutbox") ?: body.parseFailed("Cannot find root") val info = root.select("div.tbl2") @@ -136,8 +139,8 @@ internal class NudeMoonParser( } override suspend fun getPages(chapter: MangaChapter): List { - val fullUrl = chapter.url.toAbsoluteUrl(getDomain()) - val doc = context.httpGet(fullUrl).parseHtml() + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() val mangaId = chapter.url.substringAfterLast('/').substringBefore('-').toIntOrNull() val script = doc.select("script").firstNotNullOfOrNull { @@ -154,7 +157,6 @@ internal class NudeMoonParser( MangaPage( id = generateUid(url), url = url, - referer = fullUrl, preview = if (i <= MAX_THUMB_INDEX && mangaId != null) { val part2 = url.substringBeforeLast('/') val part3 = url.substringAfterLast('/') @@ -169,8 +171,8 @@ internal class NudeMoonParser( } override suspend fun getTags(): Set { - val domain = getDomain() - val doc = context.httpGet("https://$domain/all_manga").parseHtml() + val domain = domain + val doc = webClient.httpGet("https://$domain/all_manga").parseHtml() val root = doc.body().getElementsContainingOwnText("Поиск манги по тегам") .firstOrNull()?.parents()?.find { it.tag().normalName() == "tbody" } ?.selectFirst("td.textbox")?.selectFirst("td.small") @@ -186,7 +188,7 @@ internal class NudeMoonParser( } override suspend fun getUsername(): String { - val body = context.httpGet("https://${getDomain()}/").parseHtml() + val body = webClient.httpGet("https://${domain}/").parseHtml() .body() return body .getElementsContainingOwnText("Профиль") @@ -202,10 +204,6 @@ internal class NudeMoonParser( } } - override fun getFaviconUrl(): String { - return "https://${getDomain()}/favicon.jpg" - } - private fun getSortKey(sortOrder: SortOrder) = when (sortOrder) { SortOrder.POPULARITY -> "views" @@ -225,4 +223,4 @@ internal class NudeMoonParser( val suffix = url.substringAfter('-').trimStart('-') return "$prefix-online-$suffix" } -} \ No newline at end of file +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/RemangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/RemangaParser.kt index a7e39e73..23b67594 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/RemangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/RemangaParser.kt @@ -1,7 +1,6 @@ package org.koitharu.kotatsu.parsers.site import okhttp3.Headers -import okhttp3.Response import org.json.JSONArray import org.json.JSONException import org.json.JSONObject @@ -10,13 +9,11 @@ import org.koitharu.kotatsu.parsers.MangaParserAuthProvider import org.koitharu.kotatsu.parsers.MangaSourceParser import org.koitharu.kotatsu.parsers.PagedMangaParser import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.exception.AuthRequiredException import org.koitharu.kotatsu.parsers.exception.ContentUnavailableException import org.koitharu.kotatsu.parsers.exception.ParseException import org.koitharu.kotatsu.parsers.model.* import org.koitharu.kotatsu.parsers.util.* import org.koitharu.kotatsu.parsers.util.json.* -import java.net.HttpURLConnection import java.net.URLDecoder import java.text.DateFormat import java.text.SimpleDateFormat @@ -28,12 +25,19 @@ private const val STATUS_FINISHED = 0 @MangaSourceParser("REMANGA", "Remanga", "ru") internal class RemangaParser( - override val context: MangaLoaderContext, -) : PagedMangaParser(MangaSource.REMANGA, PAGE_SIZE), MangaParserAuthProvider { + context: MangaLoaderContext, +) : PagedMangaParser(context, MangaSource.REMANGA, PAGE_SIZE), MangaParserAuthProvider { - override val configKeyDomain = ConfigKey.Domain("remanga.org", null) + private val baseHeaders = Headers.Builder() + .add("User-Agent", "Mozilla/5.0 (Android 13; Mobile; rv:68.0) Gecko/68.0 Firefox/109.0") + .build() + + override val headers + get() = getApiHeaders() + + override val configKeyDomain = ConfigKey.Domain("remanga.org", arrayOf("remanga.org", "реманга.орг")) override val authUrl: String - get() = "https://${getDomain()}/user/login" + get() = "https://${domain}/user/login" override val sortOrders: Set = EnumSet.of( SortOrder.UPDATED, @@ -44,7 +48,7 @@ internal class RemangaParser( override val isAuthorized: Boolean get() { - return context.cookieJar.getCookies(getDomain()).any { + return context.cookieJar.getCookies(domain).any { it.name == "user" } } @@ -58,7 +62,7 @@ internal class RemangaParser( sortOrder: SortOrder, ): List { copyCookies() - val domain = getDomain() + val domain = domain val urlBuilder = StringBuilder() .append("https://api.") .append(domain) @@ -78,7 +82,7 @@ internal class RemangaParser( .append(page) .append("&count=") .append(PAGE_SIZE) - val content = context.httpGet(urlBuilder.toString(), getApiHeaders()).parseJson() + val content = webClient.httpGet(urlBuilder.toString()).parseJson() .getJSONArray("content") return content.mapJSON { jo -> val url = "/manga/${jo.getString("dir")}" @@ -109,13 +113,12 @@ internal class RemangaParser( override suspend fun getDetails(manga: Manga): Manga { copyCookies() - val domain = getDomain() + val domain = domain val slug = manga.url.find(regexLastUrlPath) ?: throw ParseException("Cannot obtain slug from ${manga.url}", manga.publicUrl) - val data = context.httpGet( + val data = webClient.httpGet( url = "https://api.$domain/api/titles$slug/", - headers = getApiHeaders(), - ).handle401().parseJson() + ).parseJson() val content = try { data.getJSONObject("content") } catch (e: JSONException) { @@ -140,7 +143,10 @@ internal class RemangaParser( ) }, chapters = chapters.mapChapters { i, jo -> - if (jo.getBooleanOrDefault("is_paid", false)) { + if ( + jo.getBooleanOrDefault("is_paid", false) && + !jo.getBooleanOrDefault("is_bought", false) + ) { return@mapChapters null } val id = jo.getLong("id") @@ -149,7 +155,7 @@ internal class RemangaParser( MangaChapter( id = generateUid(id), url = "/api/titles/chapters/$id/", - number = chapters.size - i, + number = jo.getIntOrDefault("index", chapters.size - i), name = buildString { append("Том ") append(jo.optString("tome", "0")) @@ -171,9 +177,7 @@ internal class RemangaParser( } override suspend fun getPages(chapter: MangaChapter): List { - val referer = "https://${getDomain()}/" - val content = context.httpGet(chapter.url.toAbsoluteUrl(getDomain("api")), getApiHeaders()) - .handle401() + val content = webClient.httpGet(chapter.url.toAbsoluteUrl(getDomain("api"))) .parseJson() .getJSONObject("content") val pages = content.optJSONArray("pages") @@ -191,8 +195,8 @@ internal class RemangaParser( val result = ArrayList(pages.length()) for (i in 0 until pages.length()) { when (val item = pages.get(i)) { - is JSONObject -> result += parsePage(item, referer) - is JSONArray -> item.mapJSONTo(result) { parsePage(it, referer) } + is JSONObject -> result += parsePage(item) + is JSONArray -> item.mapJSONTo(result) { parsePage(it) } else -> throw ParseException("Unknown json item $item", chapter.url) } } @@ -200,8 +204,8 @@ internal class RemangaParser( } override suspend fun getTags(): Set { - val domain = getDomain() - val content = context.httpGet("https://api.$domain/api/forms/titles/?get=genres", getApiHeaders()) + val domain = domain + val content = webClient.httpGet("https://api.$domain/api/forms/titles/?get=genres") .parseJson().getJSONObject("content").getJSONArray("genres") return content.mapJSONToSet { jo -> MangaTag( @@ -213,24 +217,23 @@ internal class RemangaParser( } override suspend fun getUsername(): String { - val jo = context.httpGet( - url = "https://api.${getDomain()}/api/users/current/", - headers = getApiHeaders(), - ).handle401().parseJson() + val jo = webClient.httpGet( + url = "https://api.${domain}/api/users/current/", + ).parseJson() return jo.getJSONObject("content").getString("username") } - private fun getApiHeaders(): Headers? { - val userCookie = context.cookieJar.getCookies(getDomain()).find { + private fun getApiHeaders(): Headers { + val userCookie = context.cookieJar.getCookies(domain).find { it.name == "user" - } ?: return null + } ?: return baseHeaders val jo = JSONObject(URLDecoder.decode(userCookie.value, Charsets.UTF_8.name())) - val accessToken = jo.getStringOrNull("access_token") ?: return null - return Headers.headersOf("authorization", "bearer $accessToken") + val accessToken = jo.getStringOrNull("access_token") ?: return baseHeaders + return baseHeaders.newBuilder().add("authorization", "bearer $accessToken").build() } private fun copyCookies() { - val domain = getDomain() + val domain = domain context.cookieJar.copyCookies(domain, "api.$domain") } @@ -242,11 +245,10 @@ internal class RemangaParser( else -> "-chapter_date" } - private fun parsePage(jo: JSONObject, referer: String) = MangaPage( + private fun parsePage(jo: JSONObject) = MangaPage( id = generateUid(jo.getLong("id")), url = jo.getString("link"), preview = null, - referer = referer, source = source, ) @@ -254,10 +256,9 @@ internal class RemangaParser( val result = ArrayList(100) var page = 1 while (true) { - val content = context.httpGet( + val content = webClient.httpGet( url = "https://api.$domain/api/titles/chapters/?branch_id=$branchId&page=$page&count=100", - headers = getApiHeaders(), - ).handle401().parseJson().getJSONArray("content") + ).parseJson().getJSONArray("content") val len = content.length() if (len == 0) { break @@ -270,10 +271,4 @@ internal class RemangaParser( } return result } - - private fun Response.handle401() = apply { - if (code == HttpURLConnection.HTTP_UNAUTHORIZED) { - throw AuthRequiredException(source) - } - } -} \ No newline at end of file +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/TruyentranhLHParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/TruyentranhLHParser.kt new file mode 100644 index 00000000..f3d28e9b --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/TruyentranhLHParser.kt @@ -0,0 +1,153 @@ +package org.koitharu.kotatsu.parsers.site + +import androidx.collection.ArrayMap +import androidx.collection.ArraySet +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.PagedMangaParser +import org.koitharu.kotatsu.parsers.config.ConfigKey +import org.koitharu.kotatsu.parsers.model.* +import org.koitharu.kotatsu.parsers.util.* +import java.text.SimpleDateFormat +import java.util.* + +@MangaSourceParser("TRUYENTRANHLH", "TruyentranhLH", "vi") +class TruyentranhLHParser(context: MangaLoaderContext) : + PagedMangaParser(context, source = MangaSource.TRUYENTRANHLH, pageSize = 18) { + + override val configKeyDomain: ConfigKey.Domain = ConfigKey.Domain("truyentranhlh.net", null) + override val sortOrders: Set = EnumSet.allOf(SortOrder::class.java) + + private val mutex = Mutex() + private var tagCache: Map? = null + + override suspend fun getDetails(manga: Manga): Manga { + val docs = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val infoHeaderEl = docs.selectFirst("main.section-body") + val infoEl = docs.selectFirst("main.section-body .series-information") + val tags = infoEl?.select(".info-item:contains(Thể loại) > .info-value > a")?.mapNotNullToSet { + getOrCreateTagMap()[it.text().trim()] + } + val state = when (infoEl?.selectFirst(".info-item:contains(Tình trạng) > .info-value")?.text()) { + "Đang tiến hành" -> MangaState.ONGOING + "Đã hoàn thành" -> MangaState.FINISHED + else -> null + } + val rating = infoHeaderEl?.let { + val like = it.selectFirst("#like .block.feature-name")?.text()?.toIntOrNull() + val disLike = it.selectFirst("#dislike .block.feature-name")?.text()?.toIntOrNull() + when { + like == null || disLike == null -> RATING_UNKNOWN + like == 0 && disLike == 0 -> RATING_UNKNOWN + else -> like.toFloat() / (like + disLike) + } + } + val chapterDateFormat = SimpleDateFormat("dd/MM/yyyy", Locale.US) + + return manga.copy( + altTitle = infoEl?.selectFirst(".info-item:contains(Tên khác) > .info-value")?.text(), + author = infoEl?.select(".info-item:contains(Tác giả) > .info-value")?.joinToString { it.text() }, + tags = tags ?: emptySet(), + state = state, + rating = rating ?: RATING_UNKNOWN, + description = infoHeaderEl?.selectFirst(".series-summary .summary-content")?.html(), + chapters = docs.select("ul.list-chapters.at-series > a").mapChapters(reversed = true) { index, element -> + MangaChapter( + id = generateUid(element.attrAsRelativeUrl("href")), + name = element.selectFirst(".chapter-name")?.text()?.trim().orEmpty(), + number = index + 1, + url = element.attrAsRelativeUrl("href"), + scanlator = null, + uploadDate = chapterDateFormat.tryParse(element.selectFirst(".chapter-time")?.text()), + branch = null, + source = source, + ) + }, + ) + } + + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + val sortQuery = when (sortOrder) { + SortOrder.UPDATED -> "update" + SortOrder.NEWEST -> "new" + SortOrder.RATING -> "like" + SortOrder.POPULARITY -> "top" + SortOrder.ALPHABETICAL -> "az" + } + val url = urlBuilder().apply { + addPathSegment("tim-kiem") + addQueryParameter("sort", sortQuery) + addQueryParameter("page", page.toString()) + if (!query.isNullOrEmpty()) { + addQueryParameter("q", query) + } + if (!tags.isNullOrEmpty()) { + val tagsQuery = tags.joinToString(separator = ",") { it.key } + addEncodedQueryParameter("accept_genres", tagsQuery) + } + }.build() + + return webClient.httpGet(url).parseHtml() + .select(".container .card.card-dark .row > .thumb-item-flow") + .mapNotNull { + val a = it.selectFirstOrThrow(".thumb-wrapper a") + Manga( + id = generateUid(a.attrAsRelativeUrl("href")), + url = a.attrAsRelativeUrl("href"), + publicUrl = a.attrAsAbsoluteUrl("href"), + title = it.select(".thumb_attr.series-title").text(), + altTitle = null, + rating = RATING_UNKNOWN, + isNsfw = false, + coverUrl = a.selectFirst("div[data-bg]")?.attrAsAbsoluteUrl("data-bg").orEmpty(), + tags = emptySet(), + state = null, + author = null, + source = source, + ) + } + } + + override suspend fun getPages(chapter: MangaChapter): List { + val url = chapter.url.toAbsoluteUrl(domain) + return webClient.httpGet(url).parseHtml().select("#chapter-content > img").mapNotNull { + val imageUrl = it.attrAsRelativeUrlOrNull("data-src") + ?: it.attrAsRelativeUrlOrNull("src") + ?: return@mapNotNull null + MangaPage( + id = generateUid(imageUrl), + url = imageUrl, + preview = null, + source = source, + ) + } + } + + override suspend fun getTags(): Set { + return ArraySet(getOrCreateTagMap().values) + } + + private suspend fun getOrCreateTagMap(): Map = mutex.withLock { + tagCache?.let { return it } + val docs = webClient.httpGet("/tim-kiem".toAbsoluteUrl(domain)).parseHtml() + val tags = docs.select(".search-border-left .row > .search-gerne_item").mapNotNull { + MangaTag( + title = it.text().trim(), + key = it.selectFirst("label[data-genre-id]") + ?.attr("data-genre-id") + ?.trim() ?: return@mapNotNull null, + source = source, + ) + } + val tagMap = tags.associateByTo(ArrayMap(tags.size)) { it.title } + tagCache = tagMap + return tagMap + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/UnionMangasParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/UnionMangasParser.kt index 7e1eb643..b71af6f3 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/UnionMangasParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/UnionMangasParser.kt @@ -13,7 +13,7 @@ import java.text.SimpleDateFormat import java.util.* @MangaSourceParser("UNION_MANGAS", "Union Mangás", "pt") -class UnionMangasParser(override val context: MangaLoaderContext) : PagedMangaParser(MangaSource.UNION_MANGAS, 40) { +class UnionMangasParser(context: MangaLoaderContext) : PagedMangaParser(context, MangaSource.UNION_MANGAS, 40) { override val sortOrders = EnumSet.of( SortOrder.ALPHABETICAL, @@ -45,7 +45,7 @@ class UnionMangasParser(override val context: MangaLoaderContext) : PagedMangaPa else -> "visualizacoes" }, ).addPathSegment(page.toString()) - val doc = context.httpGet(url.build()).parseHtml() + val doc = webClient.httpGet(url.build()).parseHtml() val root = doc.selectFirstOrThrow("div.tamanho-bloco-perfil") return root.select(".lista-mangas-novos").map { div -> val a = div.selectFirstOrThrow("a") @@ -70,7 +70,7 @@ class UnionMangasParser(override val context: MangaLoaderContext) : PagedMangaPa } override suspend fun getDetails(manga: Manga): Manga { - val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml() + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() val root = doc.selectFirstOrThrow(".perfil-manga") val dateFormat = SimpleDateFormat("dd/MM/yyyy", Locale.ROOT) return manga.copy( @@ -90,7 +90,7 @@ class UnionMangasParser(override val context: MangaLoaderContext) : PagedMangaPa it.toMangaTag() } ?: manga.tags, isNsfw = root.selectFirst(".alert-danger")?.html()?.contains("18 anos") == true, - chapters = root.select("div.row.capitulos").asReversed().mapChapters { i, div -> + chapters = root.select("div.row.capitulos").mapChapters(reversed = true) { i, div -> val a = div.selectFirstOrThrow("a") val href = a.attrAsRelativeUrl("href") val title = a.text() @@ -111,8 +111,8 @@ class UnionMangasParser(override val context: MangaLoaderContext) : PagedMangaPa } override suspend fun getPages(chapter: MangaChapter): List { - val fullUrl = chapter.url.toAbsoluteUrl(getDomain()) - val doc = context.httpGet(fullUrl).parseHtml() + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() val root = doc.body().selectFirstOrThrow("article") return root.selectOrThrow("img[pag]").mapNotNull { img -> val href = img.attrAsRelativeUrl("src") @@ -122,7 +122,6 @@ class UnionMangasParser(override val context: MangaLoaderContext) : PagedMangaPa MangaPage( id = generateUid(href), url = href, - referer = fullUrl, preview = null, source = source, ) @@ -130,7 +129,7 @@ class UnionMangasParser(override val context: MangaLoaderContext) : PagedMangaPa } override suspend fun getTags(): Set { - val doc = context.httpGet(urlBuilder().addPathSegment("lista-mangas").build()).parseHtml() + val doc = webClient.httpGet(urlBuilder().addPathSegment("lista-mangas").build()).parseHtml() val ul = doc.body().selectFirstOrThrow(".nav-tabs").selectFirstOrThrow("ul.dropdown-menu") return ul.select("li").mapToSet { li -> li.selectFirstOrThrow("a").toMangaTag() @@ -138,8 +137,8 @@ class UnionMangasParser(override val context: MangaLoaderContext) : PagedMangaPa } private suspend fun search(query: String): List { - val domain = getDomain() - val json = context.httpGet( + val domain = domain + val json = webClient.httpGet( urlBuilder() .addPathSegments("assets/busca.php") .addQueryParameter("nomeManga", query) @@ -170,8 +169,8 @@ class UnionMangasParser(override val context: MangaLoaderContext) : PagedMangaPa } private fun Element.toMangaTag() = MangaTag( - title = text().toTitleCase(sourceLocale ?: Locale.ROOT), + title = text().toTitleCase(sourceLocale), key = attr("href").removeSuffix('/').substringAfterLast('/'), source = source, ) -} \ No newline at end of file +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/AllHentaiParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/AllHentaiParser.kt new file mode 100644 index 00000000..77808eff --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/AllHentaiParser.kt @@ -0,0 +1,53 @@ +package org.koitharu.kotatsu.parsers.site.grouple + +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.config.ConfigKey +import org.koitharu.kotatsu.parsers.exception.AuthRequiredException +import org.koitharu.kotatsu.parsers.exception.ParseException +import org.koitharu.kotatsu.parsers.model.MangaChapter +import org.koitharu.kotatsu.parsers.model.MangaPage +import org.koitharu.kotatsu.parsers.model.MangaSource +import org.koitharu.kotatsu.parsers.util.domain +import org.koitharu.kotatsu.parsers.util.parseFailed +import org.koitharu.kotatsu.parsers.util.parseHtml +import org.koitharu.kotatsu.parsers.util.urlEncoded + +@MangaSourceParser("ALLHENTAI", "ALlHentai", "ru") +internal class AllHentaiParser( + context: MangaLoaderContext, +) : GroupleParser(context, MangaSource.ALLHENTAI, 1) { + + override val configKeyDomain = ConfigKey.Domain( + "2023.allhen.online", + null, + ) + override val defaultIsNsfw = true + + override val authUrl: String + get() { + val targetUri = "https://${domain}/".urlEncoded() + return "https://qawa.org/internal/auth/login?targetUri=$targetUri&siteId=1" + } + + override suspend fun getPages(chapter: MangaChapter): List { + try { + return super.getPages(chapter) + } catch (e: ParseException) { + if (isAuthorized) { + throw e + } else { + throw AuthRequiredException(source) + } + } + } + + override suspend fun getUsername(): String { + val root = webClient.httpGet("https://qawa.org/").parseHtml().body() + val element = root.selectFirst("img.user-avatar") ?: throw AuthRequiredException(source) + val res = element.parent()?.text() + return if (res.isNullOrEmpty()) { + root.parseFailed("Cannot find username") + } else res + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt index 1851e2a8..12ab60b0 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt @@ -1,11 +1,19 @@ package org.koitharu.kotatsu.parsers.site.grouple +import kotlinx.coroutines.flow.channelFlow +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.launch import okhttp3.Headers import okhttp3.HttpUrl.Companion.toHttpUrl +import okhttp3.HttpUrl.Companion.toHttpUrlOrNull +import okhttp3.Interceptor import okhttp3.Response +import okhttp3.internal.headersContentLength import org.json.JSONArray +import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaParser import org.koitharu.kotatsu.parsers.MangaParserAuthProvider +import org.koitharu.kotatsu.parsers.config.ConfigKey import org.koitharu.kotatsu.parsers.exception.AuthRequiredException import org.koitharu.kotatsu.parsers.exception.ParseException import org.koitharu.kotatsu.parsers.model.* @@ -18,15 +26,25 @@ private const val PAGE_SIZE = 70 private const val PAGE_SIZE_SEARCH = 50 private const val NSFW_ALERT = "сексуальные сцены" private const val NOTHING_FOUND = "Ничего не найдено" +private const val MIN_IMAGE_SIZE = 1024L +private const val HEADER_ACCEPT = "Accept" internal abstract class GroupleParser( + context: MangaLoaderContext, source: MangaSource, - userAgent: String, private val siteId: Int, -) : MangaParser(source), MangaParserAuthProvider { +) : MangaParser(context, source), MangaParserAuthProvider, Interceptor { - override val headers = Headers.Builder() - .add("User-Agent", userAgent) + @Volatile + private var cachedPagesServer: String? = null + protected open val defaultIsNsfw = false + + private val userAgentKey = ConfigKey.UserAgent( + "Mozilla/5.0 (X11; U; UNICOS lcLinux; en-US) Gecko/20140730 (KHTML, like Gecko, Safari/419.3) Arora/0.8.0", + ) + + override val headers: Headers = Headers.Builder() + .add("User-Agent", config[userAgentKey]) .build() override val sortOrders: Set = EnumSet.of( @@ -38,12 +56,12 @@ internal abstract class GroupleParser( override val authUrl: String get() { - val targetUri = "https://${getDomain()}/".urlEncoded() + val targetUri = "https://${domain}/".urlEncoded() return "https://grouple.co/internal/auth/sso?siteId=$siteId&=targetUri=$targetUri" } override val isAuthorized: Boolean - get() = context.cookieJar.getCookies(getDomain()).any { it.name == "gwt" } + get() = context.cookieJar.getCookies(domain).any { it.name == "gwt" } override suspend fun getList( offset: Int, @@ -51,29 +69,26 @@ internal abstract class GroupleParser( tags: Set?, sortOrder: SortOrder, ): List { - val domain = getDomain() + val domain = domain val doc = when { - !query.isNullOrEmpty() -> context.httpPost( + !query.isNullOrEmpty() -> webClient.httpPost( "https://$domain/search", mapOf( "q" to query.urlEncoded(), "offset" to (offset upBy PAGE_SIZE_SEARCH).toString(), ), - headers, ) - tags.isNullOrEmpty() -> context.httpGet( + tags.isNullOrEmpty() -> webClient.httpGet( "https://$domain/list?sortType=${ getSortKey(sortOrder) }&offset=${offset upBy PAGE_SIZE}", - headers, ) - tags.size == 1 -> context.httpGet( + tags.size == 1 -> webClient.httpGet( "https://$domain/list/genre/${tags.first().key}?sortType=${ getSortKey(sortOrder) }&offset=${offset upBy PAGE_SIZE}", - headers, ) offset > 0 -> return emptyList() @@ -117,7 +132,7 @@ internal abstract class GroupleParser( ?.div(5f) }.getOrNull() ?: RATING_UNKNOWN, author = tileInfo?.selectFirst("a.person-link")?.text(), - isNsfw = false, + isNsfw = defaultIsNsfw, tags = runCatching { tileInfo?.select("a.element-link") ?.mapToSet { @@ -140,7 +155,7 @@ internal abstract class GroupleParser( } override suspend fun getDetails(manga: Manga): Manga { - val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain()), headers).parseHtml() + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).checkAuthRequired().parseHtml() val root = doc.body().getElementById("mangaBox")?.selectFirst("div.leftContent") ?: doc.parseFailed("Cannot find root") val dateFormat = SimpleDateFormat("dd.MM.yy", Locale.US) @@ -159,10 +174,10 @@ internal abstract class GroupleParser( ) }, author = root.selectFirst("a.person-link")?.text() ?: manga.author, - isNsfw = root.select(".alert-warning").any { it.ownText().contains(NSFW_ALERT) }, + isNsfw = manga.isNsfw || root.select(".alert-warning").any { it.ownText().contains(NSFW_ALERT) }, chapters = root.selectFirst("div.chapters-link")?.selectFirst("table") - ?.select("tr:has(td > a)")?.asReversed()?.mapChapters { i, tr -> - val a = tr.selectFirst("a") ?: return@mapChapters null + ?.select("tr:has(td > a)")?.mapChapters(reversed = true) { i, tr -> + val a = tr.selectFirst("a.chapter-link") ?: return@mapChapters null val href = a.attrAsRelativeUrl("href") var translators = "" val translatorElement = a.attr("title") @@ -176,7 +191,7 @@ internal abstract class GroupleParser( name = tr.selectFirst("a")?.text().orEmpty().removePrefix(manga.title).trim(), number = i + 1, url = href, - uploadDate = dateFormat.tryParse(tr.selectFirst("td.d-none")?.text()), + uploadDate = dateFormat.tryParse(tr.selectFirst("td.date")?.text()), scanlator = translators, source = source, branch = null, @@ -186,11 +201,13 @@ internal abstract class GroupleParser( } override suspend fun getPages(chapter: MangaChapter): List { - val doc = context.httpGet(chapter.url.toAbsoluteUrl(getDomain()) + "?mtr=1", headers).parseHtml() + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain) + "?mtr=1") + .checkAuthRequired() + .parseHtml() val scripts = doc.select("script") for (script in scripts) { val data = script.html() - val pos = data.indexOf("rm_h.initReader(") + val pos = data.indexOf("rm_h.readerInit( 0,") if (pos == -1) { continue } @@ -203,7 +220,7 @@ internal abstract class GroupleParser( } val ja = JSONArray("[$json]") val pages = ja.getJSONArray(1) - val servers = ja.getJSONArray(4).mapJSON { it.getString("path") } + val servers = ja.getJSONArray(3).mapJSON { it.getString("path") } val serversStr = servers.joinToString("|") return (0 until pages.length()).map { i -> val page = pages.getJSONArray(i) @@ -213,7 +230,6 @@ internal abstract class GroupleParser( id = generateUid(url), url = "$primaryServer|$serversStr|$url", preview = null, - referer = chapter.url, source = source, ) } @@ -223,21 +239,46 @@ internal abstract class GroupleParser( override suspend fun getPageUrl(page: MangaPage): String { val parts = page.url.split('|') + if (parts.size < 2) { + throw ParseException("No servers found for page", page.url) + } val path = parts.last() - val servers = parts.dropLast(1).toSet() - val headers = Headers.headersOf("Referer", page.referer) - for (server in servers) { - val url = server + path - if (tryHead(url, headers)) { + // fast path + cachedPagesServer?.let { host -> + val url = concatUrl("https://$host/", path) + if (tryHead(url)) { return url + } else { + cachedPagesServer = null + } + } + // slow path + val candidates = HashSet((parts.size - 1) * 2) + for (i in 0 until parts.size - 1) { + val server = parts[i].trim().ifEmpty { "https://$domain/" } + candidates.add(concatUrl(server, path)) + candidates.add(concatUrl(server, path.substringBeforeLast('?'))) + } + return try { + channelFlow { + for (url in candidates) { + launch { + if (tryHead(url)) { + send(url) + } + } + } + }.first().also { + cachedPagesServer = it.toHttpUrlOrNull()?.host } + } catch (e: NoSuchElementException) { + assert(false) { e.toString() } + candidates.random() } - val fallbackServer = servers.firstOrNull() ?: throw ParseException("Cannot find any page url", page.url) - return fallbackServer + path } override suspend fun getTags(): Set { - val doc = context.httpGet("https://${getDomain()}/list/genres/sort_name", headers).parseHtml() + val doc = webClient.httpGet("https://${domain}/list/genres/sort_name").parseHtml() val root = doc.body().getElementById("mangaBox")?.selectFirst("div.leftContent") ?.selectFirst("table.table") ?: doc.parseFailed("Cannot find root") return root.select("a.element-link").mapToSet { a -> @@ -250,7 +291,7 @@ internal abstract class GroupleParser( } override suspend fun getUsername(): String { - val root = context.httpGet("https://grouple.co/").parseHtml().body() + val root = webClient.httpGet("https://grouple.co/").parseHtml().body() val element = root.selectFirst("img.user-avatar") ?: throw AuthRequiredException(source) val res = element.parent()?.text() return if (res.isNullOrEmpty()) { @@ -258,6 +299,28 @@ internal abstract class GroupleParser( } else res } + override fun intercept(chain: Interceptor.Chain): Response { + val request = chain.request() + if (!request.header(HEADER_ACCEPT).isNullOrEmpty()) { + return chain.proceed(request) + } + val ext = request.url.pathSegments.lastOrNull()?.substringAfterLast('.', "")?.lowercase(Locale.ROOT) + return if (ext == "jpg" || ext == "jpeg" || ext == "png" || ext == "webp") { + chain.proceed( + request.newBuilder() + .header(HEADER_ACCEPT, "image/webp,image/png;q=0.9,image/jpeg,*/*;q=0.8") + .build(), + ) + } else { + chain.proceed(request) + } + } + + override fun onCreateConfig(keys: MutableCollection>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + private fun getSortKey(sortOrder: SortOrder) = when (sortOrder) { SortOrder.ALPHABETICAL -> "name" @@ -270,7 +333,7 @@ internal abstract class GroupleParser( private suspend fun advancedSearch(domain: String, tags: Set): Response { val url = "https://$domain/search/advanced" // Step 1: map catalog genres names to advanced-search genres ids - val tagsIndex = context.httpGet(url, headers).parseHtml() + val tagsIndex = webClient.httpGet(url).parseHtml() .body().selectFirst("form.search-form") ?.select("div.form-group") ?.get(1) ?: throw ParseException("Genres filter element not found", url) @@ -301,10 +364,19 @@ internal abstract class GroupleParser( payload["s_sale"] = "" payload["years"] = "1900,2099" payload["+"] = "Искать".urlEncoded() - return context.httpPost(url, payload, headers) + return webClient.httpPost(url, payload) } - private suspend fun tryHead(url: String, headers: Headers): Boolean = runCatching { - context.httpHead(url, headers).isSuccessful + private suspend fun tryHead(url: String): Boolean = runCatchingCancellable { + val response = webClient.httpHead(url) + response.isSuccessful && response.headersContentLength() >= MIN_IMAGE_SIZE }.getOrDefault(false) -} \ No newline at end of file + + private fun Response.checkAuthRequired(): Response { + val lastPathSegment = request.url.pathSegments.lastOrNull() ?: return this + if (lastPathSegment == "login") { + throw AuthRequiredException(source) + } + return this + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/MintMangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/MintMangaParser.kt index ecce1f71..fa3c5835 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/MintMangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/MintMangaParser.kt @@ -7,15 +7,12 @@ import org.koitharu.kotatsu.parsers.model.MangaSource @MangaSourceParser("MINTMANGA", "MintManga", "ru") internal class MintMangaParser( - override val context: MangaLoaderContext, -) : GroupleParser(MangaSource.MINTMANGA, "mintmangafun", 2) { + context: MangaLoaderContext, +) : GroupleParser(context, MangaSource.MINTMANGA, 2) { override val configKeyDomain = ConfigKey.Domain( "mintmanga.live", arrayOf("mintmanga.live", "mintmanga.com"), ) - override fun getFaviconUrl(): String { - return "https://resmm.rmr.rocks/static/apple-touch-icon-8fff291039c140493adb0c7ba81065ad.png" - } -} \ No newline at end of file +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/ReadmangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/ReadmangaParser.kt index 6ab26771..a68bfe63 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/ReadmangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/ReadmangaParser.kt @@ -7,15 +7,12 @@ import org.koitharu.kotatsu.parsers.model.MangaSource @MangaSourceParser("READMANGA_RU", "ReadManga", "ru") internal class ReadmangaParser( - override val context: MangaLoaderContext, -) : GroupleParser(MangaSource.READMANGA_RU, "readmangafun", 1) { + context: MangaLoaderContext, +) : GroupleParser(context, MangaSource.READMANGA_RU, 1) { override val configKeyDomain = ConfigKey.Domain( "readmanga.live", arrayOf("readmanga.io", "readmanga.live", "readmanga.me"), ) - override fun getFaviconUrl(): String { - return "https://resrm.rmr.rocks/static/apple-touch-icon-3162037c9df9f28dca0f9a4092cb0f65.png" - } -} \ No newline at end of file +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/SelfMangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/SelfMangaParser.kt index 3591eb71..00006135 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/SelfMangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/SelfMangaParser.kt @@ -7,12 +7,9 @@ import org.koitharu.kotatsu.parsers.model.MangaSource @MangaSourceParser("SELFMANGA", "SelfManga", "ru") internal class SelfMangaParser( - override val context: MangaLoaderContext, -) : GroupleParser(MangaSource.SELFMANGA, "selfmangafun", 3) { + context: MangaLoaderContext, +) : GroupleParser(context, MangaSource.SELFMANGA, 3) { override val configKeyDomain = ConfigKey.Domain("selfmanga.live", null) - override fun getFaviconUrl(): String { - return "https://ressm.rmr.rocks/static/apple-touch-icon-a769ea533d811b73ac3eedde658bb1d3.png" - } -} \ No newline at end of file +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Hentai4Free.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Hentai4Free.kt new file mode 100644 index 00000000..45b0de08 --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Hentai4Free.kt @@ -0,0 +1,69 @@ +package org.koitharu.kotatsu.parsers.site.madara + +import org.jsoup.nodes.Document +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.model.Manga +import org.koitharu.kotatsu.parsers.model.MangaChapter +import org.koitharu.kotatsu.parsers.model.MangaSource +import org.koitharu.kotatsu.parsers.model.MangaTag +import org.koitharu.kotatsu.parsers.util.* +import java.text.SimpleDateFormat +import java.util.* + +@MangaSourceParser("HENTAI_4FREE", "Hentai4Free", "en") +internal class Hentai4Free(context: MangaLoaderContext) : + MadaraParser(context, MangaSource.HENTAI_4FREE, "hentai4free.net") { + + override val tagPrefix = "hentai-tag/" + + override val isNsfwSource = true + + override suspend fun getTags(): Set { + val doc = webClient.httpGet("https://$domain/").parseHtml() + val body = doc.body() + val root1 = body.selectFirst("header")?.selectFirst("ul.second-menu") + val list = root1?.select("li").orEmpty() + val keySet = HashSet(list.size) + return list.mapNotNullToSet { li -> + val a = li.selectFirst("a") ?: return@mapNotNullToSet null + val href = a.attr("href").removeSuffix("/") + .substringAfterLast(tagPrefix, "") + if (href.isEmpty() || !keySet.add(href)) { + return@mapNotNullToSet null + } + MangaTag( + key = href, + title = a.ownText().trim().toTitleCase(), + source = source, + ) + } + } + + override suspend fun getChapters(manga: Manga, doc: Document): List { + val slug = manga.url.removeSuffix('/').substringAfterLast('/') + val doc2 = webClient.httpPost( + "https://$domain/hentai/$slug/ajax/chapters/", + mapOf(), + ).parseHtml() + val ul = doc2.body().selectFirstOrThrow("ul") + val dateFormat = SimpleDateFormat(datePattern, Locale.US) + return ul.select("li").mapChapters(reversed = true) { i, li -> + val a = li.selectFirst("a") + val href = a?.attrAsRelativeUrlOrNull("href") ?: li.parseFailed("Link is missing") + MangaChapter( + id = generateUid(href), + name = a.ownText(), + number = i + 1, + url = href, + uploadDate = parseChapterDate( + dateFormat, + li.selectFirst("span.chapter-release-date i")?.text(), + ), + source = source, + scanlator = null, + branch = null, + ) + } + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/IsekaiScanEuParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/IsekaiScanEuParser.kt new file mode 100644 index 00000000..9e2a099c --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/IsekaiScanEuParser.kt @@ -0,0 +1,81 @@ +package org.koitharu.kotatsu.parsers.site.madara + +import org.jsoup.nodes.Document +import org.jsoup.nodes.Element +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.model.Manga +import org.koitharu.kotatsu.parsers.model.MangaChapter +import org.koitharu.kotatsu.parsers.model.MangaSource +import org.koitharu.kotatsu.parsers.model.MangaTag +import org.koitharu.kotatsu.parsers.util.* +import java.text.SimpleDateFormat +import java.util.* + +@MangaSourceParser("ISEKAISCAN_EU", "IsekaiScan", "en") +internal class IsekaiScanEuParser(context: MangaLoaderContext) : + MadaraParser(context, MangaSource.ISEKAISCAN_EU, "isekaiscan.to") { + + override val datePattern = "MM/dd/yyyy" + + override suspend fun getChapters(manga: Manga, doc: Document): List { + doc.selectFirst("ul.version-chap")?.let { + return parseChapters(it) + } + val mangaId = doc.body().requireElementById("manga-chapters-holder").attr("data-id") + val ul = webClient.httpPost( + "https://${domain}/wp-admin/admin-ajax.php", + mapOf( + "action" to "manga_get_chapters", + "manga" to mangaId, + ), + ).parseHtml().body().selectFirstOrThrow("ul") + return parseChapters(ul) + } + + override suspend fun getTags(): Set { + val doc = webClient.httpGet("https://$domain/mangax/").parseHtml() + val body = doc.body() + val root1 = body.selectFirst("header")?.selectFirst("ul.second-menu") + val root2 = body.selectFirst("div.genres_wrap")?.selectFirst("ul.list-unstyled") + if (root1 == null && root2 == null) { + doc.parseFailed("Root not found") + } + val list = root1?.select("li").orEmpty() + root2?.select("li").orEmpty() + val keySet = HashSet(list.size) + return list.mapNotNullToSet { li -> + val a = li.selectFirst("a") ?: return@mapNotNullToSet null + val href = a.attr("href").removeSuffix("/") + .substringAfterLast(tagPrefix, "") + if (href.isEmpty() || !keySet.add(href)) { + return@mapNotNullToSet null + } + MangaTag( + key = href, + title = a.ownText().toTitleCase(Locale.ENGLISH), + source = source, + ) + } + } + + private fun parseChapters(ul: Element): List { + val dateFormat = SimpleDateFormat(datePattern, Locale.US) + return ul.select("li").mapChapters(reversed = true) { i, li -> + val a = li.selectFirst("a") + val href = a?.attrAsRelativeUrlOrNull("href") ?: li.parseFailed("Link is missing") + MangaChapter( + id = generateUid(href), + name = a.ownText(), + number = i + 1, + url = href, + uploadDate = parseChapterDate( + dateFormat, + li.selectFirst("span.chapter-release-date i")?.text(), + ), + source = source, + scanlator = null, + branch = null, + ) + } + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Madara5Parser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Madara5Parser.kt index 26072e61..6b7f23d6 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Madara5Parser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Madara5Parser.kt @@ -13,10 +13,10 @@ import java.text.SimpleDateFormat import java.util.* abstract class Madara5Parser @InternalParsersApi constructor( - override val context: MangaLoaderContext, + context: MangaLoaderContext, source: MangaSource, domain: String, -) : PagedMangaParser(source, pageSize = 22) { +) : PagedMangaParser(context, source, pageSize = 22) { protected open val datePattern = "MMMM dd, HH:mm" protected open val tagPrefix = "/mangas/" @@ -32,7 +32,7 @@ abstract class Madara5Parser @InternalParsersApi constructor( tags: Set?, sortOrder: SortOrder, ): List { - val domain = getDomain() + val domain = domain val url = buildString { append("https://") append(domain) @@ -50,7 +50,7 @@ abstract class Madara5Parser @InternalParsersApi constructor( append("&op=1&author=&artist=&page=") append(page) } - val root = context.httpGet(url).parseHtml().body().selectFirstOrThrow(".search-wrap") + val root = webClient.httpGet(url).parseHtml().body().selectFirstOrThrow(".search-wrap") return root.select(".c-tabs-item__content").map { div -> val a = div.selectFirstOrThrow("a") val img = div.selectLastOrThrow("img") @@ -82,7 +82,7 @@ abstract class Madara5Parser @InternalParsersApi constructor( } override suspend fun getDetails(manga: Manga): Manga { - val root = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml().body() + val root = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml().body() .selectFirstOrThrow(".site-content") val postContent = root.selectFirstOrThrow(".post-content") val tags = postContent.getElementsContainingOwnText("Genre") @@ -105,14 +105,13 @@ abstract class Madara5Parser @InternalParsersApi constructor( } override suspend fun getPages(chapter: MangaChapter): List { - val fullUrl = chapter.url.toAbsoluteUrl(getDomain()) - val doc = context.httpGet(fullUrl).parseHtml() + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() val arrayData = doc.getElementById("arraydata") ?: doc.parseFailed("#arraydata not found") return arrayData.html().split(',').map { url -> MangaPage( id = generateUid(url), url = url, - referer = fullUrl, preview = null, source = source, ) @@ -120,15 +119,15 @@ abstract class Madara5Parser @InternalParsersApi constructor( } override suspend fun getTags(): Set { - val doc = context.httpGet("http://${getDomain()}/").parseHtml().body() + val doc = webClient.httpGet("http://${domain}/").parseHtml().body() return doc.getElementsByAttributeValueContaining("href", tagPrefix) .mapToSet { it.asMangaTag() } } private suspend fun loadChapters(mangaId: Long): List { - val dateFormat = SimpleDateFormat(datePattern, sourceLocale ?: Locale.US) - val doc = context.httpGet("https://${getDomain()}/ajax-list-chapter?mangaID=$mangaId").parseHtml() - return doc.select("li.wp-manga-chapter").asReversed().mapChapters { i, li -> + val dateFormat = SimpleDateFormat(datePattern, sourceLocale) + val doc = webClient.httpGet("https://${domain}/ajax-list-chapter?mangaID=$mangaId").parseHtml() + return doc.select("li.wp-manga-chapter").mapChapters(reversed = true) { i, li -> val a = li.selectFirstOrThrow("a") val href = a.attrAsRelativeUrl("href") MangaChapter( @@ -166,7 +165,7 @@ abstract class Madara5Parser @InternalParsersApi constructor( parseFailed("Cannot find tableValue for node ${text()}") } - private fun String.asMangaState() = when (trim().lowercase(sourceLocale ?: Locale.US)) { + private fun String.asMangaState() = when (trim().lowercase(sourceLocale)) { "ongoing" -> MangaState.ONGOING "completed" -> MangaState.FINISHED else -> null @@ -180,8 +179,5 @@ abstract class Madara5Parser @InternalParsersApi constructor( ) @MangaSourceParser("MANGAOWLS", "BeautyManga", "en") - class BeautyManga(context: MangaLoaderContext) : Madara5Parser(context, MangaSource.MANGAOWLS, "beautymanga.com") { - - override fun getFaviconUrl() = "http://${getDomain()}/frontend/images/favico.png" - } + class BeautyManga(context: MangaLoaderContext) : Madara5Parser(context, MangaSource.MANGAOWLS, "beautymanga.com") } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Madara6Parser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Madara6Parser.kt index 5fa04a6c..3eb178fd 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Madara6Parser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Madara6Parser.kt @@ -7,7 +7,6 @@ import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.model.* import org.koitharu.kotatsu.parsers.util.* import java.text.SimpleDateFormat -import java.util.* internal abstract class Madara6Parser( context: MangaLoaderContext, @@ -20,7 +19,7 @@ internal abstract class Madara6Parser( override suspend fun getDetails(manga: Manga): Manga { return coroutineScope { val chapters = async { loadChapters(manga.url) } - val body = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml().body() + val body = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml().body() parseDetails(manga, body, chapters.await()) } } @@ -45,10 +44,10 @@ internal abstract class Madara6Parser( ) protected open suspend fun loadChapters(mangaUrl: String): List { - val url = mangaUrl.toAbsoluteUrl(getDomain()).removeSuffix('/') + "/ajax/chapters/" - val dateFormat = SimpleDateFormat(datePattern, sourceLocale ?: Locale.ROOT) - val doc = context.httpPost(url, emptyMap()).parseHtml() - return doc.select("li.wp-manga-chapter").asReversed().mapChapters { i, li -> + val url = mangaUrl.toAbsoluteUrl(domain).removeSuffix('/') + "/ajax/chapters/" + val dateFormat = SimpleDateFormat(datePattern, sourceLocale) + val doc = webClient.httpPost(url, emptyMap()).parseHtml() + return doc.select("li.wp-manga-chapter").mapChapters(reversed = true) { i, li -> val a = li.selectFirstOrThrow("a") val href = a.attrAsRelativeUrl("href") MangaChapter( @@ -67,4 +66,4 @@ internal abstract class Madara6Parser( } protected abstract fun parseDetails(manga: Manga, body: Element, chapters: List): Manga -} \ No newline at end of file +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt index 06cf018c..d89b4109 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt @@ -16,10 +16,10 @@ import java.text.SimpleDateFormat import java.util.* internal abstract class MadaraParser( - override val context: MangaLoaderContext, + context: MangaLoaderContext, source: MangaSource, domain: String, -) : PagedMangaParser(source, pageSize = 12) { +) : PagedMangaParser(context, source, pageSize = 12) { override val configKeyDomain = ConfigKey.Domain(domain, null) @@ -53,27 +53,29 @@ internal abstract class MadaraParser( } payload["vars[wp-manga-genre]"] = tag?.key.orEmpty() payload["vars[s]"] = query?.urlEncoded().orEmpty() - val doc = context.httpPost( - "https://${getDomain()}/wp-admin/admin-ajax.php", + val doc = webClient.httpPost( + "https://$domain/wp-admin/admin-ajax.php", payload, ).parseHtml() - return doc.select("div.row.c-tabs-item__content").map { div -> + return doc.select("div.row.c-tabs-item__content").ifEmpty { + doc.select("div.page-item-detail.manga") + }.map { div -> val href = div.selectFirst("a")?.attrAsRelativeUrlOrNull("href") ?: div.parseFailed("Link not found") - val summary = div.selectFirst(".tab-summary") + val summary = div.selectFirst(".tab-summary") ?: div.selectFirst(".item-summary") Manga( id = generateUid(href), url = href, - publicUrl = href.toAbsoluteUrl(div.host ?: getDomain()), + publicUrl = href.toAbsoluteUrl(div.host ?: domain), coverUrl = div.selectFirst("img")?.src().orEmpty(), title = (summary?.selectFirst("h3") ?: summary?.selectFirst("h4"))?.text().orEmpty(), altTitle = null, rating = div.selectFirst("span.total_votes")?.ownText() ?.toFloatOrNull()?.div(5f) ?: -1f, - tags = summary?.selectFirst(".mg_genres")?.select("a")?.mapToSet { a -> + tags = summary?.selectFirst(".mg_genres")?.select("a")?.mapNotNullToSet { a -> MangaTag( - key = a.attr("href").removeSuffix("/").substringAfterLast('/'), - title = a.text().toTitleCase(), + key = a.attr("href").removeSuffix('/').substringAfterLast('/'), + title = a.text().ifEmpty { return@mapNotNullToSet null }.toTitleCase(), source = source, ) }.orEmpty(), @@ -93,7 +95,7 @@ internal abstract class MadaraParser( } override suspend fun getTags(): Set { - val doc = context.httpGet("https://${getDomain()}/manga/").parseHtml() + val doc = webClient.httpGet("https://$domain/manga/").parseHtml() val body = doc.body() val root1 = body.selectFirst("header")?.selectFirst("ul.second-menu") val root2 = body.selectFirst("div.genres_wrap")?.selectFirst("ul.list-unstyled") @@ -120,8 +122,8 @@ internal abstract class MadaraParser( } override suspend fun getDetails(manga: Manga): Manga = coroutineScope { - val fullUrl = manga.url.toAbsoluteUrl(getDomain()) - val doc = context.httpGet(fullUrl).parseHtml() + val fullUrl = manga.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() val chaptersDeferred = async { getChapters(manga, doc) } val root = doc.body().selectFirst("div.profile-manga") ?.selectFirst("div.summary_content") @@ -151,8 +153,8 @@ internal abstract class MadaraParser( protected open suspend fun getChapters(manga: Manga, doc: Document): List { val root2 = doc.body().selectFirstOrThrow("div.content-area") .selectFirstOrThrow("div.c-page") - val dateFormat = SimpleDateFormat(datePattern, sourceLocale ?: Locale.US) - return root2.select("li").asReversed().mapChapters { i, li -> + val dateFormat = SimpleDateFormat(datePattern, sourceLocale) + return root2.select("li").mapChapters(reversed = true) { i, li -> val a = li.selectFirst("a") val href = a?.attrAsRelativeUrlOrNull("href") ?: li.parseFailed("Link is missing") MangaChapter( @@ -172,19 +174,18 @@ internal abstract class MadaraParser( } override suspend fun getPages(chapter: MangaChapter): List { - val fullUrl = chapter.url.toAbsoluteUrl(getDomain()) - val doc = context.httpGet(fullUrl).parseHtml() + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() val root = doc.body().selectFirst("div.main-col-inner") ?.selectFirst("div.reading-content") ?: throw ParseException("Root not found", fullUrl) return root.select("div.page-break").map { div -> val img = div.selectFirst("img") ?: div.parseFailed("Page image not found") - val url = img.src()?.toRelativeUrl(getDomain()) ?: div.parseFailed("Image src not found") + val url = img.src()?.toRelativeUrl(domain) ?: div.parseFailed("Image src not found") MangaPage( id = generateUid(url), url = url, preview = null, - referer = fullUrl, source = source, ) } @@ -283,57 +284,42 @@ internal abstract class MadaraParser( } } - private fun Element.src(): String? { - return absUrl("data-src").ifEmpty { - absUrl("src") - }.takeUnless { it.isEmpty() } + protected fun Element.src(): String? { + var result = absUrl("data-src") + if (result.isEmpty()) result = absUrl("data-cfsrc") + if (result.isEmpty()) result = absUrl("src") + return result.ifEmpty { null } } private fun createRequestTemplate() = ( - "action=madara_load_more&page=1&template=madara-core%2Fcontent%2Fcontent-search&vars%5Bs%5D=&vars%5B" + - "orderby%5D=meta_value_num&vars%5Bpaged%5D=1&vars%5Btemplate%5D=search&vars%5Bmeta_query" + - "%5D%5B0%5D%5Brelation%5D=AND&vars%5Bmeta_query%5D%5Brelation%5D=OR&vars%5Bpost_type" + - "%5D=wp-manga&vars%5Bpost_status%5D=publish&vars%5Bmeta_key%5D=_latest_update&vars%5Border" + - "%5D=desc&vars%5Bmanga_archives_item_layout%5D=default" - ).split('&') + "action=madara_load_more&page=1&template=madara-core%2Fcontent%2Fcontent-search&vars%5Bs%5D=&vars%5B" + + "orderby%5D=meta_value_num&vars%5Bpaged%5D=1&vars%5Btemplate%5D=search&vars%5Bmeta_query" + + "%5D%5B0%5D%5Brelation%5D=AND&vars%5Bmeta_query%5D%5Brelation%5D=OR&vars%5Bpost_type" + + "%5D=wp-manga&vars%5Bpost_status%5D=publish&vars%5Bmeta_key%5D=_latest_update&vars%5Border" + + "%5D=desc&vars%5Bmanga_archives_item_layout%5D=default" + ).split('&') .map { val pos = it.indexOf('=') it.substring(0, pos) to it.substring(pos + 1) }.toMutableMap() - @MangaSourceParser("MANGAREAD", "MangaRead", "en") - class MangaRead(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGAREAD, "www.mangaread.org") { - override val tagPrefix = "genres/" - override val datePattern = "dd.MM.yyyy" - } - @MangaSourceParser("MANGAWEEBS", "MangaWeebs", "en") class MangaWeebs(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGAWEEBS, "mangaweebs.in") { override val datePattern = "dd MMMM HH:mm" } - @MangaSourceParser("KINGMANGA", "KingManga", "en") - class KingManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.KINGMANGA, "king-manga.com") { + @MangaSourceParser("HACHIMANGA", "HachiManga", "ja") + class HachiManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.HACHIMANGA, "hachiraw.com") { - override fun getFaviconUrl(): String { - return "https://${getDomain()}/wp-content/uploads/2022/05/cropped-logo-king-02-03-32x32.png" - } + override val sourceLocale: Locale = Locale.ENGLISH } - @Deprecated("Not working, unknown reason, maybe shutdown or domain expired") - @MangaSourceParser("MANGAHATACHI", "MangahaTachi", "ja") - class MangahaTachi(context: MangaLoaderContext) : - MadaraParser(context, MangaSource.MANGAHATACHI, "mangahatachi.com") - @MangaSourceParser("PIANMANGA", "PianManga", "en") - class PianManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.PIANMANGA, "pianmanga.com") + class PianManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.PIANMANGA, "pianmanga.me") @MangaSourceParser("MANGAROSIE", "MangaRosie", "en") - class MangaRosie(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGAROSIE, "mangarosie.love") - - @MangaSourceParser("READMANWHA", "ReadManwha", "en") - class ReadManwha(context: MangaLoaderContext) : MadaraParser(context, MangaSource.READMANWHA, "readmanwha.net") + class MangaRosie(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGAROSIE, "mangarosie.in") @MangaSourceParser("MANGATX", "MangaTx", "en") class MangaTx(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGATX, "mangatx.com") @@ -344,23 +330,10 @@ internal abstract class MadaraParser( } @MangaSourceParser("AQUAMANGA", "AquaManga", "en") - class AquaManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.AQUAMANGA, "aquamanga.com") { - - override fun getFaviconUrl(): String { - return "https://${getDomain()}/wp-content/uploads/2021/03/cropped-cropped-favicon-1-32x32.png" - } - } - - @MangaSourceParser("MANGATX_OT", "MangaTx (ot)", "en") - class MangaTxOt(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGATX_OT, "manga-tx.com") + class AquaManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.AQUAMANGA, "aquamanga.com") @MangaSourceParser("MANGALEK", "MangaLek", "ar") - class MangaLek(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGALEK, "mangalek.com") { - - override fun getFaviconUrl(): String { - return "https://${getDomain()}/wp-content/logo192.png" - } - } + class MangaLek(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGALEK, "mangalek.com") @MangaSourceParser("HARIMANGA", "HariManga", "en") class HariManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.HARIMANGA, "harimanga.com") { @@ -376,26 +349,9 @@ internal abstract class MadaraParser( @MangaSourceParser("FREEMANGA", "FreeManga", "en") class FreeManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.FREEMANGA, "freemanga.me") - @MangaSourceParser("ISEKAISCAN", "IsekaiScan", "en") - class IsekaiScan(context: MangaLoaderContext) : MadaraParser(context, MangaSource.ISEKAISCAN, "isekaiscan.com") { - - override fun getFaviconUrl(): String { - return "https://${getDomain()}/wp-content/uploads/2021/10/isekai-scan-02-01-150x150.png" - } - } - @MangaSourceParser("MANGA_KOMI", "MangaKomi", "en") class MangaKomi(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGA_KOMI, "mangakomi.io") - @MangaSourceParser("READMANHWA", "ReadManhwa", "en") - class ReadManhwa(context: MangaLoaderContext) : MadaraParser(context, MangaSource.READMANHWA, "readmanhwa.info") - - @MangaSourceParser("NEATMANGA", "NeatManga", "en") - class NeatManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.NEATMANGA, "neatmanga.com") - - @MangaSourceParser("MANGAKIK", "MangaKik", "en") - class MangaKik(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGAKIK, "mangakik.net") - @MangaSourceParser("MANHWACLAN", "ManhwaClan", "en") class ManhwaClan(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANHWACLAN, "manhwaclan.com") @@ -406,9 +362,7 @@ internal abstract class MadaraParser( @MangaSourceParser("MANHWAKOOL", "Manhwa Kool", "en") class ManhwaKool(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANHWAKOOL, "manhwakool.com") { - override fun getFaviconUrl(): String { - return "https://${getDomain()}/wp-content/uploads/2021/10/cropped-logo-kool-32x32.jpeg" - } + override val datePattern: String = "MM/dd" } @MangaSourceParser("TOPMANHUA", "Top Manhua", "en") @@ -417,65 +371,29 @@ internal abstract class MadaraParser( override val datePattern = "MM/dd/yyyy" } - @MangaSourceParser("X2MANGA", "X2Manga", "en") - class X2Manga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.X2MANGA, "x2manga.com") - - @MangaSourceParser("VINLOAD", "VinLoad", "en") - class VinLoad(context: MangaLoaderContext) : MadaraParser(context, MangaSource.VINLOAD, "vinload.com") - @MangaSourceParser("S2MANGA", "S2Manga", "en") class S2Manga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.S2MANGA, "s2manga.com") - @MangaSourceParser("MANHWA", "Manhwa", "en") - class Manhwa(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANHWA, "manhwa.pics") - @MangaSourceParser("SKY_MANGA", "Sky Manga", "en") class SkyManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.SKY_MANGA, "skymanga.xyz") { override val isNsfwSource = true - override fun getFaviconUrl(): String { - return "https://${getDomain()}/wp-content/uploads/cropped-sky-tv-1-32x32.png" - } } - @MangaSourceParser("MANGA_DISTRICT", "Manga District", "en") - class MangaDistrict(context: MangaLoaderContext) : - MadaraParser(context, MangaSource.MANGA_DISTRICT, "mangadistrict.com") { - - override val tagPrefix = "publication-genre/" + @MangaSourceParser("BAKAMAN", "BakaMan", "th") + class BakaMan(context: MangaLoaderContext) : MadaraParser(context, MangaSource.BAKAMAN, "bakaman.net") { - override val isNsfwSource = true + override val isNsfwSource = false } - @MangaSourceParser("HENTAI_4FREE", "Hentai4Free", "en") - class Hentai4Free(context: MangaLoaderContext) : - MadaraParser(context, MangaSource.HENTAI_4FREE, "hentai4free.net") { + @MangaSourceParser("HENTAI20", "Hentai20", "en") + class Hentai20(context: MangaLoaderContext) : + MadaraParser(context, MangaSource.HENTAI20, "hentai20.io") { - override val tagPrefix = "hentai-tag/" + override val tagPrefix = "manga-genre/" override val isNsfwSource = true - - override suspend fun getTags(): Set { - val doc = context.httpGet("https://${getDomain()}/").parseHtml() - val body = doc.body() - val root1 = body.selectFirst("header")?.selectFirst("ul.second-menu") - val list = root1?.select("li").orEmpty() - val keySet = HashSet(list.size) - return list.mapNotNullToSet { li -> - val a = li.selectFirst("a") ?: return@mapNotNullToSet null - val href = a.attr("href").removeSuffix("/") - .substringAfterLast(tagPrefix, "") - if (href.isEmpty() || !keySet.add(href)) { - return@mapNotNullToSet null - } - MangaTag( - key = href, - title = a.ownText().trim().toTitleCase(), - source = source, - ) - } - } } @MangaSourceParser("ALLPORN_COMIC", "All Porn Comic", "en") @@ -486,9 +404,6 @@ internal abstract class MadaraParser( override val isNsfwSource = true - override fun getFaviconUrl(): String { - return "https://cdn.${getDomain()}/wp-content/uploads/2019/01/cropped-cropped-pcround-32x32.png" - } } @MangaSourceParser("CAT_300", "Cat300", "th") @@ -500,36 +415,25 @@ internal abstract class MadaraParser( @MangaSourceParser("BIBIMANGA", "BibiManga", "en") class BibiManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.BIBIMANGA, "bibimanga.com") { - override val isNsfwSource = true + override val isNsfwSource = false } - @MangaSourceParser("MANHWA_CHILL", "Manhwa Chill", "en") - class ManhwaChill(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANHWA_CHILL, "manhwachill.me") - @MangaSourceParser("TREE_MANGA", "Tree Manga", "en") class TreeManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.TREE_MANGA, "treemanga.com") { override val datePattern = "MM/dd/yyyy" - override fun getFaviconUrl(): String { - return "https://${getDomain()}/wp-content/uploads/2017/10/lgoo-treemanga-2-1.jpg" - } } - @MangaSourceParser("ALLTOPMANGA", "All Top Manga", "en") - class AllTopManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.ALLTOPMANGA, "alltopmanga.com") { + @MangaSourceParser("MANGACV", "Manga Cv", "en") + class MangaCv(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGACV, "mangacv.com") - override fun getFaviconUrl(): String { - return "https://${getDomain()}/wp-content/uploads/2021/12/cropped-Screenshot_4-removebg-preview-32x32.png" - } - } + @MangaSourceParser("TOONILY", "Toonily", "en") + class Toonily(context: MangaLoaderContext) : MadaraParser(context, MangaSource.TOONILY, "toonily.com") { - @MangaSourceParser("MANGACV", "Manga Cv", "en") - class MangaCv(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGACV, "mangacv.com") { + override val tagPrefix = "webtoon-genre/" - override fun getFaviconUrl(): String { - return "https://${getDomain()}/wp-content/uploads/2020/10/cropped-mangaCV-32x32.png" - } + override val isNsfwSource = false } @MangaSourceParser("MANGA_MANHUA", "Manga Manhua", "en") @@ -551,4 +455,4 @@ internal abstract class MadaraParser( @MangaSourceParser("ZINMANGA", "ZINMANGA", "en") class Zinmanga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.ZINMANGA, "zinmanga.com") -} \ No newline at end of file +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/IsekaiScanEu.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangaDistrict.kt similarity index 62% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/IsekaiScanEu.kt rename to src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangaDistrict.kt index f2b2c6b9..6a97a0d7 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/IsekaiScanEu.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangaDistrict.kt @@ -10,23 +10,25 @@ import org.koitharu.kotatsu.parsers.util.* import java.text.SimpleDateFormat import java.util.* -@MangaSourceParser("ISEKAISCAN_EU", "IsekaiScan (eu)", "en") -internal class IsekaiScanEuParser(context: MangaLoaderContext) : - MadaraParser(context, MangaSource.ISEKAISCAN_EU, "isekaiscan.eu") { +@MangaSourceParser("MANGA_DISTRICT", "Manga District", "en") +internal class MangaDistrict(context: MangaLoaderContext) : + MadaraParser(context, MangaSource.MANGA_DISTRICT, "mangadistrict.com") { - override val datePattern = "MM/dd/yyyy" + override val tagPrefix = "publication-genre/" + + override val isNsfwSource = true + + override val datePattern = "MMM dd,yyyy" override suspend fun getChapters(manga: Manga, doc: Document): List { - val mangaId = doc.body().requireElementById("manga-chapters-holder").attr("data-id") - val ul = context.httpPost( - "https://${getDomain()}/wp-admin/admin-ajax.php", - mapOf( - "action" to "manga_get_chapters", - "manga" to mangaId, - ), - ).parseHtml().body().selectFirstOrThrow("ul") + val slug = manga.url.removeSuffix('/').substringAfterLast('/') + val doc2 = webClient.httpPost( + "https://$domain/read-scan/$slug/ajax/chapters/", + mapOf(), + ).parseHtml() + val ul = doc2.body().selectFirstOrThrow("ul") val dateFormat = SimpleDateFormat(datePattern, Locale.US) - return ul.select("li").asReversed().mapChapters { i, li -> + return ul.select("li").mapChapters(reversed = true) { i, li -> val a = li.selectFirst("a") val href = a?.attrAsRelativeUrlOrNull("href") ?: li.parseFailed("Link is missing") MangaChapter( @@ -44,4 +46,4 @@ internal class IsekaiScanEuParser(context: MangaLoaderContext) : ) } } -} \ No newline at end of file +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangaRead.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangaRead.kt new file mode 100644 index 00000000..d376dfaf --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangaRead.kt @@ -0,0 +1,120 @@ +package org.koitharu.kotatsu.parsers.site.madara + +import androidx.collection.arraySetOf +import org.jsoup.nodes.Element +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.model.* +import org.koitharu.kotatsu.parsers.util.* +import java.util.* + +@MangaSourceParser("MANGAREAD", "MangaRead", "en") +internal class MangaRead(context: MangaLoaderContext) : + MadaraParser(context, MangaSource.MANGAREAD, "www.mangaread.org") { + + override val tagPrefix = "genres/" + override val datePattern = "dd.MM.yyyy" + private val nsfwTags = arraySetOf("yaoi", "yuri", "mature") + + init { + paginator.firstPage = 1 + searchPaginator.firstPage = 1 + } + + override val sortOrders: Set = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.ALPHABETICAL, + SortOrder.RATING, + SortOrder.POPULARITY, + ) + + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + val url = urlBuilder() + .addPathSegment("page") + .addPathSegment(page.toString()) + .addQueryParameter("s", query.orEmpty()) + .addQueryParameter("post_type", "wp-manga") + tags?.forEachIndexed { index, tag -> + url.addQueryParameter("genre[$index]", tag.key) + } + url.addQueryParameter("op", "") + .addQueryParameter("author", "") + .addQueryParameter("artist", "") + .addQueryParameter("release", "") + .addQueryParameter("adult", "") + if (query.isNullOrEmpty()) { + url.addQueryParameter( + "&m_orderby=", + when (sortOrder) { + SortOrder.RATING -> "trending" + SortOrder.ALPHABETICAL -> "alphabet" + SortOrder.POPULARITY -> "views" + SortOrder.NEWEST -> "new-manga" + SortOrder.UPDATED -> "latest" + }, + ) + } + val root = webClient.httpGet(url.build()).parseHtml().body().selectFirstOrThrow(".search-wrap") + return root.select(".c-tabs-item__content").map { div -> + val a = div.selectFirstOrThrow("a") + val img = div.selectLastOrThrow("img") + val href = a.attrAsRelativeUrl("href") + val postContent = root.selectFirstOrThrow(".post-content") + val tagSet = postContent.getElementsContainingOwnText("Genre") + .firstOrNull()?.tableValue() + ?.getElementsByAttributeValueContaining("href", tagPrefix) + ?.mapToSet { it.asMangaTag() }.orEmpty() + Manga( + id = generateUid(href), + title = a.attr("title"), + altTitle = postContent.getElementsContainingOwnText("Alternative") + .firstOrNull()?.tableValue()?.text()?.trim(), + url = href, + publicUrl = a.attrAsAbsoluteUrl("href"), + coverUrl = img.src().orEmpty(), + author = postContent.getElementsContainingOwnText("Author") + .firstOrNull()?.tableValue()?.text()?.trim(), + state = postContent.getElementsContainingOwnText("Status") + .firstOrNull()?.tableValue()?.text()?.asMangaState(), + isNsfw = isNsfw(tagSet), + rating = div.selectFirstOrThrow(".score").text() + .toFloatOrNull()?.div(5f) ?: RATING_UNKNOWN, + tags = tagSet, + source = source, + ) + } + } + + private fun Element.tableValue(): Element { + for (p in parents()) { + val children = p.children() + if (children.size == 2) { + return children[1] + } + } + parseFailed("Cannot find tableValue for node ${text()}") + } + + private fun isNsfw(tags: Set): Boolean { + return tags.any { it.key in nsfwTags } + } + + private fun String.asMangaState() = when (trim().lowercase(sourceLocale)) { + "ongoing" -> MangaState.ONGOING + "completed" -> MangaState.FINISHED + else -> null + } + + private fun Element.asMangaTag() = MangaTag( + title = ownText(), + key = attr("href").removeSuffix('/').substringAfterLast('/') + .replace('-', '+'), + source = source, + ) +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangalinkParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangalinkParser.kt new file mode 100644 index 00000000..25404f35 --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangalinkParser.kt @@ -0,0 +1,46 @@ +package org.koitharu.kotatsu.parsers.site.madara + +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.exception.ParseException +import org.koitharu.kotatsu.parsers.model.Manga +import org.koitharu.kotatsu.parsers.model.MangaSource +import org.koitharu.kotatsu.parsers.model.MangaTag +import org.koitharu.kotatsu.parsers.util.* + +@MangaSourceParser("MANGALINK_AR", "Mangalink", "ar") +internal class MangalinkParser(context: MangaLoaderContext) : + MadaraParser(context, MangaSource.MANGALINK_AR, "mangalink.online") { + + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { + val fullUrl = manga.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + val chaptersDeferred = async { getChapters(manga, doc) } + val root = doc.body().selectFirst("div.profile-manga") + ?.selectFirst("div.summary_content") + ?.selectFirst("div.post-content") + ?: throw ParseException("Root not found", fullUrl) + val root2 = doc.body().selectFirst("div.content-area") + ?.selectFirst("div.c-page") + ?: throw ParseException("Root2 not found", fullUrl) + manga.copy( + tags = root.selectFirst("div.genres-content")?.select("a") + ?.mapNotNullToSet { a -> + MangaTag( + key = a.attr("href").removeSuffix("/").substringAfterLast('/'), + title = a.text().toTitleCase(), + source = source, + ) + } ?: manga.tags, + description = root2.selectFirst("div.description-summary") + ?.selectFirst("div.summary__content") + ?.select("p") + ?.filterNot { it.ownText().startsWith("A brief description") } + ?.joinToString { it.html() }, + chapters = chaptersDeferred.await(), + ) + } + +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangasOriginesParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangasOriginesParser.kt deleted file mode 100644 index 8b9b567d..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangasOriginesParser.kt +++ /dev/null @@ -1,56 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.madara - -import org.jsoup.nodes.Element -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.Manga -import org.koitharu.kotatsu.parsers.model.MangaChapter -import org.koitharu.kotatsu.parsers.model.MangaSource -import org.koitharu.kotatsu.parsers.model.MangaState -import org.koitharu.kotatsu.parsers.util.attrAsAbsoluteUrlOrNull -import org.koitharu.kotatsu.parsers.util.mapToSet -import org.koitharu.kotatsu.parsers.util.selectFirstOrThrow -import java.util.* - -@MangaSourceParser("MANGAS_ORIGINES", "Mangas Origines", "fr") -internal class MangasOriginesParser(context: MangaLoaderContext) : - Madara6Parser(context, MangaSource.MANGAS_ORIGINES, "mangas-origines.fr") { - - override val tagPrefix = "catalogues-genre/" - - override fun getFaviconUrl(): String { - return "https://${getDomain()}/wp-content/uploads/2020/11/Mangas-150x150.png" - } - - override fun parseDetails(manga: Manga, body: Element, chapters: List): Manga { - val root = body.selectFirstOrThrow(".site-content") - val postContent = root.selectFirstOrThrow(".post-content") - val tags = postContent.getElementsContainingOwnText("Genre") - .firstOrNull()?.tableValue() - ?.getElementsByAttributeValueContaining("href", tagPrefix) - ?.mapToSet { a -> a.asMangaTag() } ?: manga.tags - return manga.copy( - largeCoverUrl = root.selectFirst("picture") - ?.selectFirst("img[data-src]") - ?.attrAsAbsoluteUrlOrNull("data-src"), - description = (root.selectFirst(".detail-content") - ?: root.selectFirstOrThrow(".manga-excerpt")).html(), - author = postContent.getElementsContainingOwnText("Auteur") - .firstOrNull()?.tableValue()?.text()?.trim(), - state = postContent.getElementsContainingOwnText("STATUS") - .firstOrNull()?.tableValue()?.text()?.asMangaState(), - tags = tags, - isNsfw = body.hasClass("adult-content"), - chapters = chapters, - ) - } - - override fun String.asMangaState() = when (trim().lowercase(Locale.FRANCE)) { - "en cours" -> MangaState.ONGOING - "abandonné", - "terminé", - -> MangaState.FINISHED - - else -> null - } -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/NeatManga.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/NeatManga.kt new file mode 100644 index 00000000..ca4f5cda --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/NeatManga.kt @@ -0,0 +1,95 @@ +package org.koitharu.kotatsu.parsers.site.madara + +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.exception.ParseException +import org.koitharu.kotatsu.parsers.model.* +import org.koitharu.kotatsu.parsers.util.* +import java.text.SimpleDateFormat +import java.util.* + +@MangaSourceParser("NEATMANGA", "NeatManga", "en") +internal class NeatManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.NEATMANGA, "neatmangas.com") { + + override val datePattern = "dd MMMM yyyy" + + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { + val chaptersDeferred = async { getChapters(manga) } + val fullUrl = manga.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + val root = doc.body().selectFirst("div.profile-manga") + ?.selectFirst("div.summary_content") + ?.selectFirst("div.post-content") + ?: throw ParseException("Root not found", fullUrl) + val root2 = doc.body().selectFirst("div.content-area") + ?.selectFirst("div.c-page") + ?: throw ParseException("Root2 not found", fullUrl) + manga.copy( + tags = root.selectFirst("div.genres-content")?.select("a") + ?.mapNotNullToSet { a -> + MangaTag( + key = a.attr("href").removeSuffix("/").substringAfterLast('/'), + title = a.text().toTitleCase(), + source = source, + ) + } ?: manga.tags, + description = root2.getElementsMatchingOwnText("Summary") + .firstOrNull() + ?.nextElementSibling() + ?.select("p") + ?.filterNot { it.ownText().startsWith("A brief description") } + ?.joinToString { it.html() }, + chapters = chaptersDeferred.await(), + ) + } + + private suspend fun getChapters(manga: Manga): List { + val slug = manga.url.removeSuffix('/').substringAfterLast('/') + val doc2 = webClient.httpPost( + "https://$domain/manga/$slug/ajax/chapters/", + mapOf(), + ).parseHtml() + val ul = doc2.body().selectFirstOrThrow("ul") + val dateFormat = SimpleDateFormat(datePattern, Locale.US) + return ul.select("li").mapChapters(reversed = true) { i, li -> + val a = li.selectFirst("a") + val href = a?.attrAsRelativeUrlOrNull("href") ?: li.parseFailed("Link is missing") + MangaChapter( + id = generateUid(href), + name = a.ownText(), + number = i + 1, + url = href, + uploadDate = parseChapterDate( + dateFormat, + li.selectFirst("span.chapter-release-date i")?.text(), + ), + source = source, + scanlator = null, + branch = null, + ) + } + } + + override suspend fun getPages(chapter: MangaChapter): List { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + val root = doc.body().selectFirst("div.main-col-inner") + ?.selectFirst("div.reading-content") + ?: throw ParseException("Root not found", fullUrl) + return root.select("div.page-break").mapNotNull { div -> + val img = div.selectFirst("img") + if (img == null || img.attr("id").isNullOrEmpty()) { + return@mapNotNull null + } + val url = img.src()?.toRelativeUrl(domain) ?: div.parseFailed("Image src not found") + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/NeoxScansParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/NeoxScansParser.kt index 6dd722ed..38fdd722 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/NeoxScansParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/NeoxScansParser.kt @@ -16,10 +16,6 @@ internal class NeoxScansParser(context: MangaLoaderContext) : override val datePattern: String = "dd/MM/yyyy" - override fun getFaviconUrl(): String { - return "https://${getDomain()}/wp-content/uploads/2022/05/cropped-cropped-neoxscans-192x192.png" - } - override fun parseDetails(manga: Manga, body: Element, chapters: List): Manga { val root = body.selectFirstOrThrow(".site-content") val postContent = root.selectFirstOrThrow(".post-content") @@ -44,8 +40,8 @@ internal class NeoxScansParser(context: MangaLoaderContext) : } override suspend fun getPages(chapter: MangaChapter): List { - val fullUrl = chapter.url.toAbsoluteUrl(getDomain()) - val doc = context.httpGet(fullUrl).parseHtml() + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() val root = doc.body().selectFirst("div.main-col-inner") ?.selectFirst("div.reading-content") ?: throw ParseException("Root not found", fullUrl) @@ -56,7 +52,6 @@ internal class NeoxScansParser(context: MangaLoaderContext) : id = generateUid(url), url = url, preview = null, - referer = fullUrl, source = source, ) } @@ -67,4 +62,4 @@ internal class NeoxScansParser(context: MangaLoaderContext) : "Completo" -> MangaState.FINISHED else -> null } -} \ No newline at end of file +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/PrismaScansParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/PrismaScansParser.kt index da7dff3a..b9d0c822 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/PrismaScansParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/PrismaScansParser.kt @@ -10,17 +10,13 @@ import org.koitharu.kotatsu.parsers.model.MangaState import org.koitharu.kotatsu.parsers.util.attrAsAbsoluteUrlOrNull import org.koitharu.kotatsu.parsers.util.mapToSet import org.koitharu.kotatsu.parsers.util.selectFirstOrThrow -import java.util.* @MangaSourceParser("PRISMA_SCANS", "Prisma Scans", "pt") internal class PrismaScansParser(context: MangaLoaderContext) : Madara6Parser(context, MangaSource.PRISMA_SCANS, "prismascans.net") { override val tagPrefix = "manga-genre/" - - override fun getFaviconUrl(): String { - return "https://${getDomain()}/wp-content/uploads/2022/07/cropped-branca-1-192x192.png" - } + override val datePattern = "MMM dd, yyyy" override fun parseDetails(manga: Manga, body: Element, chapters: List): Manga { val root = body.selectFirstOrThrow(".site-content") @@ -33,7 +29,7 @@ internal class PrismaScansParser(context: MangaLoaderContext) : largeCoverUrl = root.selectFirst("picture") ?.selectFirst("img[data-src]") ?.attrAsAbsoluteUrlOrNull("data-src"), - description = root.selectFirstOrThrow(".description-summary").firstElementChild()?.html(), + description = root.selectFirstOrThrow(".manga-excerpt").firstElementChild()?.html(), author = postContent.getElementsContainingOwnText("Artista") .firstOrNull()?.tableValue()?.text()?.trim(), altTitle = postContent.getElementsContainingOwnText("Título Alternativo") @@ -46,7 +42,7 @@ internal class PrismaScansParser(context: MangaLoaderContext) : ) } - override fun String.asMangaState() = when (trim().lowercase(sourceLocale ?: Locale.ROOT)) { + override fun String.asMangaState() = when (trim().lowercase(sourceLocale)) { "em lançamento" -> MangaState.ONGOING "completo", "cancelado", @@ -54,4 +50,4 @@ internal class PrismaScansParser(context: MangaLoaderContext) : else -> null } -} \ No newline at end of file +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/ReaperScansParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/ReaperScansParser.kt new file mode 100644 index 00000000..d613607d --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/ReaperScansParser.kt @@ -0,0 +1,86 @@ +package org.koitharu.kotatsu.parsers.site.madara + +import org.jsoup.nodes.Element +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.model.* +import org.koitharu.kotatsu.parsers.util.* +import java.util.* + +@MangaSourceParser("REAPER_SCANS_ID", "ReaperScansID", "in") +internal class ReaperScansParser(context: MangaLoaderContext) : + Madara6Parser(context, MangaSource.REAPER_SCANS_ID, "reaperscans.id") { + + override val datePattern = "MMMM dd, yyyy" + override val tagPrefix = "genre/" + override val sourceLocale: Locale = Locale.ENGLISH + + override fun String.asMangaState(): MangaState? = when (this) { + "OnGoing", + "Upcoming", + -> MangaState.ONGOING + + "Completed", + "Dropped", + -> MangaState.FINISHED + + else -> null + } + + override fun parseDetails(manga: Manga, body: Element, chapters: List): Manga { + val root = body.selectFirstOrThrow(".site-content") + val postContent = root.requireElementById("nav-info") + val tags = postContent.getElementsContainingOwnText("Gênero") + .firstOrNull()?.tableValue() + ?.getElementsByAttributeValueContaining("href", tagPrefix) + ?.mapToSet { a -> a.asMangaTag() } ?: manga.tags + return manga.copy( + rating = postContent.selectFirstOrThrow(".post-rating") + .selectFirstOrThrow(".total_votes").text().toFloat() / 5f, + largeCoverUrl = root.selectFirst(".summary_image") + ?.selectFirst("img[data-src]") + ?.attrAsAbsoluteUrlOrNull("data-src") + .assertNotNull("largeCoverUrl"), + description = root.requireElementById("nav-profile") + .selectFirstOrThrow(".description-summary") + .firstElementChild()?.html(), + author = postContent.getElementsContainingOwnText("Author(s)") + .firstOrNull()?.tableValue()?.text()?.trim(), + altTitle = postContent.getElementsContainingOwnText("Alternative") + .firstOrNull()?.tableValue()?.text()?.trim(), + state = postContent.getElementsContainingOwnText("Status") + .firstOrNull()?.tableValue()?.text()?.asMangaState(), + tags = tags, + isNsfw = body.hasClass("adult-content"), + chapters = chapters, + ) + } + + override suspend fun getTags(): Set { + val doc = webClient.httpGet("https://${domain}/semua-komik/").parseHtml() + val body = doc.body() + val root1 = body.selectFirst("header")?.selectFirst("ul.second-menu") + val root2 = body.selectFirst("div.genres_wrap")?.selectFirst("ul.list-unstyled") + if (root1 == null && root2 == null) { + doc.parseFailed("Root not found") + } + val list = root1?.select("li").orEmpty() + root2?.select("li").orEmpty() + val keySet = HashSet(list.size) + return list.mapNotNullToSet { li -> + val a = li.selectFirst("a") ?: return@mapNotNullToSet null + val href = a.attr("href").removeSuffix("/") + .substringAfterLast(tagPrefix, "") + if (href.isEmpty() || !keySet.add(href)) { + return@mapNotNullToSet null + } + MangaTag( + key = href, + title = a.ownText().trim().ifEmpty { + a.selectFirst(".menu-image-title")?.text()?.trim() ?: return@mapNotNullToSet null + }.toTitleCase(), + source = source, + ) + } + } + +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/TatakaeScansParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/TatakaeScansParser.kt index d9366459..661f33c6 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/TatakaeScansParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/TatakaeScansParser.kt @@ -3,7 +3,10 @@ package org.koitharu.kotatsu.parsers.site.madara import org.jsoup.nodes.Element import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.* +import org.koitharu.kotatsu.parsers.model.Manga +import org.koitharu.kotatsu.parsers.model.MangaChapter +import org.koitharu.kotatsu.parsers.model.MangaSource +import org.koitharu.kotatsu.parsers.model.MangaState import org.koitharu.kotatsu.parsers.util.attrAsAbsoluteUrlOrNull import org.koitharu.kotatsu.parsers.util.mapToSet import org.koitharu.kotatsu.parsers.util.selectFirstOrThrow @@ -11,41 +14,37 @@ import java.util.* @MangaSourceParser("TATAKAE_SCANS", "Tatakae Scans", "pt") internal class TatakaeScansParser(context: MangaLoaderContext) : - Madara6Parser(context, MangaSource.TATAKAE_SCANS, "tatakaescan.com") { - - override val datePattern: String = "dd 'de' MMMMM 'de' yyyy" - - override fun parseDetails(manga: Manga, body: Element, chapters: List): Manga { - val root = body.selectFirstOrThrow(".site-content") - val postContent = root.selectFirstOrThrow(".post-content") - val tags = postContent.getElementsContainingOwnText("Gênero") - .firstOrNull()?.tableValue() - ?.getElementsByAttributeValueContaining("href", tagPrefix) - ?.mapToSet { a -> a.asMangaTag() } ?: manga.tags - return manga.copy( - largeCoverUrl = root.selectFirst("picture") - ?.selectFirst("img[data-src]") - ?.attrAsAbsoluteUrlOrNull("data-src"), - description = (root.selectFirst(".detail-content") - ?: root.selectFirstOrThrow(".manga-excerpt")).html(), - author = postContent.getElementsContainingOwnText("Autor") - .firstOrNull()?.tableValue()?.text()?.trim(), - state = postContent.getElementsContainingOwnText("Status") - .firstOrNull()?.tableValue()?.text()?.asMangaState(), - tags = tags, - isNsfw = body.hasClass("adult-content"), - chapters = chapters, - ) - } - - override fun String.asMangaState() = when (trim().lowercase(Locale.ROOT)) { - "em lançamento" -> MangaState.ONGOING - - else -> null - } - - override fun getFaviconUrl(): String { - return "https://${getDomain()}/wp-content/uploads/2022/07/cropped-favicon-180x180.png" - } - -} \ No newline at end of file + Madara6Parser(context, MangaSource.TATAKAE_SCANS, "tatakaescan.com") { + + override val datePattern: String = "dd 'de' MMMMM 'de' yyyy" + + override fun parseDetails(manga: Manga, body: Element, chapters: List): Manga { + val root = body.selectFirstOrThrow(".site-content") + val postContent = root.selectFirstOrThrow(".post-content") + val tags = postContent.getElementsContainingOwnText("Gênero") + .firstOrNull()?.tableValue() + ?.getElementsByAttributeValueContaining("href", tagPrefix) + ?.mapToSet { a -> a.asMangaTag() } ?: manga.tags + return manga.copy( + largeCoverUrl = root.selectFirst("picture") + ?.selectFirst("img[data-src]") + ?.attrAsAbsoluteUrlOrNull("data-src"), + description = (root.selectFirst(".detail-content") + ?: root.selectFirstOrThrow(".manga-excerpt")).html(), + author = postContent.getElementsContainingOwnText("Autor") + .firstOrNull()?.tableValue()?.text()?.trim(), + state = postContent.getElementsContainingOwnText("Status") + .firstOrNull()?.tableValue()?.text()?.asMangaState(), + tags = tags, + isNsfw = body.hasClass("adult-content"), + chapters = chapters, + ) + } + + override fun String.asMangaState() = when (trim().lowercase(Locale.ROOT)) { + "em lançamento" -> MangaState.ONGOING + + else -> null + } + +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt new file mode 100644 index 00000000..17f41d05 --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt @@ -0,0 +1,669 @@ +package org.koitharu.kotatsu.parsers.site.mangareader + +import androidx.collection.ArrayMap +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import org.json.JSONObject +import org.jsoup.nodes.Document +import org.jsoup.nodes.Element +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.PagedMangaParser +import org.koitharu.kotatsu.parsers.config.ConfigKey +import org.koitharu.kotatsu.parsers.model.* +import org.koitharu.kotatsu.parsers.util.* +import java.text.SimpleDateFormat +import java.util.* + +internal abstract class MangaReaderParser( + context: MangaLoaderContext, + source: MangaSource, + pageSize: Int, + searchPageSize: Int, +) : PagedMangaParser(context, source, pageSize, searchPageSize) { + + override val sortOrders: Set + get() = EnumSet.of(SortOrder.UPDATED, SortOrder.POPULARITY, SortOrder.ALPHABETICAL, SortOrder.NEWEST) + + protected val idLocale + get() = Locale("in", "ID") + + abstract val listUrl: String + abstract val tableMode: Boolean + open val chapterDateFormat = SimpleDateFormat("MMM d, yyyy", idLocale) + + private var tagCache: ArrayMap? = null + private val mutex = Mutex() + private var lastSearchPage = 1 + + override suspend fun getDetails(manga: Manga): Manga { + val docs = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val chapters = docs.select("#chapterlist > ul > li").mapChapters(reversed = true) { index, element -> + val url = element.selectFirst("a")?.attrAsRelativeUrl("href") ?: return@mapChapters null + MangaChapter( + id = generateUid(url), + name = element.selectFirst(".chapternum")?.text() ?: "Chapter ${index + 1}", + url = url, + number = index + 1, + scanlator = null, + uploadDate = chapterDateFormat.tryParse(element.selectFirst(".chapterdate")?.text()), + branch = null, + source = source, + ) + } + return if (tableMode) parseInfoTable(docs, manga, chapters) else parseInfoList(docs, manga, chapters) + } + + open suspend fun parseInfoTable(docs: Document, manga: Manga, chapters: List): Manga { + val mangaInfo = docs.selectFirst("div.seriestucontent > div.seriestucontentr") + val mangaState = mangaInfo?.selectFirst(".infotable td:contains(Status)")?.lastElementSibling()?.let { + when (it.text()) { + "Ongoing" -> MangaState.ONGOING + "Completed" -> MangaState.FINISHED + else -> null + } + } + val tagMap = getOrCreateTagMap() + val tags = mangaInfo?.select(".seriestugenre > a")?.mapNotNullToSet { tagMap[it.text()] } + + return manga.copy( + description = mangaInfo?.selectFirst("div.entry-content")?.html(), + state = mangaState, + author = mangaInfo?.selectFirst(".infotable td:contains(Author)")?.lastElementSibling()?.text(), + isNsfw = docs.selectFirst(".restrictcontainer") != null, + tags = tags.orEmpty(), + chapters = chapters, + ) + } + + open suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { + val mangaState = docs.selectFirst(".info-left .tsinfo div:contains(Status)")?.lastElementChild()?.let { + when (it.text()) { + "Ongoing" -> MangaState.ONGOING + "Completed" -> MangaState.FINISHED + else -> null + } + } + val tagMap = getOrCreateTagMap() + val tags = docs.select(".info-right .mgen > a").mapNotNullToSet { tagMap[it.text()] } + + return manga.copy( + description = docs.selectFirst(".info-right div.entry-content > p")?.html(), + state = mangaState, + author = docs.selectFirst(".info-left .tsinfo div:contains(Author)")?.lastElementChild()?.text(), + isNsfw = docs.selectFirst(".info-right .alr") != null, + tags = tags, + chapters = chapters, + ) + } + + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + if (!query.isNullOrEmpty()) { + if (page > lastSearchPage) { + return emptyList() + } + + val url = buildString { + append("https://") + append(domain) + append("/page/") + append(page) + append("/?s=") + append(query.urlEncoded()) + } + + val docs = webClient.httpGet(url).parseHtml() + lastSearchPage = docs.selectFirst(".pagination .next") + ?.previousElementSibling() + ?.text()?.toIntOrNull() ?: 1 + return parseMangaList(docs) + } + + val sortQuery = when (sortOrder) { + SortOrder.ALPHABETICAL -> "title" + SortOrder.NEWEST -> "latest" + SortOrder.POPULARITY -> "popular" + SortOrder.UPDATED -> "update" + else -> "" + } + val tagKey = "genre[]".urlEncoded() + val tagQuery = + if (tags.isNullOrEmpty()) "" else tags.joinToString(separator = "&", prefix = "&") { "$tagKey=${it.key}" } + val url = buildString { + append("https://") + append(domain) + append(listUrl) + append("/?order=") + append(sortQuery) + append(tagQuery) + append("&page=") + append(page) + } + + return parseMangaList(webClient.httpGet(url).parseHtml()) + } + + private fun parseMangaList(docs: Document): List { + return docs.select(".postbody .listupd .bs .bsx").mapNotNull { + val a = it.selectFirst("a") ?: return@mapNotNull null + val relativeUrl = a.attrAsRelativeUrl("href") + val rating = it.selectFirst(".numscore")?.text() + ?.toFloatOrNull()?.div(10) ?: RATING_UNKNOWN + + Manga( + id = generateUid(relativeUrl), + url = relativeUrl, + title = a.attr("title"), + altTitle = null, + publicUrl = a.attrAsAbsoluteUrl("href"), + rating = rating, + isNsfw = false, + coverUrl = it.selectFirst("img.ts-post-image")?.imageUrl().orEmpty(), + tags = emptySet(), + state = null, + author = null, + source = source, + ) + } + } + + override suspend fun getPages(chapter: MangaChapter): List { + val chapterUrl = chapter.url.toAbsoluteUrl(domain) + val docs = webClient.httpGet(chapterUrl).parseHtml() + val script = docs.selectFirstOrThrow("script:containsData(ts_reader)") + val images = JSONObject(script.data().substringAfter('(').substringBeforeLast(')')) + .getJSONArray("sources") + .getJSONObject(0) + .getJSONArray("images") + + val pages = ArrayList(images.length()) + for (i in 0 until images.length()) { + pages.add( + MangaPage( + id = generateUid(images.getString(i)), + url = images.getString(i), + preview = null, + source = source, + ), + ) + } + + return pages + } + + override suspend fun getTags(): Set { + return getOrCreateTagMap().values.toSet() + } + + protected suspend fun getOrCreateTagMap(): Map = mutex.withLock { + tagCache?.let { return@withLock it } + val tagMap = ArrayMap() + + val url = listUrl.toAbsoluteUrl(domain) + val tagElements = webClient.httpGet(url).parseHtml().select("ul.genrez > li") + for (el in tagElements) { + if (el.text().isEmpty()) continue + + tagMap[el.text()] = MangaTag( + title = el.text(), + key = el.selectFirst("input")?.attr("value") ?: continue, + source = source, + ) + } + + tagCache = tagMap + return@withLock tagMap + } + + private fun Element.imageUrl(): String { + return attrAsAbsoluteUrlOrNull("src") + ?: attrAsAbsoluteUrlOrNull("data-src") + ?: attrAsAbsoluteUrlOrNull("data-cfsrc") + ?: "" + } + + @MangaSourceParser("MANHWALAND", "Manhwaland", "id") + class ManhwaLandParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.MANHWALAND, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("manhwaland.us", arrayOf("manhwaland.us", "manhwaland.guru")) + + override val listUrl: String + get() = "/manga" + override val tableMode: Boolean + get() = false + } + + @MangaSourceParser("SEKAIKOMIK", "Sekaikomik", "id") + class SekaikomikParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.SEKAIKOMIK, pageSize = 20, searchPageSize = 100) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("sekaikomik.pro", null) + + override val listUrl: String + get() = "/manga" + override val tableMode: Boolean + get() = false + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMMM D, yyyy", idLocale) + } + + @MangaSourceParser("MANHWAINDO", "Manhwaindo", "id") + class ManhwaIndoParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.MANHWAINDO, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("manhwaindo.id", null) + + override val chapterDateFormat = SimpleDateFormat("MMMM dd, yyyy", Locale.ENGLISH) + override val listUrl: String get() = "/series" + override val tableMode: Boolean get() = false + } + + @MangaSourceParser("MANHWALIST", "Manhwalist", "id") + class ManhwalistParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.MANHWALIST, pageSize = 24, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("manhwalist.in", null) + + override val listUrl: String = "/manga" + override val tableMode: Boolean get() = false + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMMM d, yyyy", Locale.ENGLISH) + } + + @MangaSourceParser("KIRYUU", "Kiryuu", "id") + class KiryuuParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.KIRYUU, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("kiryuu.id", null) + + override val listUrl: String + get() = "/manga" + override val tableMode: Boolean + get() = true + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMMM d, yyyy", idLocale) + } + + @MangaSourceParser("TURKTOON", "Turktoon", "tr") + class TurktoonParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.TURKTOON, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("turktoon.com", null) + + override val listUrl: String + get() = "/manga" + override val tableMode: Boolean + get() = false + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMMM d, yyyy", Locale("tr", "TR")) + + override suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { + val mangaState = docs.selectFirst(".info-left .tsinfo div:contains(Durum)")?.lastElementChild()?.let { + when (it.text()) { + "Devam Ediyor" -> MangaState.ONGOING + "Tamamlandı" -> MangaState.FINISHED + else -> null + } + } + + return super.parseInfoList(docs, manga, chapters).copy(state = mangaState) + } + } + + @MangaSourceParser("WESTMANGA", "Westmanga", "id") + class WestmangaParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.WESTMANGA, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("westmanga.info", null) + + override val listUrl: String + get() = "/manga" + override val tableMode: Boolean + get() = true + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMMM d, yyyy", Locale.ENGLISH) + } + + @MangaSourceParser("TEMPESTFANSUB", "Tempestfansub", "tr") + class TempestfansubParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.TEMPESTFANSUB, pageSize = 40, searchPageSize = 40) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("tempestscans.com", null) + + override val listUrl: String get() = "/manga" + override val tableMode: Boolean get() = true + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMMM d, yyyy", Locale("tr", "TR")) + + override suspend fun parseInfoTable(docs: Document, manga: Manga, chapters: List): Manga { + val infoElement = docs.selectFirst("div.infox") + return manga.copy( + chapters = chapters, + description = infoElement?.selectFirst("div.entry-content")?.html(), + author = infoElement?.selectFirst(".flex-wrap div:contains(Artist)")?.lastElementSibling()?.text(), + tags = infoElement?.select(".wd-full .mgen > a") + ?.mapNotNullToSet { getOrCreateTagMap()[it.text()] } + .orEmpty(), + isNsfw = docs.selectFirst(".postbody .alr") != null, + ) + } + } + + @MangaSourceParser("MANHWADESU", "ManhwaDesu", "id") + class ManhwadesuParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.MANHWADESU, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("manhwadesu.pro", arrayOf("manhwadesu.pro", "manhwadesu.org")) + + override val listUrl: String get() = "/komik" + override val tableMode: Boolean get() = false + } + + @MangaSourceParser("MANGATALE", "MangaTale", "id") + class MangaTaleParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.MANGATALE, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("mangatale.co", null) + + override val listUrl: String get() = "/manga" + override val tableMode: Boolean get() = false + + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMMM d, yyyy", Locale.ENGLISH) + + override suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { + val infoElement = docs.selectFirst("div.infox") + return manga.copy( + chapters = chapters, + description = infoElement?.selectFirst("div.entry-content")?.html(), + author = infoElement?.selectFirst(".flex-wrap div:contains(Author)")?.lastElementSibling()?.text(), + tags = infoElement?.select(".wd-full .mgen > a") + ?.mapNotNullToSet { getOrCreateTagMap()[it.text()] } + .orEmpty(), + isNsfw = docs.selectFirst(".postbody .alr") != null, + ) + } + } + + @MangaSourceParser("DRAGONTRANSLATION", "DragonTranslation", "es") + class DragonTranslationParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.DRAGONTRANSLATION, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("dragontranslation.com", null) + + override val listUrl: String get() = "/manga" + override val tableMode: Boolean get() = false + + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMMM d, yyyy", Locale.ENGLISH) + + override suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { + val infoElement = docs.selectFirst("div.infox") + return manga.copy( + chapters = chapters, + description = infoElement?.selectFirst("div.entry-content")?.html(), + author = infoElement?.selectFirst(".flex-wrap div:contains(Author)")?.lastElementSibling()?.text(), + tags = infoElement?.select(".wd-full .mgen > a") + ?.mapNotNullToSet { getOrCreateTagMap()[it.text()] } + .orEmpty(), + isNsfw = docs.selectFirst(".postbody .alr") != null, + ) + } + } + + @MangaSourceParser("ASURATR", "Asura Scans (tr)", "tr") + class AsuraTRParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.ASURATR, pageSize = 30, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("asurascanstr.com", null) + + override val listUrl: String get() = "/manga" + override val tableMode: Boolean get() = false + + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", Locale("tr")) + + override suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { + val infoElement = docs.selectFirst("div.infox") + return manga.copy( + chapters = chapters, + description = infoElement?.selectFirst("div.entry-content")?.html(), + author = infoElement?.selectFirst(".flex-wrap div:contains(Yazar)")?.lastElementSibling()?.text(), + tags = infoElement?.select(".wd-full .mgen > a") + ?.mapNotNullToSet { getOrCreateTagMap()[it.text()] } + .orEmpty(), + isNsfw = docs.selectFirst(".postbody .alr") != null, + ) + } + } + + @MangaSourceParser("KOMIKTAP", "KomikTap", "id") + class KomikTapParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.KOMIKTAP, pageSize = 15, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("92.87.6.124", null) + + override val listUrl: String + get() = "/manga" + override val tableMode: Boolean + get() = true + + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", idLocale) + } + + @MangaSourceParser("KUMAPOI", "KumaPoi", "id") + class KumaPoiParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.KUMAPOI, pageSize = 15, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("kumapoi.me", null) + + override val listUrl: String + get() = "/manga" + override val tableMode: Boolean + get() = true + + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH) + } + + @MangaSourceParser("ASURASCANS", "Asura Scans", "en") + class AsuraScansParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.ASURASCANS, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("asurascans.com", null) + + override val listUrl: String + get() = "/manga" + override val tableMode: Boolean + get() = false + + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH) + + override suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { + val infoElement = docs.selectFirst("div.infox") + return manga.copy( + chapters = chapters, + description = infoElement?.selectFirst("div.entry-content")?.html(), + author = infoElement?.selectFirst(".flex-wrap div:contains(Author)")?.lastElementSibling()?.text(), + tags = infoElement?.select(".wd-full .mgen > a") + ?.mapNotNullToSet { getOrCreateTagMap()[it.text()] } + .orEmpty(), + isNsfw = docs.selectFirst(".postbody .alr") != null, + ) + } + } + + @MangaSourceParser("TOONHUNTER", "Toon Hunter", "th") + class ToonHunterParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.TOONHUNTER, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("toonhunter.com", null) + + override val listUrl: String + get() = "/manga" + override val tableMode: Boolean + get() = false + + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", sourceLocale) + + override suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { + val infoElement = docs.selectFirst("div.infox") + return manga.copy( + chapters = chapters, + description = infoElement?.selectFirst("div.entry-content")?.html(), + author = infoElement?.selectFirst(".flex-wrap div:contains(Author)")?.lastElementSibling()?.text(), + isNsfw = docs.selectFirst(".postbody .alr") != null, + ) + } + } + + @MangaSourceParser("COSMICSCANS", "CosmicScans", "en") + class CosmicScansParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.COSMICSCANS, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("cosmicscans.com", null) + + override val listUrl: String + get() = "/manga" + override val tableMode: Boolean + get() = false + + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH) + + override suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { + val infoElement = docs.selectFirst("div.infox") + return manga.copy( + chapters = chapters, + description = infoElement?.selectFirst("div.entry-content")?.html(), + author = infoElement?.selectFirst(".flex-wrap div:contains(Author)")?.lastElementSibling()?.text(), + tags = infoElement?.select(".wd-full .mgen > a") + ?.mapNotNullToSet { getOrCreateTagMap()[it.text()] } + .orEmpty(), + ) + } + } + + @MangaSourceParser("KOMIKLOKAL", "KomikLokal", "id") + class KomikLokalParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.KOMIKLOKAL, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("komikmirror.art", null) + + override val listUrl: String + get() = "/manga" + override val tableMode: Boolean + get() = false + + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH) + + override suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { + val infoElement = docs.selectFirst("div.infox") + return manga.copy( + chapters = chapters, + tags = infoElement?.select(".wd-full .mgen > a") + ?.mapNotNullToSet { getOrCreateTagMap()[it.text()] } + .orEmpty(), + ) + } + } + + @MangaSourceParser("KOMIKAV", "KomiKav", "id") + class KomiKavParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.KOMIKAV, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("komikav.com", null) + + override val listUrl: String + get() = "/manga" + override val tableMode: Boolean + get() = false + + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH) + + override suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { + val infoElement = docs.selectFirst("div.infox") + return manga.copy( + chapters = chapters, + description = infoElement?.selectFirst("div.entry-content")?.html(), + author = infoElement?.selectFirst(".flex-wrap div:contains(Author)")?.lastElementSibling()?.text(), + tags = infoElement?.select(".wd-full .mgen > a") + ?.mapNotNullToSet { getOrCreateTagMap()[it.text()] } + .orEmpty(), + ) + } + } + + @MangaSourceParser("KOMIKDEWASA", "KomikDewasa", "id") + class KomikDewasaParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.KOMIKDEWASA, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("komikdewasa.us", arrayOf("komikdewasa.us", "komikdewasa.info")) + + override val listUrl: String + get() = "/manga" + override val tableMode: Boolean + get() = false + + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH) + + override suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { + val infoElement = docs.selectFirst("div.infox") + return manga.copy( + chapters = chapters, + description = infoElement?.selectFirst("div.entry-content")?.html(), + ) + } + } + + @MangaSourceParser("MANGASUSU", "Mangasusu", "id") + class MangasusuParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.MANGASUSU, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("mangasusu.co.in", null) + + override val listUrl: String + get() = "/project" + override val tableMode: Boolean + get() = true + + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH) + } + + @MangaSourceParser("KOMIKLAB", "KomikLab", "id") + class KomikLabParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.KOMIKLAB, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("komiklab.com", null) + + override val listUrl: String + get() = "/project" + override val tableMode: Boolean + get() = true + + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH) + } + + @MangaSourceParser("KOMIKINDO", "KomikIndo", "id") + class KomikIndoParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.KOMIKINDO, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("komikindo.co", null) + + override val listUrl: String + get() = "/project" + override val tableMode: Boolean + get() = true + + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", idLocale) + } + + @MangaSourceParser("KOMIKMANGA", "KomikManga", "id") + class KomikMangaParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.KOMIKMANGA, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("komikhentai.co", null) + + override val listUrl: String + get() = "/project" + override val tableMode: Boolean + get() = true + + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH) + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/multichan/ChanParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/multichan/ChanParser.kt index 5792505d..d89a4a87 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/multichan/ChanParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/multichan/ChanParser.kt @@ -1,5 +1,6 @@ package org.koitharu.kotatsu.parsers.site.multichan +import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaParser import org.koitharu.kotatsu.parsers.MangaParserAuthProvider import org.koitharu.kotatsu.parsers.exception.AuthRequiredException @@ -8,7 +9,10 @@ import org.koitharu.kotatsu.parsers.util.* import java.text.SimpleDateFormat import java.util.* -internal abstract class ChanParser(source: MangaSource) : MangaParser(source), MangaParserAuthProvider { +internal abstract class ChanParser( + context: MangaLoaderContext, + source: MangaSource, +) : MangaParser(context, source), MangaParserAuthProvider { override val sortOrders: Set = EnumSet.of( SortOrder.NEWEST, @@ -17,10 +21,10 @@ internal abstract class ChanParser(source: MangaSource) : MangaParser(source), M ) override val authUrl: String - get() = "https://${getDomain()}" + get() = "https://${domain}" override val isAuthorized: Boolean - get() = context.cookieJar.getCookies(getDomain()).any { it.name == "dle_user_id" } + get() = context.cookieJar.getCookies(domain).any { it.name == "dle_user_id" } override suspend fun getList( offset: Int, @@ -28,7 +32,7 @@ internal abstract class ChanParser(source: MangaSource) : MangaParser(source), M tags: Set?, sortOrder: SortOrder, ): List { - val domain = getDomain() + val domain = domain val url = when { !query.isNullOrEmpty() -> { if (offset != 0) { @@ -45,7 +49,7 @@ internal abstract class ChanParser(source: MangaSource) : MangaParser(source), M else -> "https://$domain/${getSortKey(sortOrder)}?offset=$offset" } - val doc = context.httpGet(url).parseHtml() + val doc = webClient.httpGet(url).parseHtml() val root = doc.body().selectFirst("div.main_fon")?.getElementById("content") ?: doc.parseFailed("Cannot find root") return root.select("div.content_row").mapNotNull { row -> @@ -82,13 +86,13 @@ internal abstract class ChanParser(source: MangaSource) : MangaParser(source), M } override suspend fun getDetails(manga: Manga): Manga { - val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml() + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() val root = doc.body().getElementById("dle-content") ?: doc.parseFailed("Cannot find root") val dateFormat = SimpleDateFormat("yyyy-MM-dd", Locale.US) return manga.copy( description = root.getElementById("description")?.html()?.substringBeforeLast(" + chapters = root.select("table.table_cha tr:gt(1)").mapChapters(reversed = true) { i, tr -> val href = tr?.selectFirst("a")?.attrAsRelativeUrlOrNull("href") ?: return@mapChapters null MangaChapter( @@ -106,8 +110,8 @@ internal abstract class ChanParser(source: MangaSource) : MangaParser(source), M } override suspend fun getPages(chapter: MangaChapter): List { - val fullUrl = chapter.url.toAbsoluteUrl(getDomain()) - val doc = context.httpGet(fullUrl).parseHtml() + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() val scripts = doc.select("script") for (script in scripts) { val data = script.html() @@ -117,7 +121,7 @@ internal abstract class ChanParser(source: MangaSource) : MangaParser(source), M } val json = data.substring(pos).substringAfter('[').substringBefore(';') .substringBeforeLast(']') - val domain = getDomain() + val domain = domain return json.split(",").mapNotNull { it.trim() .removeSurrounding('"', '\'') @@ -128,7 +132,6 @@ internal abstract class ChanParser(source: MangaSource) : MangaParser(source), M id = generateUid(url), url = url, preview = null, - referer = fullUrl, source = source, ) } @@ -137,8 +140,8 @@ internal abstract class ChanParser(source: MangaSource) : MangaParser(source), M } override suspend fun getTags(): Set { - val domain = getDomain() - val doc = context.httpGet("https://$domain/mostfavorites&sort=manga").parseHtml() + val domain = domain + val doc = webClient.httpGet("https://$domain/mostfavorites&sort=manga").parseHtml() val root = doc.body().selectFirst("div.main_fon")?.getElementById("side") ?.select("ul")?.last() ?: doc.parseFailed("Cannot find root") return root.select("li.sidetag").mapToSet { li -> @@ -152,7 +155,7 @@ internal abstract class ChanParser(source: MangaSource) : MangaParser(source), M } override suspend fun getUsername(): String { - val doc = context.httpGet("https://${getDomain()}").parseHtml().body() + val doc = webClient.httpGet("https://${domain}").parseHtml().body() val root = doc.requireElementById("top_user") val a = root.getElementsByAttributeValueContaining("href", "/user/").firstOrNull() ?: throw AuthRequiredException(source) @@ -176,4 +179,4 @@ internal abstract class ChanParser(source: MangaSource) : MangaParser(source), M } private fun String.toTagName() = replace('_', ' ').toTitleCase() -} \ No newline at end of file +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/multichan/HenChanParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/multichan/HenChanParser.kt index 97a5f032..eec7465c 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/multichan/HenChanParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/multichan/HenChanParser.kt @@ -7,54 +7,54 @@ import org.koitharu.kotatsu.parsers.model.* import org.koitharu.kotatsu.parsers.util.* @MangaSourceParser("HENCHAN", "Хентай-тян", "ru") -internal class HenChanParser(override val context: MangaLoaderContext) : ChanParser(MangaSource.HENCHAN) { +internal class HenChanParser(context: MangaLoaderContext) : ChanParser(context, MangaSource.HENCHAN) { - override val configKeyDomain = ConfigKey.Domain( - "y.hentaichan.live", - arrayOf("y.hentaichan.live", "xxx.hentaichan.live", "xx.hentaichan.live", "hentaichan.live", "hentaichan.pro"), - ) + override val configKeyDomain = ConfigKey.Domain( + "y.hentaichan.live", + arrayOf("y.hentaichan.live", "xxx.hentaichan.live", "xx.hentaichan.live", "hentaichan.live", "hentaichan.pro"), + ) - override suspend fun getList( - offset: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - return super.getList(offset, query, tags, sortOrder).map { - it.copy( - coverUrl = it.coverUrl.replace("_blur", ""), - isNsfw = true, - ) - } - } + override suspend fun getList( + offset: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + return super.getList(offset, query, tags, sortOrder).map { + it.copy( + coverUrl = it.coverUrl.replace("_blur", ""), + isNsfw = true, + ) + } + } - override suspend fun getDetails(manga: Manga): Manga { - val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml() - val root = doc.body().requireElementById("dle-content") - val readLink = manga.url.replace("manga", "online") - return manga.copy( - description = root.getElementById("description")?.html()?.substringBeforeLast(" table.select("div.manga") - }.mapNotNull { it.selectFirst("a") }.reversed().mapChapters { i, a -> + }.mapNotNull { it.selectFirst("a") }.mapChapters(reversed = true) { i, a -> val href = a.attrAsRelativeUrl("href") MangaChapter( id = generateUid(href), @@ -36,4 +36,4 @@ internal class YaoiChanParser(override val context: MangaLoaderContext) : ChanPa }, ) } -} \ No newline at end of file +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/rulib/MangaLibParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/rulib/MangaLibParser.kt index 8c0bb874..ed6fba15 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/rulib/MangaLibParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/rulib/MangaLibParser.kt @@ -20,14 +20,14 @@ import java.text.SimpleDateFormat import java.util.* internal open class MangaLibParser( - override val context: MangaLoaderContext, + context: MangaLoaderContext, source: MangaSource, -) : PagedMangaParser(source, pageSize = 60), MangaParserAuthProvider { +) : PagedMangaParser(context, source, pageSize = 60), MangaParserAuthProvider { override val configKeyDomain = ConfigKey.Domain("mangalib.me", null) override val authUrl: String - get() = "https://${getDomain()}/login" + get() = "https://${domain}/login" override val sortOrders: Set = EnumSet.of( SortOrder.RATING, @@ -48,7 +48,7 @@ internal open class MangaLibParser( } val url = buildString { append("https://") - append(getDomain()) + append(domain) append("/manga-list?dir=") append(getSortKey(sortOrder)) append("&page=") @@ -58,7 +58,7 @@ internal open class MangaLibParser( append(tag.key) } } - val doc = context.httpGet(url).parseHtml() + val doc = webClient.httpGet(url).parseHtml() val root = doc.body().getElementById("manga-list") ?: doc.parseFailed("Root not found") val items = root.selectFirst("div.media-cards-grid")?.select("div.media-card-wrap") ?: return emptyList() @@ -73,7 +73,7 @@ internal open class MangaLibParser( author = null, rating = RATING_UNKNOWN, url = href, - publicUrl = href.toAbsoluteUrl(a.host ?: getDomain()), + publicUrl = href.toAbsoluteUrl(a.host ?: domain), tags = emptySet(), state = null, isNsfw = false, @@ -83,12 +83,12 @@ internal open class MangaLibParser( } override suspend fun getDetails(manga: Manga): Manga { - val fullUrl = manga.url.toAbsoluteUrl(getDomain()) - val doc = context.httpGet("$fullUrl?section=info").parseHtml() + val fullUrl = manga.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet("$fullUrl?section=info").parseHtml() val root = doc.body().getElementById("main-page") ?: throw ParseException("Root not found", fullUrl) val title = root.selectFirst("div.media-header__wrap")?.children() val info = root.selectFirst("div.media-content") - val chaptersDoc = context.httpGet("$fullUrl?section=chapters").parseHtml() + val chaptersDoc = webClient.httpGet("$fullUrl?section=chapters").parseHtml() val scripts = chaptersDoc.select("script") val dateFormat = SimpleDateFormat("yyy-MM-dd", Locale.US) var chapters: ChaptersListBuilder? = null @@ -98,6 +98,7 @@ internal open class MangaLibParser( if (line.startsWith("window.__DATA__")) { val json = JSONObject(line.substringAfter('=').substringBeforeLast(';')) val list = json.getJSONObject("chapters").getJSONArray("list") + val id = json.optJSONObject("user")?.getLong("id")?.toString() ?: "not" val total = list.length() chapters = ChaptersListBuilder(total) for (i in 0 until total) { @@ -105,13 +106,21 @@ internal open class MangaLibParser( val chapterId = item.getLong("chapter_id") val scanlator = item.getStringOrNull("username") val url = buildString { - append(manga.url) - append("/v") - append(item.getInt("chapter_volume")) - append("/c") - append(item.getString("chapter_number")) - append('/') - append(item.optString("chapter_string")) + if (isAuthorized) { + append(manga.url) + append("/v") + append(item.getInt("chapter_volume")) + append("/c") + append(item.getString("chapter_number")) + append("?ui=") + append(id) + } else { + append(manga.url) + append("/v") + append(item.getInt("chapter_volume")) + append("/c") + append(item.getString("chapter_number")) + } } val nameChapter = item.getStringOrNull("chapter_name") val volume = item.getInt("chapter_volume") @@ -163,8 +172,8 @@ internal open class MangaLibParser( } override suspend fun getPages(chapter: MangaChapter): List { - val fullUrl = chapter.url.toAbsoluteUrl(getDomain()) - val doc = context.httpGet(fullUrl).parseHtml() + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() if (doc.location().substringBefore('?').endsWith("/register")) { throw AuthRequiredException(source) } @@ -193,7 +202,6 @@ internal open class MangaLibParser( id = generateUid(pageUrl), url = pageUrl, preview = null, - referer = fullUrl, source = source, ) } @@ -203,8 +211,8 @@ internal open class MangaLibParser( } override suspend fun getTags(): Set { - val url = "https://${getDomain()}/manga-list" - val doc = context.httpGet(url).parseHtml() + val url = "https://${domain}/manga-list" + val doc = webClient.httpGet(url).parseHtml() val scripts = doc.body().select("script") for (script in scripts) { val raw = script.html().trim() @@ -227,13 +235,13 @@ internal open class MangaLibParser( override val isAuthorized: Boolean get() { - return context.cookieJar.getCookies(getDomain()).any { + return context.cookieJar.getCookies(domain).any { it.name.startsWith("remember_web_") } } override suspend fun getUsername(): String { - val body = context.httpGet("https://${getDomain()}/messages").parseHtml().body() + val body = webClient.httpGet("https://${LibConst.LIB_SOCIAL_LINK}/messages").parseHtml().body() if (body.baseUri().endsWith("/login")) { throw AuthRequiredException(source) } @@ -257,8 +265,8 @@ internal open class MangaLibParser( } private suspend fun search(query: String): List { - val domain = getDomain() - val json = context.httpGet("https://$domain/search?type=manga&q=$query") + val domain = domain + val json = webClient.httpGet("https://$domain/search?type=manga&q=$query") .parseJsonArray() return json.mapJSON { jo -> val slug = jo.getString("slug") @@ -285,4 +293,10 @@ internal open class MangaLibParser( @MangaSourceParser("MANGALIB", "MangaLib", "ru") class Impl(context: MangaLoaderContext) : MangaLibParser(context, MangaSource.MANGALIB) -} \ No newline at end of file + + object LibConst { + + val LIB_SOCIAL_LINK = "lib.social" + + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Chapters.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Chapters.kt index a9ed843c..289080ec 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Chapters.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Chapters.kt @@ -4,10 +4,14 @@ import org.koitharu.kotatsu.parsers.InternalParsersApi import org.koitharu.kotatsu.parsers.model.MangaChapter @InternalParsersApi -inline fun Iterable.mapChapters(transform: (index: Int, T) -> MangaChapter?): List { +inline fun List.mapChapters( + reversed: Boolean = false, + transform: (index: Int, T) -> MangaChapter?, +): List { val builder = ChaptersListBuilder(collectionSize()) var index = 0 - for (item in this) { + val elements = if (reversed) this.asReversed() else this + for (item in elements) { if (builder.add(transform(index, item))) { index++ } @@ -39,4 +43,4 @@ internal class ChaptersListBuilder(initialSize: Int) { } fun toList(): List = list -} \ No newline at end of file +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Coroutines.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Coroutines.kt new file mode 100644 index 00000000..fffb94f2 --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Coroutines.kt @@ -0,0 +1,35 @@ +package org.koitharu.kotatsu.parsers.util + +import kotlinx.coroutines.Deferred +import kotlinx.coroutines.Job +import kotlinx.coroutines.flow.channelFlow +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.launch +import kotlin.coroutines.cancellation.CancellationException + +fun Iterable.cancelAll(cause: CancellationException? = null) { + forEach { it.cancel(cause) } +} + +suspend fun Iterable>.awaitFirst(): T { + return channelFlow { + for (deferred in this@awaitFirst) { + launch { + send(deferred.await()) + } + } + }.first().also { this@awaitFirst.cancelAll() } +} + +suspend fun Collection>.awaitFirst(condition: (T) -> Boolean): T { + return channelFlow { + for (deferred in this@awaitFirst) { + launch { + val result = deferred.await() + if (condition(result)) { + send(result) + } + } + } + }.first().also { this@awaitFirst.cancelAll() } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/FaviconParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/FaviconParser.kt index fec7f706..cb8a6a3d 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/FaviconParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/FaviconParser.kt @@ -1,94 +1,92 @@ package org.koitharu.kotatsu.parsers.util -import okhttp3.Headers import org.jsoup.nodes.Element -import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.model.Favicon import org.koitharu.kotatsu.parsers.model.Favicons +import org.koitharu.kotatsu.parsers.network.WebClient import org.koitharu.kotatsu.parsers.util.json.mapJSON class FaviconParser( - private val context: MangaLoaderContext, - private val domain: String, - private val headers: Headers?, + private val webClient: WebClient, + private val domain: String, ) { - suspend fun parseFavicons(): Favicons { - val url = "https://$domain" - val doc = context.httpGet(url, headers).parseHtml() - val result = HashSet() - val manifestLink = doc.getElementsByAttributeValue("rel", "manifest").firstOrNull() - ?.attrAsAbsoluteUrlOrNull("href") - if (manifestLink != null) { - result += parseManifest(manifestLink) - } - val links = doc.getElementsByAttributeValueContaining("rel", "icon") - links.mapNotNullTo(result) { link -> - parseLink(link) - } - if (result.isEmpty()) { - result.add(createFallback()) - } - return Favicons(result, url) - } + suspend fun parseFavicons(): Favicons { + val url = "https://$domain" + val doc = webClient.httpGet(url).parseHtml() + val result = HashSet() + val manifestLink = doc.getElementsByAttributeValue("rel", "manifest").firstOrNull() + ?.attrAsAbsoluteUrlOrNull("href") + if (manifestLink != null) { + result += parseManifest(manifestLink) + } + val links = doc.getElementsByAttributeValueContaining("rel", "icon") + links.mapNotNullTo(result) { link -> + parseLink(link) + } + if (result.isEmpty()) { + result.add(createFallback()) + } + return Favicons(result, url) + } - private fun parseLink(link: Element): Favicon? { - val href = link.attrAsAbsoluteUrlOrNull("href") - if (href == null || href.endsWith('/')) { - return null - } - val sizes = link.attr("sizes") - return Favicon( - url = href, - size = parseSize(sizes), - rel = link.attrOrNull("rel"), - ) - } + private fun parseLink(link: Element): Favicon? { + val href = link.attrAsAbsoluteUrlOrNull("href") + if (href == null || href.endsWith('/')) { + return null + } + val sizes = link.attr("sizes") + return Favicon( + url = href, + size = parseSize(sizes), + rel = link.attrOrNull("rel"), + ) + } - private fun parseSize(sizes: String): Int { - if (sizes.isEmpty() || sizes == "any") { - return 0 - } - return sizes.substringBefore(' ') - .split('x', 'X', '*') - .firstNotNullOfOrNull { it.toIntOrNull() } - ?: 0 - } + private fun parseSize(sizes: String): Int { + if (sizes.isEmpty() || sizes == "any") { + return 0 + } + return sizes.substringBefore(' ') + .split('x', 'X', '*') + .firstNotNullOfOrNull { it.toIntOrNull() } + ?: 0 + } - private suspend fun parseManifest(url: String): List { - val json = context.httpGet(url, headers).parseJson() - val icons = json.getJSONArray("icons") - return icons.mapJSON { jo -> - Favicon( - url = jo.getString("src").resolveLink(), - size = parseSize(jo.getString("sizes")), - rel = null, - ) - } - } + private suspend fun parseManifest(url: String): List { + val json = webClient.httpGet(url).parseJson() + val icons = json.getJSONArray("icons") + return icons.mapJSON { jo -> + Favicon( + url = jo.getString("src").resolveLink(), + size = parseSize(jo.getString("sizes")), + rel = null, + ) + } + } - private fun createFallback(): Favicon { - val href = "https://$domain/favicon.ico" - return Favicon( - url = href, - size = 0, - rel = null, - ) - } + private fun createFallback(): Favicon { + val href = "https://$domain/favicon.ico" + return Favicon( + url = href, + size = 0, + rel = null, + ) + } - private fun String.resolveLink(): String { - return when { - startsWith("http:") || startsWith("https:") -> { - this - } + private fun String.resolveLink(): String { + return when { + startsWith("http:") || startsWith("https:") -> { + this + } - startsWith('/') -> { - "https://$domain$this" - } + startsWith('/') -> { + "https://$domain$this" + } - else -> { - "https://$domain/$this" - } - } - } -} \ No newline at end of file + else -> { + "https://$domain/$this" + } + } + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Jsoup.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Jsoup.kt index f76e079e..24d1a6bd 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Jsoup.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Jsoup.kt @@ -32,7 +32,7 @@ fun Element.attrOrNull(attributeKey: String) = attr(attributeKey).takeUnless { i */ fun Element.attrAsRelativeUrlOrNull(attributeKey: String): String? { val attr = attr(attributeKey).trim() - if (attr.isEmpty()) { + if (attr.isEmpty() || attr.startsWith("data:")) { return null } if (attr.startsWith("/")) { @@ -63,7 +63,7 @@ fun Element.attrAsRelativeUrl(attributeKey: String): String { */ fun Element.attrAsAbsoluteUrlOrNull(attributeKey: String): String? { val attr = attr(attributeKey).trim() - if (attr.isEmpty()) { + if (attr.isEmpty() || attr.startsWith("data:")) { return null } return (baseUri().toHttpUrlOrNull()?.newBuilder(attr) ?: return null).toString() @@ -114,4 +114,8 @@ fun Element.selectLast(cssQuery: String): Element? { fun Element.selectLastOrThrow(cssQuery: String): Element { return selectLast(cssQuery) ?: throw ParseException("Cannot find \"$cssQuery\"", baseUri()) -} \ No newline at end of file +} + +fun Element.textOrNull(): String? = text().takeUnless { it.isEmpty() } + +fun Element.ownTextOrNull(): String? = ownText().takeUnless { it.isEmpty() } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/MangaParserEnv.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/MangaParserEnv.kt new file mode 100644 index 00000000..1e20c9e0 --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/MangaParserEnv.kt @@ -0,0 +1,78 @@ +package org.koitharu.kotatsu.parsers.util + +import okhttp3.HttpUrl +import org.jsoup.nodes.Element +import org.koitharu.kotatsu.parsers.InternalParsersApi +import org.koitharu.kotatsu.parsers.MangaParser +import org.koitharu.kotatsu.parsers.exception.ParseException +import org.koitharu.kotatsu.parsers.model.Manga +import org.koitharu.kotatsu.parsers.model.MangaChapter +import org.koitharu.kotatsu.parsers.model.MangaPage +import org.koitharu.kotatsu.parsers.model.MangaTag + + +/** + * Create a unique id for [Manga]/[MangaChapter]/[MangaPage]. + * @param url must be relative url, without a domain + * @see [Manga.id] + * @see [MangaChapter.id] + * @see [MangaPage.id] + */ +@InternalParsersApi +fun MangaParser.generateUid(url: String): Long { + var h = 1125899906842597L + source.name.forEach { c -> + h = 31 * h + c.code + } + url.forEach { c -> + h = 31 * h + c.code + } + return h +} + +/** + * Create a unique id for [Manga]/[MangaChapter]/[MangaPage]. + * @param id an internal identifier + * @see [Manga.id] + * @see [MangaChapter.id] + * @see [MangaPage.id] + */ +@InternalParsersApi +fun MangaParser.generateUid(id: Long): Long { + var h = 1125899906842597L + source.name.forEach { c -> + h = 31 * h + c.code + } + h = 31 * h + id + return h +} + +@InternalParsersApi +fun Element.parseFailed(message: String? = null): Nothing { + throw ParseException(message, ownerDocument()?.location() ?: baseUri(), null) +} + +@InternalParsersApi +fun Set?.oneOrThrowIfMany(): MangaTag? { + return when { + isNullOrEmpty() -> null + size == 1 -> first() + else -> throw IllegalArgumentException("Multiple genres are not supported by this source") + } +} + +val MangaParser.domain: String + get() { + return config[configKeyDomain] + } + +fun MangaParser.getDomain(subdomain: String): String { + val domain = domain + return subdomain + "." + domain.removePrefix("www.") +} + +fun MangaParser.urlBuilder(): HttpUrl.Builder { + return HttpUrl.Builder() + .scheme("https") + .host(domain) +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/OkHttp.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/OkHttp.kt index 08c5d3c0..df363e70 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/OkHttp.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/OkHttp.kt @@ -4,16 +4,26 @@ package org.koitharu.kotatsu.parsers.util import kotlinx.coroutines.suspendCancellableCoroutine import okhttp3.Call +import okhttp3.Headers import okhttp3.Response suspend fun Call.await(): Response = suspendCancellableCoroutine { continuation -> - val callback = ContinuationCallCallback(this, continuation) - enqueue(callback) - continuation.invokeOnCancellation(callback) + val callback = ContinuationCallCallback(this, continuation) + enqueue(callback) + continuation.invokeOnCancellation(callback) } val Response.mimeType: String? - get() = body?.contentType()?.run { "$type/$subtype" } + get() = body?.contentType()?.run { "$type/$subtype" } val Response.contentDisposition: String? - get() = header("Content-Disposition") \ No newline at end of file + get() = header("Content-Disposition") + +fun Headers.Builder.mergeWith(other: Headers, replaceExisting: Boolean): Headers.Builder { + for ((name, value) in other) { + if (replaceExisting || this[name] == null) { + this[name] = value + } + } + return this +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Parse.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Parse.kt index 7c965778..d1aecc44 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Parse.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Parse.kt @@ -46,6 +46,12 @@ fun Response.parseJsonArray(): JSONArray = try { closeQuietly() } +fun Response.parseRaw(): String = try { + requireBody().string() +} finally { + closeQuietly() +} + /** * Convert url to relative if it is on [domain] * @return an url relative to the [domain] or absolute, if domain is mismatching @@ -67,6 +73,16 @@ fun String.toAbsoluteUrl(domain: String): String = when { else -> this } +fun concatUrl(host: String, path: String): String { + val hostWithSlash = host.endsWith('/') + val pathWithSlash = path.startsWith('/') + return when { + hostWithSlash && pathWithSlash -> host + path.drop(1) + !hostWithSlash && !pathWithSlash -> "$host/$path" + else -> host + path + } +} + fun DateFormat.tryParse(str: String?): Long = if (str.isNullOrEmpty()) { assert(false) { "Date string is null or empty" } 0L @@ -78,4 +94,4 @@ fun DateFormat.tryParse(str: String?): Long = if (str.isNullOrEmpty()) { }.getOrDefault(0L) } -private fun Response.requireBody(): ResponseBody = requireNotNull(body) { "Response body is null" } \ No newline at end of file +private fun Response.requireBody(): ResponseBody = requireNotNull(body) { "Response body is null" } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Result.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Result.kt new file mode 100644 index 00000000..17732922 --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Result.kt @@ -0,0 +1,13 @@ +package org.koitharu.kotatsu.parsers.util + +import kotlinx.coroutines.CancellationException + +inline fun T.runCatchingCancellable(block: T.() -> R): Result { + return try { + Result.success(block()) + } catch (e: CancellationException) { + throw e + } catch (e: Throwable) { + Result.failure(e) + } +} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/SuspendLazy.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/SuspendLazy.kt new file mode 100644 index 00000000..cb9a911a --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/SuspendLazy.kt @@ -0,0 +1,36 @@ +package org.koitharu.kotatsu.parsers.util + +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock + +class SuspendLazy( + private val initializer: suspend () -> T, +) { + + private val mutex = Mutex() + private var cachedValue: Any? = Uninitialized + + @Suppress("UNCHECKED_CAST") + suspend fun get(): T { + // fast way + cachedValue.let { + if (it !== Uninitialized) { + return it as T + } + } + return mutex.withLock { + cachedValue.let { + if (it !== Uninitialized) { + return it as T + } + } + val result = initializer() + cachedValue = result + result + } + } + + suspend fun tryGet() = runCatchingCancellable { get() } + + private object Uninitialized +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/json/JsonExt.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/json/JsonExt.kt index 9e429b3c..7a994a84 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/json/JsonExt.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/json/JsonExt.kt @@ -140,4 +140,4 @@ fun JSONArray?.isNullOrEmpty(): Boolean { fun JSONArray.toJSONList(): List { return List(length()) { i -> getJSONObject(i) } -} \ No newline at end of file +} diff --git a/src/test/kotlin/org/koitharu/kotatsu/parsers/AuthCheckExtension.kt b/src/test/kotlin/org/koitharu/kotatsu/parsers/AuthCheckExtension.kt index d058e68e..05ab3c6f 100644 --- a/src/test/kotlin/org/koitharu/kotatsu/parsers/AuthCheckExtension.kt +++ b/src/test/kotlin/org/koitharu/kotatsu/parsers/AuthCheckExtension.kt @@ -4,30 +4,31 @@ import kotlinx.coroutines.test.runTest import org.junit.jupiter.api.extension.BeforeAllCallback import org.junit.jupiter.api.extension.ExtensionContext import org.koitharu.kotatsu.parsers.model.MangaSource +import org.koitharu.kotatsu.parsers.util.runCatchingCancellable class AuthCheckExtension : BeforeAllCallback { - private val loaderContext: MangaLoaderContext = MangaLoaderContextMock() + private val loaderContext: MangaLoaderContext = MangaLoaderContextMock - override fun beforeAll(context: ExtensionContext) { - for (source in MangaSource.values()) { - if (source == MangaSource.LOCAL || source == MangaSource.DUMMY) { - continue - } - val parser = source.newParser(loaderContext) - if (parser is MangaParserAuthProvider) { - checkAuthorization(source, parser) - } - } - } + override fun beforeAll(context: ExtensionContext) { + for (source in MangaSource.values()) { + if (source == MangaSource.LOCAL || source == MangaSource.DUMMY) { + continue + } + val parser = source.newParser(loaderContext) + if (parser is MangaParserAuthProvider) { + checkAuthorization(source, parser) + } + } + } - private fun checkAuthorization(source: MangaSource, parser: MangaParserAuthProvider) = runTest { - runCatching { - parser.getUsername() - }.onSuccess { username -> - println("Signed in to ${source.name} as $username") - }.onFailure { error -> - System.err.println("Auth failed for ${source.name}: ${error.javaClass.name}(${error.message})") - } - } -} \ No newline at end of file + private fun checkAuthorization(source: MangaSource, parser: MangaParserAuthProvider) = runTest { + runCatchingCancellable { + parser.getUsername() + }.onSuccess { username -> + println("Signed in to ${source.name} as $username") + }.onFailure { error -> + System.err.println("Auth failed for ${source.name}: ${error.javaClass.name}(${error.message})") + } + } +} diff --git a/src/test/kotlin/org/koitharu/kotatsu/parsers/CloudFlareInterceptor.kt b/src/test/kotlin/org/koitharu/kotatsu/parsers/CloudFlareInterceptor.kt index 5f9a64c3..8878779d 100644 --- a/src/test/kotlin/org/koitharu/kotatsu/parsers/CloudFlareInterceptor.kt +++ b/src/test/kotlin/org/koitharu/kotatsu/parsers/CloudFlareInterceptor.kt @@ -4,22 +4,25 @@ import okhttp3.Interceptor import okhttp3.Response import okhttp3.internal.closeQuietly import org.koitharu.kotatsu.parsers.exception.CloudFlareProtectedException -import java.net.HttpURLConnection.HTTP_FORBIDDEN -import java.net.HttpURLConnection.HTTP_UNAVAILABLE +import java.net.HttpURLConnection private const val HEADER_SERVER = "Server" private const val SERVER_CLOUDFLARE = "cloudflare" -class CloudFlareInterceptor : Interceptor { +internal class CloudFlareInterceptor : Interceptor { - override fun intercept(chain: Interceptor.Chain): Response { - val response = chain.proceed(chain.request()) - if (response.code == HTTP_FORBIDDEN || response.code == HTTP_UNAVAILABLE) { - if (response.header(HEADER_SERVER)?.startsWith(SERVER_CLOUDFLARE) == true) { - response.closeQuietly() - throw CloudFlareProtectedException(response.request.url.toString()) - } - } - return response - } -} \ No newline at end of file + override fun intercept(chain: Interceptor.Chain): Response { + val request = chain.request() + val response = chain.proceed(request) + if (response.code == HttpURLConnection.HTTP_FORBIDDEN || response.code == HttpURLConnection.HTTP_UNAVAILABLE) { + if (response.header(HEADER_SERVER)?.startsWith(SERVER_CLOUDFLARE) == true) { + response.closeQuietly() + throw CloudFlareProtectedException( + url = response.request.url.toString(), + headers = request.headers, + ) + } + } + return response + } +} diff --git a/src/test/kotlin/org/koitharu/kotatsu/parsers/CommonHeadersInterceptor.kt b/src/test/kotlin/org/koitharu/kotatsu/parsers/CommonHeadersInterceptor.kt new file mode 100644 index 00000000..fa67687b --- /dev/null +++ b/src/test/kotlin/org/koitharu/kotatsu/parsers/CommonHeadersInterceptor.kt @@ -0,0 +1,49 @@ +package org.koitharu.kotatsu.parsers + +import okhttp3.Interceptor +import okhttp3.Request +import okhttp3.Response +import org.koitharu.kotatsu.parsers.model.MangaSource +import org.koitharu.kotatsu.parsers.util.domain +import org.koitharu.kotatsu.parsers.util.mergeWith +import java.net.IDN + +private const val HEADER_USER_AGENT = "User-Agent" +private const val HEADER_REFERER = "Referer" + +internal class CommonHeadersInterceptor( + private val userAgent: String, +) : Interceptor { + + override fun intercept(chain: Interceptor.Chain): Response { + val request = chain.request() + val source = request.tag(MangaSource::class.java) + val parser = source?.newParser(MangaLoaderContextMock) + val sourceHeaders = parser?.headers + val headersBuilder = request.headers.newBuilder() + if (sourceHeaders != null) { + headersBuilder.mergeWith(sourceHeaders, replaceExisting = false) + } + if (headersBuilder[HEADER_USER_AGENT] == null) { + headersBuilder[HEADER_USER_AGENT] = userAgent + } + if (headersBuilder[HEADER_REFERER] == null && parser != null) { + val idn = IDN.toASCII(parser.domain) + headersBuilder[HEADER_REFERER] = "https://$idn/" + } + val newRequest = request.newBuilder().headers(headersBuilder.build()).build() + return if (parser is Interceptor) { + parser.intercept(ProxyChain(chain, newRequest)) + } else { + return chain.proceed(newRequest) + } + } + + private class ProxyChain( + private val delegate: Interceptor.Chain, + private val request: Request, + ) : Interceptor.Chain by delegate { + + override fun request(): Request = request + } +} diff --git a/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaLoaderContextMock.kt b/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaLoaderContextMock.kt index 95ebb59c..6a01b4cc 100644 --- a/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaLoaderContextMock.kt +++ b/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaLoaderContextMock.kt @@ -1,7 +1,6 @@ package org.koitharu.kotatsu.parsers import com.koushikdutta.quack.QuackContext -import okhttp3.Headers import okhttp3.OkHttpClient import okhttp3.Request import okhttp3.Response @@ -10,10 +9,10 @@ import org.koitharu.kotatsu.parsers.model.MangaSource import org.koitharu.kotatsu.parsers.util.await import java.util.concurrent.TimeUnit -internal class MangaLoaderContextMock : MangaLoaderContext() { +internal object MangaLoaderContextMock : MangaLoaderContext() { private val userAgent = "Kotatsu/%s (Android %s; %s; %s %s; %s)".format( - /*BuildConfig.VERSION_NAME*/ "3.0", + /*BuildConfig.VERSION_NAME*/ "4.4", /*Build.VERSION.RELEASE*/ "r", /*Build.MODEL*/ "", /*Build.BRAND*/ "", @@ -25,7 +24,7 @@ internal class MangaLoaderContextMock : MangaLoaderContext() { override val httpClient: OkHttpClient = OkHttpClient.Builder() .cookieJar(cookieJar) - .addInterceptor(UserAgentInterceptor(userAgent)) + .addInterceptor(CommonHeadersInterceptor(userAgent)) .addInterceptor(CloudFlareInterceptor()) .connectTimeout(20, TimeUnit.SECONDS) .readTimeout(60, TimeUnit.SECONDS) @@ -46,15 +45,12 @@ internal class MangaLoaderContextMock : MangaLoaderContext() { return SourceConfigMock() } - suspend fun doRequest(url: String, referer: String? = null, extraHeaders: Headers? = null): Response { + suspend fun doRequest(url: String, source: MangaSource?): Response { val request = Request.Builder() .get() .url(url) - if (extraHeaders != null) { - request.headers(extraHeaders) - } - if (referer != null) { - request.header("Referer", referer) + if (source != null) { + request.tag(MangaSource::class.java, source) } return httpClient.newCall(request.build()).await() } @@ -65,4 +61,4 @@ internal class MangaLoaderContextMock : MangaLoaderContext() { cookieJar.loadFromStream(it) } ?: println("No cookies loaded!") } -} \ No newline at end of file +} diff --git a/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaParserTest.kt b/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaParserTest.kt index 2c1ecdaf..84d7343a 100644 --- a/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaParserTest.kt +++ b/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaParserTest.kt @@ -8,24 +8,22 @@ import org.junit.jupiter.params.ParameterizedTest import org.koitharu.kotatsu.parsers.model.Manga import org.koitharu.kotatsu.parsers.model.MangaSource import org.koitharu.kotatsu.parsers.model.SortOrder +import org.koitharu.kotatsu.parsers.util.domain import org.koitharu.kotatsu.parsers.util.medianOrNull import org.koitharu.kotatsu.parsers.util.mimeType -import org.koitharu.kotatsu.test_util.isDistinct -import org.koitharu.kotatsu.test_util.isDistinctBy -import org.koitharu.kotatsu.test_util.isUrlAbsolute -import org.koitharu.kotatsu.test_util.maxDuplicates +import org.koitharu.kotatsu.test_util.* @ExtendWith(AuthCheckExtension::class) internal class MangaParserTest { - private val context = MangaLoaderContextMock() + private val context = MangaLoaderContextMock @ParameterizedTest(name = "{index}|list|{0}") @MangaSources fun list(source: MangaSource) = runTest { val parser = source.newParser(context) - val list = parser.getList(20, sortOrder = SortOrder.POPULARITY, tags = null) + val list = parser.getList(0, sortOrder = SortOrder.POPULARITY, tags = null) checkMangaList(list, "list") assert(list.all { it.source == source }) } @@ -36,6 +34,9 @@ internal class MangaParserTest { val parser = source.newParser(context) val page1 = parser.getList(0, sortOrder = null, tags = null) val page2 = parser.getList(page1.size, sortOrder = null, tags = null) + assert(page1.isNotEmpty()) { "Page 1 is empty" } + assert(page2.isNotEmpty()) { "Page 2 is empty" } + assert(page1 != page2) { "Pages are equal" } val intersection = page1.intersect(page2.toSet()) assert(intersection.isEmpty()) { "Pages are intersected by " + intersection.size @@ -46,12 +47,13 @@ internal class MangaParserTest { @MangaSources fun search(source: MangaSource) = runTest { val parser = source.newParser(context) - val subject = parser.getList(20, sortOrder = SortOrder.POPULARITY, tags = null).minByOrNull { + val subject = parser.getList(0, sortOrder = SortOrder.POPULARITY, tags = null).minByOrNull { it.title.length } ?: error("No manga found") val query = subject.title check(query.isNotBlank()) { "Manga title '$query' is blank" } val list = parser.getList(0, query) + assert(list.isNotEmpty()) { "Empty search results by \"$query\"" } assert(list.singleOrNull { it.url == subject.url && it.id == subject.id } != null) { "Single subject '${subject.title} (${subject.publicUrl})' not found in search results" } @@ -69,7 +71,7 @@ internal class MangaParserTest { assert(keys.isDistinct()) assert("" !in keys) val titles = tags.map { it.title } - assert(titles.isDistinct()) +// assert(titles.isDistinct()) assert("" !in titles) assert(tags.all { it.source == source }) @@ -83,7 +85,7 @@ internal class MangaParserTest { @MangaSources fun details(source: MangaSource) = runTest { val parser = source.newParser(context) - val list = parser.getList(20, sortOrder = SortOrder.POPULARITY, tags = null) + val list = parser.getList(0, sortOrder = SortOrder.POPULARITY, tags = null) val manga = list[3] parser.getDetails(manga).apply { assert(!chapters.isNullOrEmpty()) { "Chapters are null or empty" } @@ -101,9 +103,9 @@ internal class MangaParserTest { "Chapters are not distinct by number: ${c.maxDuplicates { it.number to it.branch }} for $publicUrl" } assert(c.all { it.source == source }) - checkImageRequest(coverUrl, publicUrl) + checkImageRequest(coverUrl, source) largeCoverUrl?.let { - checkImageRequest(it, publicUrl) + checkImageRequest(it, source) } } } @@ -112,20 +114,24 @@ internal class MangaParserTest { @MangaSources fun pages(source: MangaSource) = runTest { val parser = source.newParser(context) - val list = parser.getList(20, sortOrder = SortOrder.POPULARITY, tags = null) + val list = parser.getList(0, sortOrder = SortOrder.UPDATED, tags = null) val manga = list.first() - val chapter = parser.getDetails(manga).chapters?.firstOrNull() ?: error("Chapter is null") + val chapter = parser.getDetails(manga).chapters?.firstOrNull() ?: error("Chapter is null at ${manga.publicUrl}") val pages = parser.getPages(chapter) assert(pages.isNotEmpty()) assert(pages.isDistinctBy { it.id }) assert(pages.all { it.source == source }) - val page = pages.medianOrNull() ?: error("No page") - val pageUrl = parser.getPageUrl(page) - assert(pageUrl.isNotEmpty()) - assert(pageUrl.isUrlAbsolute()) - checkImageRequest(pageUrl, page.referer) + arrayOf( + pages.first(), + pages.medianOrNull() ?: error("No page"), + ).forEach { page -> + val pageUrl = parser.getPageUrl(page) + assert(pageUrl.isNotEmpty()) + assert(pageUrl.isUrlAbsolute()) + checkImageRequest(pageUrl, page.source) + } } @ParameterizedTest(name = "{index}|favicon|{0}") @@ -141,19 +147,19 @@ internal class MangaParserTest { } val favicon = favicons.find(24) checkNotNull(favicon) - checkImageRequest(favicon.url, favicons.referer) + checkImageRequest(favicon.url, source) } @ParameterizedTest(name = "{index}|domain|{0}") @MangaSources fun domain(source: MangaSource) = runTest { val parser = source.newParser(context) - val defaultDomain = parser.getDomain() + val defaultDomain = parser.domain val url = HttpUrl.Builder() .host(defaultDomain) .scheme("https") .toString() - val response = context.doRequest(url, extraHeaders = parser.headers) + val response = context.doRequest(url, source) val realUrl = response.request.url val realDomain = realUrl.topPrivateDomain() val realHost = realUrl.host @@ -185,15 +191,15 @@ internal class MangaParserTest { assert(item.publicUrl.isUrlAbsolute()) } val testItem = list.random() - checkImageRequest(testItem.coverUrl, testItem.publicUrl) + checkImageRequest(testItem.coverUrl, testItem.source) } - private suspend fun checkImageRequest(url: String, referer: String?) { - context.doRequest(url, referer).use { + private suspend fun checkImageRequest(url: String, source: MangaSource) { + context.doRequest(url, source).use { assert(it.isSuccessful) { "Request failed: ${it.code}(${it.message}): $url" } assert(it.mimeType?.startsWith("image/") == true) { "Wrong response mime type: ${it.mimeType}" } } } -} \ No newline at end of file +} diff --git a/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaSources.kt b/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaSources.kt index 5795da96..ef660faa 100644 --- a/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaSources.kt +++ b/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaSources.kt @@ -4,4 +4,4 @@ import org.junit.jupiter.params.provider.EnumSource import org.koitharu.kotatsu.parsers.model.MangaSource @EnumSource(MangaSource::class, names = ["LOCAL", "DUMMY"], mode = EnumSource.Mode.EXCLUDE) -internal annotation class MangaSources \ No newline at end of file +internal annotation class MangaSources diff --git a/src/test/kotlin/org/koitharu/kotatsu/parsers/UserAgentInterceptor.kt b/src/test/kotlin/org/koitharu/kotatsu/parsers/UserAgentInterceptor.kt deleted file mode 100644 index 10320160..00000000 --- a/src/test/kotlin/org/koitharu/kotatsu/parsers/UserAgentInterceptor.kt +++ /dev/null @@ -1,20 +0,0 @@ -package org.koitharu.kotatsu.parsers - -import okhttp3.Interceptor -import okhttp3.Response - -private const val HEADER_USER_AGENT = "User-Agent" -internal class UserAgentInterceptor( - private val userAgent: String, -) : Interceptor { - - override fun intercept(chain: Interceptor.Chain): Response { - val request = chain.request() - val newRequest = if (request.header(HEADER_USER_AGENT) == null) { - request.newBuilder().header(HEADER_USER_AGENT, userAgent).build() - } else { - request - } - return chain.proceed(newRequest) - } -} \ No newline at end of file