From de0c12c078230f570e813ee2d93964ac3214a72e Mon Sep 17 00:00:00 2001 From: Koitharu Date: Wed, 14 Sep 2022 20:07:34 +0300 Subject: [PATCH 001/121] Add contribution guidelines --- CONTRIBUTING.md | 90 +++++++++++++++++++++++++++++++++++++++++++++++++ README.md | 13 +++++-- 2 files changed, 101 insertions(+), 2 deletions(-) create mode 100644 CONTRIBUTING.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..3b38769b --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,90 @@ +# Contributing + +The following is guide for creating a Kotatsu parsers. Thanks for taking the time to contribute! + +## Prerequisites + +Before you start, please note that the ability to use following technologies is **required**. + +- Basic [Android development](https://developer.android.com/) +- [Kotlin](https://kotlinlang.org/) +- Web scraping ([JSoup](https://jsoup.org/)) or JSON API + +### Tools + +- [Android Studio](https://developer.android.com/studio) +- [IntelliJ IDEA](https://www.jetbrains.com/idea/) (Community edition is enough) +- Android device (or emulator) + +Kotatsu parsers is not a part of Android application, but you can easily develop and test it directly inside an Android +application project and relocate it to the library project when done. + +### Before you start + +First, take a look at `kotatsu-parsers` project structure. Each parser is a single class that +extends `MangaParser` class and have a `MangaSourceParser` annotation. +Also pay attention on extensions in `util` package. For example, extensions from `Jsoup` file +should be used instead of existing JSoup functions because they have better nullability support +and improved error messages. + +## Writing your parser + +So, you want to create a parser, that will provide access to manga from a website. +First, you should explore a website for API availability. +If it does not contain any documentation about +API, [explore network requests](https://firefox-source-docs.mozilla.org/devtools-user/): +some websites use ajax. + +- [Example](https://github.com/KotatsuApp/kotatsu-parsers/blob/master/src/main/kotlin/org/koitharu/kotatsu/parsers/site/DesuMeParser.kt) + of Json API usage. +- [Example](https://github.com/KotatsuApp/kotatsu-parsers/blob/master/src/main/kotlin/org/koitharu/kotatsu/parsers/site/AnibelParser.kt) + of GraphQL API usage +- [Example](https://github.com/KotatsuApp/kotatsu-parsers/blob/master/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaTownParser.kt) + of pure HTML parsing. + +If website is based on some engine it is rationally to use common base class for this one (for example, Madara wordress +theme +and the `MadaraParser` class) + +### Parser class skeleton + +Parser class must have exactly one primary constructor parameter of type `MangaLoaderContext` and have an +`MangaSourceParser` annotation that provides internal name, title and language of a manga source. + +All functions in `MangaParser` class are documented. Pay attention to some peculiarities: + +- Never hardcode domain. Specify default domain in `configKeyDomain` field and obtain an actual one using `getDomain()`. +- All ids must be unique and domain-independent. Use `generateUid` functions with relative url or some internal id which + is unique across the manga source. +- `sortOrders` set should not be empty. If your source is not support sorting, specify one most relevance value. +- If you cannot obtain direct links to pages images inside `getPages` method, it is ok to use an intermediate url + as `Page.url` and fetch a direct link at `getPageUrl` function. +- `getFaviconUrl` function is deprecated, but Kotatsu application before v4 still use it, so it is recommended to + override it too. +- You can use _asserts_ to check some optional fields. For example. `Manga.author` field is not required, but if your + source provide such information, add `assert(it != null)`. This will not have any effect on production but help to + find issues during unit testing. +- If your source website (or it's api) uses pages for pagination instead of offset you should extend `PagedMangaParser` + instead of `MangaParser`. + +## Development process + +During the development it is recommended (but not necessary) to write it directly +in the Kotatsu android application project. You can use `core.parser.DummyParser` class as a sandbox. `Dummy` manga +source is available in debug Kotatsu build. + +Once parser is ready you can relocate your code into `kotatsu-parsers` library project in a `site` package and create a +Pull Request. + +### Testing + +It is recommended to run unit tests before submitting a PR. + +- Temporary modify the `MangaSources` annotation class: specify your parser(s) name(s) and change mode + to `EnumSource.Mode.INCLUDE` +- Run the `MangaParserTest` (`gradlew :test --tests "org.koitharu.kotatsu.parsers.MangaParserTest"`) +- Optionally, you can run the `generateTestsReport` gradle task to get a pretty readable html report from test results. + +## Help + +If you need a help or have some questions, ask a community in our [Discord server](https://discord.gg/NNJ5RgVBC5). \ No newline at end of file diff --git a/README.md b/README.md index 504c9d0a..2dad2486 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Library that provides manga sources. [![](https://jitpack.io/v/KotatsuApp/kotatsu-parsers.svg)](https://jitpack.io/#KotatsuApp/kotatsu-parsers) ![Kotlin](https://img.shields.io/github/languages/top/KotatsuApp/kotatsu-parsers) ![License](https://img.shields.io/github/license/KotatsuApp/Kotatsu) [![Discord](https://img.shields.io/discord/898363402467045416?color=5865f2&label=discord)](https://discord.gg/NNJ5RgVBC5) -### Usage +## Usage 1. Add it in your root build.gradle at the end of repositories: @@ -48,4 +48,13 @@ Library that provides manga sources. and [Non-Android](https://github.com/KotatsuApp/kotatsu-dl/blob/master/src/main/kotlin/org/koitharu/kotatsu_dl/env/MangaLoaderContextImpl.kt) implementation examples. - Note that the `MangaSource.LOCAL` and `MangaSource.DUMMY` parsers cannot be instantiated. \ No newline at end of file + Note that the `MangaSource.LOCAL` and `MangaSource.DUMMY` parsers cannot be instantiated. + +## Contribution + +See [CONTRIBUTING.md](./CONTRIBUTING.md) for the guidelines. + +## DMCA disclaimer + +The developers of this application does not have any affiliation with the content available in the app. It is collecting +from the sources freely available through any web browser. \ No newline at end of file From e8b801684fd45405bec20f44e1b31eebff3e5bf4 Mon Sep 17 00:00:00 2001 From: Koitharu Date: Thu, 29 Sep 2022 14:25:41 +0300 Subject: [PATCH 002/121] [MangaDex] Fix error on empty description --- .../kotlin/org/koitharu/kotatsu/parsers/site/MangaDexParser.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaDexParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaDexParser.kt index db06ace9..b436111b 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaDexParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaDexParser.kt @@ -139,7 +139,7 @@ internal class MangaDexParser(override val context: MangaLoaderContext) : MangaP Locale.ROOT, ) manga.copy( - description = mangaAttrs.getJSONObject("description").selectByLocale() + description = mangaAttrs.optJSONObject("description")?.selectByLocale() ?: manga.description, chapters = feed.mapChapters { _, jo -> val id = jo.getString("id") From b1990c791897248aa5b75f24affcc98af41b3d67 Mon Sep 17 00:00:00 2001 From: Koitharu Date: Sat, 1 Oct 2022 13:43:09 +0300 Subject: [PATCH 003/121] Fixes --- .../koitharu/kotatsu/parsers/site/madara/MadaraParser.kt | 7 ++++--- .../kotatsu/parsers/site/madara/PrismaScansParser.kt | 3 ++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt index 06cf018c..d988e5df 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt @@ -284,9 +284,10 @@ internal abstract class MadaraParser( } private fun Element.src(): String? { - return absUrl("data-src").ifEmpty { - absUrl("src") - }.takeUnless { it.isEmpty() } + var result = absUrl("data-src") + if (result.isEmpty()) result = absUrl("data-cfsrc") + if (result.isEmpty()) result = absUrl("src") + return if (result.isEmpty()) null else result } private fun createRequestTemplate() = diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/PrismaScansParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/PrismaScansParser.kt index da7dff3a..6a95d095 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/PrismaScansParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/PrismaScansParser.kt @@ -17,6 +17,7 @@ internal class PrismaScansParser(context: MangaLoaderContext) : Madara6Parser(context, MangaSource.PRISMA_SCANS, "prismascans.net") { override val tagPrefix = "manga-genre/" + override val datePattern = "MMM dd, yyyy" override fun getFaviconUrl(): String { return "https://${getDomain()}/wp-content/uploads/2022/07/cropped-branca-1-192x192.png" @@ -33,7 +34,7 @@ internal class PrismaScansParser(context: MangaLoaderContext) : largeCoverUrl = root.selectFirst("picture") ?.selectFirst("img[data-src]") ?.attrAsAbsoluteUrlOrNull("data-src"), - description = root.selectFirstOrThrow(".description-summary").firstElementChild()?.html(), + description = root.selectFirstOrThrow(".manga-excerpt").firstElementChild()?.html(), author = postContent.getElementsContainingOwnText("Artista") .firstOrNull()?.tableValue()?.text()?.trim(), altTitle = postContent.getElementsContainingOwnText("Título Alternativo") From a45f9bfc58f372e4facbbdaf8cd9ea0b15c6cc68 Mon Sep 17 00:00:00 2001 From: Koitharu Date: Mon, 10 Oct 2022 15:13:30 +0300 Subject: [PATCH 004/121] ReaperScansID parser #83 --- .../koitharu/kotatsu/parsers/MangaParser.kt | 2 +- .../parsers/site/madara/ReaperScansParser.kt | 88 +++++++++++++++++++ 2 files changed, 89 insertions(+), 1 deletion(-) create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/ReaperScansParser.kt diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaParser.kt index 2788529c..0c2bc8fd 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaParser.kt @@ -25,7 +25,7 @@ abstract class MangaParser @InternalParsersApi constructor(val source: MangaSour val config by lazy { context.getConfig(source) } - val sourceLocale: Locale? + open val sourceLocale: Locale? get() = source.locale?.let { Locale(it) } /** diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/ReaperScansParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/ReaperScansParser.kt new file mode 100644 index 00000000..7e3a2002 --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/ReaperScansParser.kt @@ -0,0 +1,88 @@ +package org.koitharu.kotatsu.parsers.site.madara + +import org.jsoup.nodes.Element +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.model.* +import org.koitharu.kotatsu.parsers.util.* +import java.util.* + +@MangaSourceParser("REAPER_SCANS_ID", "ReaperScansID", "in") +internal class ReaperScansParser(context: MangaLoaderContext) : + Madara6Parser(context, MangaSource.REAPER_SCANS_ID, "reaperscans.id") { + + override val datePattern = "MMMM dd, yyyy" + override val tagPrefix = "genre/" + override val sourceLocale: Locale = Locale.ENGLISH + + override fun String.asMangaState(): MangaState? = when (this) { + "OnGoing", + "Upcoming", + -> MangaState.ONGOING + + "Completed", + "Dropped", + -> MangaState.FINISHED + + else -> null + } + + override fun parseDetails(manga: Manga, body: Element, chapters: List): Manga { + val root = body.selectFirstOrThrow(".site-content") + val postContent = root.requireElementById("nav-info") + val tags = postContent.getElementsContainingOwnText("Gênero") + .firstOrNull()?.tableValue() + ?.getElementsByAttributeValueContaining("href", tagPrefix) + ?.mapToSet { a -> a.asMangaTag() } ?: manga.tags + return manga.copy( + rating = postContent.selectFirstOrThrow(".post-rating") + .selectFirstOrThrow(".total_votes").text().toFloat() / 5f, + largeCoverUrl = root.selectFirst(".summary_image") + ?.selectFirst("img[data-src]") + ?.attrAsAbsoluteUrlOrNull("data-src") + .assertNotNull("largeCoverUrl"), + description = root.requireElementById("nav-profile") + .selectFirstOrThrow(".description-summary") + .firstElementChild()?.html(), + author = postContent.getElementsContainingOwnText("Author(s)") + .firstOrNull()?.tableValue()?.text()?.trim(), + altTitle = postContent.getElementsContainingOwnText("Alternative") + .firstOrNull()?.tableValue()?.text()?.trim(), + state = postContent.getElementsContainingOwnText("Status") + .firstOrNull()?.tableValue()?.text()?.asMangaState(), + tags = tags, + isNsfw = body.hasClass("adult-content"), + chapters = chapters, + ) + } + + override suspend fun getTags(): Set { + val doc = context.httpGet("https://${getDomain()}/semua-komik/").parseHtml() + val body = doc.body() + val root1 = body.selectFirst("header")?.selectFirst("ul.second-menu") + val root2 = body.selectFirst("div.genres_wrap")?.selectFirst("ul.list-unstyled") + if (root1 == null && root2 == null) { + doc.parseFailed("Root not found") + } + val list = root1?.select("li").orEmpty() + root2?.select("li").orEmpty() + val keySet = HashSet(list.size) + return list.mapNotNullToSet { li -> + val a = li.selectFirst("a") ?: return@mapNotNullToSet null + val href = a.attr("href").removeSuffix("/") + .substringAfterLast(tagPrefix, "") + if (href.isEmpty() || !keySet.add(href)) { + return@mapNotNullToSet null + } + MangaTag( + key = href, + title = a.ownText().trim().ifEmpty { + a.selectFirst(".menu-image-title")?.text()?.trim() ?: return@mapNotNullToSet null + }.toTitleCase(), + source = source, + ) + } + } + + override fun getFaviconUrl(): String = + "https://wuz.${getDomain()}/wp-content/uploads/2022/03/20063242/cropped-ms-icon-310x310-2-192x192.png" +} \ No newline at end of file From 505ffcf405cdef7943839141eb49e1165acef076 Mon Sep 17 00:00:00 2001 From: Koitharu Date: Mon, 10 Oct 2022 16:07:14 +0300 Subject: [PATCH 005/121] Mangalink parser #66 --- .../parsers/site/madara/MangalinkParser.kt | 66 +++++++++++++++++++ .../kotatsu/parsers/MangaParserTest.kt | 2 +- 2 files changed, 67 insertions(+), 1 deletion(-) create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangalinkParser.kt diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangalinkParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangalinkParser.kt new file mode 100644 index 00000000..4045728f --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangalinkParser.kt @@ -0,0 +1,66 @@ +package org.koitharu.kotatsu.parsers.site.madara + +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.exception.ParseException +import org.koitharu.kotatsu.parsers.model.Manga +import org.koitharu.kotatsu.parsers.model.MangaSource +import org.koitharu.kotatsu.parsers.model.MangaTag +import org.koitharu.kotatsu.parsers.model.SortOrder +import org.koitharu.kotatsu.parsers.util.mapNotNullToSet +import org.koitharu.kotatsu.parsers.util.parseHtml +import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl +import org.koitharu.kotatsu.parsers.util.toTitleCase + +@MangaSourceParser("MANGALINK_AR", "Mangalink", "ar") +internal class MangalinkParser(context: MangaLoaderContext) : + MadaraParser(context, MangaSource.MANGALINK_AR, "mangalink.online") { + + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + val res = super.getListPage(page, query, tags, sortOrder) + val oldDomain = getDomain() + val newDomain = "cdn.$oldDomain" + return res.map { x -> + x.copy(coverUrl = x.coverUrl.replace(oldDomain, newDomain)) + } + } + + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { + val fullUrl = manga.url.toAbsoluteUrl(getDomain()) + val doc = context.httpGet(fullUrl).parseHtml() + val chaptersDeferred = async { getChapters(manga, doc) } + val root = doc.body().selectFirst("div.profile-manga") + ?.selectFirst("div.summary_content") + ?.selectFirst("div.post-content") + ?: throw ParseException("Root not found", fullUrl) + val root2 = doc.body().selectFirst("div.content-area") + ?.selectFirst("div.c-page") + ?: throw ParseException("Root2 not found", fullUrl) + manga.copy( + tags = root.selectFirst("div.genres-content")?.select("a") + ?.mapNotNullToSet { a -> + MangaTag( + key = a.attr("href").removeSuffix("/").substringAfterLast('/'), + title = a.text().toTitleCase(), + source = source, + ) + } ?: manga.tags, + description = root2.selectFirst("div.description-summary") + ?.selectFirst("div.summary__content") + ?.select("p") + ?.filterNot { it.ownText().startsWith("A brief description") } + ?.joinToString { it.html() }, + chapters = chaptersDeferred.await(), + ) + } + + override fun getFaviconUrl(): String = + "https://cdn.${getDomain()}/wp-content/uploads/2020/05/cropped-mangalink-180x180.jpg" +} \ No newline at end of file diff --git a/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaParserTest.kt b/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaParserTest.kt index 2c1ecdaf..acffcc37 100644 --- a/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaParserTest.kt +++ b/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaParserTest.kt @@ -69,7 +69,7 @@ internal class MangaParserTest { assert(keys.isDistinct()) assert("" !in keys) val titles = tags.map { it.title } - assert(titles.isDistinct()) +// assert(titles.isDistinct()) assert("" !in titles) assert(tags.all { it.source == source }) From 5cb953eb8657bb7f323c602272d8a8a1d871d57c Mon Sep 17 00:00:00 2001 From: Koitharu Date: Mon, 10 Oct 2022 16:27:34 +0300 Subject: [PATCH 006/121] Small fixes --- buildSrc/src/main/resources/report.html | 121 +++++++++--------- .../parsers/site/madara/MadaraParser.kt | 7 +- 2 files changed, 69 insertions(+), 59 deletions(-) diff --git a/buildSrc/src/main/resources/report.html b/buildSrc/src/main/resources/report.html index 0a6c8dbf..7e7280ba 100644 --- a/buildSrc/src/main/resources/report.html +++ b/buildSrc/src/main/resources/report.html @@ -2,13 +2,15 @@ - + {{ testSuite.name }} - - + + + + @@ -25,67 +27,72 @@ {{ testSuite.errors }} ({{ error_percent }}%) -
- - - - - {% for test in tests %} - - {% endfor %} - - - {% for name, cases in results %} - - - {% for test in tests %} - {% set case = cases[test] %} - {% if case.failure == null %} - - {% else %} - {% if case.failure.type == 'java.lang.AssertionError' %} - - {% else %} - - {% endif %} - -
Source{{ test }}
{{ name }} - - - - - -
+ + + + {% for test in tests %} + + {% endfor %} + + + {% for name, cases in results %} + + + {% for test in tests %} + {% set case = cases[test] %} + {% if case.failure == null %} + + {% else %} + {% if case.failure.type == 'java.lang.AssertionError' %} + + {% else %} + + {% endif %} + + + + {% endif %} {% endfor %} -
Source{{ test }}
{{ name }} + + + + + +
-
+ + {% endfor %} + \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt index d988e5df..c26340af 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt @@ -322,10 +322,12 @@ internal abstract class MadaraParser( } } - @Deprecated("Not working, unknown reason, maybe shutdown or domain expired") @MangaSourceParser("MANGAHATACHI", "MangahaTachi", "ja") class MangahaTachi(context: MangaLoaderContext) : - MadaraParser(context, MangaSource.MANGAHATACHI, "mangahatachi.com") + MadaraParser(context, MangaSource.MANGAHATACHI, "mangahatachi.com") { + + override val sourceLocale: Locale = Locale.ENGLISH + } @MangaSourceParser("PIANMANGA", "PianManga", "en") class PianManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.PIANMANGA, "pianmanga.com") @@ -407,6 +409,7 @@ internal abstract class MadaraParser( @MangaSourceParser("MANHWAKOOL", "Manhwa Kool", "en") class ManhwaKool(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANHWAKOOL, "manhwakool.com") { + override val datePattern: String = "MM/dd" override fun getFaviconUrl(): String { return "https://${getDomain()}/wp-content/uploads/2021/10/cropped-logo-kool-32x32.jpeg" } From a62662d116fbb1c2bdac65c1a79c90781b9d0388 Mon Sep 17 00:00:00 2001 From: Koitharu Date: Wed, 19 Oct 2022 13:37:49 +0300 Subject: [PATCH 007/121] Fix some madara parsers --- .../kotatsu/parsers/site/madara/MadaraParser.kt | 9 ++++----- .../parsers/site/madara/MangalinkParser.kt | 15 --------------- .../koitharu/kotatsu/parsers/MangaParserTest.kt | 1 + 3 files changed, 5 insertions(+), 20 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt index c26340af..d69aee6e 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt @@ -57,10 +57,12 @@ internal abstract class MadaraParser( "https://${getDomain()}/wp-admin/admin-ajax.php", payload, ).parseHtml() - return doc.select("div.row.c-tabs-item__content").map { div -> + return doc.select("div.row.c-tabs-item__content").ifEmpty { + doc.select("div.page-item-detail.manga") + }.map { div -> val href = div.selectFirst("a")?.attrAsRelativeUrlOrNull("href") ?: div.parseFailed("Link not found") - val summary = div.selectFirst(".tab-summary") + val summary = div.selectFirst(".tab-summary") ?: div.selectFirst(".item-summary") Manga( id = generateUid(href), url = href, @@ -424,9 +426,6 @@ internal abstract class MadaraParser( @MangaSourceParser("X2MANGA", "X2Manga", "en") class X2Manga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.X2MANGA, "x2manga.com") - @MangaSourceParser("VINLOAD", "VinLoad", "en") - class VinLoad(context: MangaLoaderContext) : MadaraParser(context, MangaSource.VINLOAD, "vinload.com") - @MangaSourceParser("S2MANGA", "S2Manga", "en") class S2Manga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.S2MANGA, "s2manga.com") diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangalinkParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangalinkParser.kt index 4045728f..6303c813 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangalinkParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangalinkParser.kt @@ -8,7 +8,6 @@ import org.koitharu.kotatsu.parsers.exception.ParseException import org.koitharu.kotatsu.parsers.model.Manga import org.koitharu.kotatsu.parsers.model.MangaSource import org.koitharu.kotatsu.parsers.model.MangaTag -import org.koitharu.kotatsu.parsers.model.SortOrder import org.koitharu.kotatsu.parsers.util.mapNotNullToSet import org.koitharu.kotatsu.parsers.util.parseHtml import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl @@ -18,20 +17,6 @@ import org.koitharu.kotatsu.parsers.util.toTitleCase internal class MangalinkParser(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGALINK_AR, "mangalink.online") { - override suspend fun getListPage( - page: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - val res = super.getListPage(page, query, tags, sortOrder) - val oldDomain = getDomain() - val newDomain = "cdn.$oldDomain" - return res.map { x -> - x.copy(coverUrl = x.coverUrl.replace(oldDomain, newDomain)) - } - } - override suspend fun getDetails(manga: Manga): Manga = coroutineScope { val fullUrl = manga.url.toAbsoluteUrl(getDomain()) val doc = context.httpGet(fullUrl).parseHtml() diff --git a/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaParserTest.kt b/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaParserTest.kt index acffcc37..ea1f85eb 100644 --- a/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaParserTest.kt +++ b/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaParserTest.kt @@ -52,6 +52,7 @@ internal class MangaParserTest { val query = subject.title check(query.isNotBlank()) { "Manga title '$query' is blank" } val list = parser.getList(0, query) + assert(list.isNotEmpty()) { "Empty search results by \"$query\"" } assert(list.singleOrNull { it.url == subject.url && it.id == subject.id } != null) { "Single subject '${subject.title} (${subject.publicUrl})' not found in search results" } From a1441e7ed7b0bd1a6b838747fb9f8ef1401cf038 Mon Sep 17 00:00:00 2001 From: Koitharu Date: Thu, 20 Oct 2022 09:56:00 +0300 Subject: [PATCH 008/121] [MangaDex] Fix large chapters loading error --- .../koitharu/kotatsu/parsers/site/MangaDexParser.kt | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaDexParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaDexParser.kt index b436111b..a5632c46 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaDexParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaDexParser.kt @@ -20,6 +20,7 @@ private const val PAGE_SIZE = 20 private const val CHAPTERS_FIRST_PAGE_SIZE = 120 private const val CHAPTERS_MAX_PAGE_SIZE = 500 private const val CHAPTERS_PARALLELISM = 3 +private const val CHAPTERS_MAX_COUNT = 10_000 // strange api behavior, looks like a bug private const val CONTENT_RATING = "contentRating[]=safe&contentRating[]=suggestive&contentRating[]=erotica&contentRating[]=pornographic" private const val LOCALE_FALLBACK = "en" @@ -214,7 +215,7 @@ internal class MangaDexParser(override val context: MangaLoaderContext) : MangaP return firstPage.data } val tail = coroutineScope { - val leftCount = firstPage.total - firstPage.size + val leftCount = firstPage.total.coerceAtMost(CHAPTERS_MAX_COUNT) - firstPage.size val pages = (leftCount / CHAPTERS_MAX_PAGE_SIZE.toFloat()).toIntUp() val dispatcher = Dispatchers.Default.limitedParallelism(CHAPTERS_PARALLELISM) List(pages) { page -> @@ -231,6 +232,11 @@ internal class MangaDexParser(override val context: MangaLoaderContext) : MangaP } private suspend fun loadChapters(mangaId: String, offset: Int, limit: Int): Chapters { + val limitedLimit = when { + offset >= CHAPTERS_MAX_COUNT -> return Chapters(emptyList(), CHAPTERS_MAX_COUNT) + offset + limit > CHAPTERS_MAX_COUNT -> CHAPTERS_MAX_COUNT - offset + else -> limit + } val url = buildString { append("https://api.") append(getDomain()) @@ -238,7 +244,7 @@ internal class MangaDexParser(override val context: MangaLoaderContext) : MangaP append(mangaId) append("/feed") append("?limit=") - append(limit) + append(limitedLimit) append("&includes[]=scanlation_group&order[volume]=asc&order[chapter]=asc&offset=") append(offset) append('&') From 5d13c55dd5d3ea0a24bc490286dcf2fbe0e3fcf9 Mon Sep 17 00:00:00 2001 From: vianh Date: Sun, 23 Oct 2022 20:37:40 +0700 Subject: [PATCH 009/121] [BlogTruyen] Fix image src attr change --- .../kotatsu/parsers/site/BlogTruyenParser.kt | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BlogTruyenParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BlogTruyenParser.kt index 08c1514d..42cff91d 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BlogTruyenParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BlogTruyenParser.kt @@ -67,7 +67,7 @@ class BlogTruyenParser(override val context: MangaLoaderContext) : author = descriptionElement.selectFirst("p:contains(Tác giả) > a")?.text(), description = doc.selectFirst(".detail .content")?.html(), chapters = parseChapterList(doc), - largeCoverUrl = doc.selectLast("div.thumbnail > img")?.attrAsAbsoluteUrlOrNull("src"), + largeCoverUrl = doc.selectLast("div.thumbnail > img")?.imageUrl(), state = state, rating = rating ?: RATING_UNKNOWN, isNsfw = doc.getElementById("warningCategory") != null @@ -140,7 +140,7 @@ class BlogTruyenParser(override val context: MangaLoaderContext) : description = it.selectFirst("p.al-j.break.line-height-15")?.text(), url = relativeUrl, publicUrl = relativeUrl.toAbsoluteUrl(getDomain()), - coverUrl = linkTag.selectLast("img")?.attr("src").orEmpty(), + coverUrl = linkTag.selectLast("img")?.imageUrl().orEmpty(), source = source, tags = tags, isNsfw = false, @@ -165,7 +165,7 @@ class BlogTruyenParser(override val context: MangaLoaderContext) : description = mangaInfo.select("div.al-j.fs-12").text(), url = relativeUrl, publicUrl = relativeUrl.toAbsoluteUrl(getDomain()), - coverUrl = mangaInfo.selectFirst("div > img.img")?.absUrl("src").orEmpty(), + coverUrl = mangaInfo.selectFirst("div > img.img")?.imageUrl().orEmpty(), isNsfw = false, rating = RATING_UNKNOWN, tags = emptySet(), @@ -183,11 +183,10 @@ class BlogTruyenParser(override val context: MangaLoaderContext) : val pages = ArrayList() val referer = chapter.url.toAbsoluteUrl(getDomain()) doc.select("#content > img").forEach { img -> - val url = img.attrAsRelativeUrl("src") pages.add( MangaPage( - id = generateImageId(pages.lastIndex), - url = url, + id = generateImageId(pages.size), + url = img.imageUrl(), referer = referer, preview = null, source = source, @@ -204,7 +203,7 @@ class BlogTruyenParser(override val context: MangaLoaderContext) : val imageUrl = imageArr.getJSONObject(i).getString("url") pages.add( MangaPage( - id = generateImageId(pages.lastIndex), + id = generateImageId(pages.size), url = imageUrl, referer = referer, preview = null, @@ -245,4 +244,10 @@ class BlogTruyenParser(override val context: MangaLoaderContext) : cacheTags = tagMap tagMap } + + private fun Element.imageUrl(): String { + return attrAsAbsoluteUrlOrNull("src") + ?: attrAsAbsoluteUrlOrNull("data-cfsrc") + ?: "" + } } From ccf9af0e51f0633430d791d4e6e8b562881e106f Mon Sep 17 00:00:00 2001 From: vianh Date: Mon, 24 Oct 2022 23:05:13 +0700 Subject: [PATCH 010/121] [Manhwa18] New source --- .../kotatsu/parsers/site/Manhwa18Parser.kt | 180 ++++++++++++++++++ 1 file changed, 180 insertions(+) create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/Manhwa18Parser.kt diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/Manhwa18Parser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/Manhwa18Parser.kt new file mode 100644 index 00000000..dc29a337 --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/Manhwa18Parser.kt @@ -0,0 +1,180 @@ +package org.koitharu.kotatsu.parsers.site + +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.PagedMangaParser +import org.koitharu.kotatsu.parsers.config.ConfigKey +import org.koitharu.kotatsu.parsers.model.* +import org.koitharu.kotatsu.parsers.util.* +import java.util.* + +@MangaSourceParser("MANHWA18", "Manhwa18", "en") +class Manhwa18Parser(override val context: MangaLoaderContext) : PagedMangaParser(MangaSource.MANHWA18, pageSize = 20, searchPageSize = 20) { + + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("manhwa18.net", null) + + override val sortOrders: Set + get() = EnumSet.of(SortOrder.UPDATED, SortOrder.POPULARITY, SortOrder.ALPHABETICAL) + + override suspend fun getFavicons(): Favicons { + return Favicons(listOf( + Favicon("https://${getDomain()}/uploads/logos/logo-mini.png", 92, null) + ), getDomain()) + } + + override suspend fun getDetails(manga: Manga): Manga { + val docs = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml() + val cardInfoElement = docs.selectFirst(".card .manga-info") + val author = cardInfoElement?.selectFirst("b:contains(Author(s))")?.parent() + ?.select("a.btn") + ?.joinToString(", ") { it.text() } + val tags = cardInfoElement?.selectFirst("b:contains(Genre(s))")?.parent() + ?.select("a.btn") + ?.mapToSet { MangaTag(it.text(), it.text().lowercase(), MangaSource.MANHWA18) } + val state = cardInfoElement?.selectFirst("b:contains(Status)")?.parent() + ?.selectFirst("a.btn") + ?.let { + when (it.text()) { + "On going" -> MangaState.ONGOING + "Completed" -> MangaState.FINISHED + else -> null + } + } + + return manga.copy( + altTitle = cardInfoElement?.selectFirst("b:contains(Other names)")?.parent()?.ownText()?.removePrefix(": "), + author = author, + description = docs.selectFirst(".series-summary .summary-content")?.html(), + tags = tags.orEmpty(), + state = state, + chapters = docs.select(".card-body > .list-chapters > a").asReversed().mapChapters { index, element -> + // attrAsRelativeUrl only return page url without the '/' + val chapterUrl = element.attrAsAbsoluteUrlOrNull("href")?.toRelativeUrl(getDomain()) + ?: return@mapChapters null + val uploadDate = parseUploadDate(element.selectFirst(".chapter-time")?.text()) + MangaChapter( + id = generateUid(chapterUrl), + name = element.selectFirst(".chapter-name")?.text().orEmpty(), + number = index + 1, + url = chapterUrl, + scanlator = null, + uploadDate = uploadDate, + branch = null, + source = MangaSource.MANHWA18, + ) + } + ) + } + + // 7 minutes ago + // 5 hours ago + // 2 days ago + // 2 weeks ago + // 4 years ago + private fun parseUploadDate(timeStr: String?): Long { + timeStr ?: return 0 + + val timeWords = timeStr.split(' ') + if (timeWords.size != 3) return 0 + val timeWord = timeWords[1] + val timeAmount = timeWords[0].toIntOrNull() ?: return 0 + val timeUnit = when (timeWord) { + "minute", "minutes" -> Calendar.MINUTE + "hour", "hours" -> Calendar.HOUR + "day", "days" -> Calendar.DAY_OF_YEAR + "month", "months" -> Calendar.MONTH + "year", "years" -> Calendar.YEAR + else -> return 0 + } + val cal = Calendar.getInstance() + cal.add(timeUnit, timeAmount) + return cal.time.time + } + + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + val sortQuery = when(sortOrder) { + SortOrder.ALPHABETICAL -> "name" + SortOrder.POPULARITY -> "views" + SortOrder.UPDATED -> "last_update" + else -> "" + } + + val sortType = if (sortOrder == SortOrder.ALPHABETICAL) "ASC" else "DESC" + val tagQuery = tags.orEmpty().joinToString(",") { it.key } + val url = buildString { + append("https://") + append(getDomain()) + append("/manga-list.html?listType=pagination&page=") + append(page) + append("&artist=&author=&group=&m_status=&name=") + append(query.orEmpty()) + append("&genre=$tagQuery") + append("&ungenre=") + append("&sort=") + append(sortQuery) + append("&sort_type=") + append(sortType) + } + + val docs = context.httpGet(url).parseHtml() + val actualPage = docs.selectFirst("ul.pagination a.active")?.text()?.toIntOrNull() + if (actualPage != page) { + return emptyList() + } + + return docs.select(".card-body .thumb-item-flow") + .map { + val titleElement = it.selectFirstOrThrow(".thumb_attr.series-title > a") + val absUrl = titleElement.attrAsAbsoluteUrl("href") + Manga( + id = generateUid(absUrl.toRelativeUrl(getDomain())), + title = titleElement.text(), + altTitle = null, + url = absUrl.toRelativeUrl(getDomain()), + publicUrl = absUrl, + rating = RATING_UNKNOWN, + isNsfw = true, + coverUrl = it.selectFirst("div.img-in-ratio")?.attrAsAbsoluteUrl("data-bg").orEmpty(), + tags = emptySet(), + state = null, + author = null, + largeCoverUrl = null, + description = null, + source = MangaSource.MANHWA18, + ) + } + } + + override suspend fun getPages(chapter: MangaChapter): List { + val chapterUrl = chapter.url.toAbsoluteUrl(getDomain()) + return context.httpGet(chapterUrl).parseHtml() + .select("div.chapter-content > img").map { + val url = it.attrAsRelativeUrlOrNull("src").orEmpty() + MangaPage( + id = generateUid(url), + url = url, + referer = chapterUrl, + preview = null, + source = MangaSource.MANHWA18, + ) + } + } + + override suspend fun getTags(): Set { + return context.httpGet("https://${getDomain()}/").parseHtml().selectFirst(".genres-menu") + ?.select("a.genres-item").orEmpty() + .mapToSet { + MangaTag( + title = it.text(), + key = it.text().lowercase(), + source = MangaSource.MANHWA18, + ) + } + } +} \ No newline at end of file From 7907b3126cc72506e6ef913a4b9686a2f4897da4 Mon Sep 17 00:00:00 2001 From: ViAnh <37103340+VietAnh14@users.noreply.github.com> Date: Mon, 24 Oct 2022 23:29:53 +0700 Subject: [PATCH 011/121] Apply suggestions from code review Co-authored-by: Koitharu --- .../org/koitharu/kotatsu/parsers/site/Manhwa18Parser.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/Manhwa18Parser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/Manhwa18Parser.kt index dc29a337..5f6da4c7 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/Manhwa18Parser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/Manhwa18Parser.kt @@ -106,14 +106,14 @@ class Manhwa18Parser(override val context: MangaLoaderContext) : PagedMangaParse } val sortType = if (sortOrder == SortOrder.ALPHABETICAL) "ASC" else "DESC" - val tagQuery = tags.orEmpty().joinToString(",") { it.key } + val tagQuery = tags?.joinToString(",") { it.key }.orEmpty() val url = buildString { append("https://") append(getDomain()) append("/manga-list.html?listType=pagination&page=") append(page) append("&artist=&author=&group=&m_status=&name=") - append(query.orEmpty()) + append(query?.urlEncoded().orEmpty()) append("&genre=$tagQuery") append("&ungenre=") append("&sort=") @@ -167,7 +167,7 @@ class Manhwa18Parser(override val context: MangaLoaderContext) : PagedMangaParse } override suspend fun getTags(): Set { - return context.httpGet("https://${getDomain()}/").parseHtml().selectFirst(".genres-menu") + return context.httpGet("https://${getDomain()}/").parseHtml().selectFirstOrThrow(".genres-menu") ?.select("a.genres-item").orEmpty() .mapToSet { MangaTag( From bf8a1f3db22885429eefd1eea1429b9a00f65077 Mon Sep 17 00:00:00 2001 From: Zakhar Timoshenko Date: Fri, 4 Nov 2022 17:30:38 +0300 Subject: [PATCH 012/121] [MangaInUa] Fix #104 --- .../org/koitharu/kotatsu/parsers/site/MangaInUaParser.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaInUaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaInUaParser.kt index 928d85c9..e041a086 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaInUaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaInUaParser.kt @@ -45,7 +45,7 @@ class MangaInUaParser(override val context: MangaLoaderContext) : PagedMangaPars else -> "/mangas/page/$page".toAbsoluteUrl(getDomain()) } val doc = context.httpGet(url).parseHtml() - val container = doc.body().requireElementById("dle-content") + val container = doc.body().requireElementById("site-content") val items = container.select("div.col-6") return items.mapNotNull { item -> val href = item.selectFirst("a")?.attrAsRelativeUrl("href") ?: return@mapNotNull null @@ -81,7 +81,7 @@ class MangaInUaParser(override val context: MangaLoaderContext) : PagedMangaPars override suspend fun getDetails(manga: Manga): Manga { val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml() - val root = doc.body().requireElementById("dle-content") + val root = doc.body().requireElementById("site-content") val dateFormat = SimpleDateFormat("dd.MM.yyyy", Locale.US) val chapterNodes = root.selectFirstOrThrow(".linkstocomics").select(".ltcitems") var prevChapterName: String? = null From 1e49d4095b1b449ff3a506d572d0f8d45ddeff99 Mon Sep 17 00:00:00 2001 From: Koitharu Date: Mon, 21 Nov 2022 08:59:21 +0200 Subject: [PATCH 013/121] [Grouple] Fix pages parsing --- .../koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt index 1851e2a8..232db97b 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt @@ -202,8 +202,8 @@ internal abstract class GroupleParser( continue } val ja = JSONArray("[$json]") - val pages = ja.getJSONArray(1) - val servers = ja.getJSONArray(4).mapJSON { it.getString("path") } + val pages = ja.getJSONArray(0) + val servers = ja.getJSONArray(3).mapJSON { it.getString("path") } val serversStr = servers.joinToString("|") return (0 until pages.length()).map { i -> val page = pages.getJSONArray(i) From b78ddc616696a1558e884a595dc42ee43918f90a Mon Sep 17 00:00:00 2001 From: ViAnh <37103340+VietAnh14@users.noreply.github.com> Date: Sat, 26 Nov 2022 23:36:41 +0700 Subject: [PATCH 014/121] [NetTruyen] Update domain (#116) --- .../kotlin/org/koitharu/kotatsu/parsers/site/NetTruyenParser.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NetTruyenParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NetTruyenParser.kt index d05b9bf9..29171eeb 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NetTruyenParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NetTruyenParser.kt @@ -17,7 +17,7 @@ class NetTruyenParser(override val context: MangaLoaderContext) : PagedMangaParser(MangaSource.NETTRUYEN, pageSize = 36) { override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("www.nettruyenme.com", null) + get() = ConfigKey.Domain("nettruyenin.com", null) override val sortOrders: Set get() = EnumSet.of(SortOrder.UPDATED, SortOrder.POPULARITY, SortOrder.NEWEST, SortOrder.RATING) From 2ab03cb668d1db94669b8a54f513880666869da3 Mon Sep 17 00:00:00 2001 From: ViAnh <37103340+VietAnh14@users.noreply.github.com> Date: Sat, 26 Nov 2022 23:37:11 +0700 Subject: [PATCH 015/121] [Manhwa18] Fix parse upload date (#115) --- .../org/koitharu/kotatsu/parsers/site/Manhwa18Parser.kt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/Manhwa18Parser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/Manhwa18Parser.kt index 5f6da4c7..0f8b82b1 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/Manhwa18Parser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/Manhwa18Parser.kt @@ -83,12 +83,13 @@ class Manhwa18Parser(override val context: MangaLoaderContext) : PagedMangaParse "minute", "minutes" -> Calendar.MINUTE "hour", "hours" -> Calendar.HOUR "day", "days" -> Calendar.DAY_OF_YEAR + "week", "weeks" -> Calendar.WEEK_OF_YEAR "month", "months" -> Calendar.MONTH "year", "years" -> Calendar.YEAR else -> return 0 } val cal = Calendar.getInstance() - cal.add(timeUnit, timeAmount) + cal.add(timeUnit, -timeAmount) return cal.time.time } @@ -168,7 +169,7 @@ class Manhwa18Parser(override val context: MangaLoaderContext) : PagedMangaParse override suspend fun getTags(): Set { return context.httpGet("https://${getDomain()}/").parseHtml().selectFirstOrThrow(".genres-menu") - ?.select("a.genres-item").orEmpty() + .select("a.genres-item").orEmpty() .mapToSet { MangaTag( title = it.text(), From ad500a9db6b56ef2223de8c27d08b7e2a95d91ce Mon Sep 17 00:00:00 2001 From: vianh Date: Tue, 6 Dec 2022 23:17:31 +0700 Subject: [PATCH 016/121] [MangaReadTheme] Add some new sources using MangaReader theme --- .../site/mangareader/MangaReaderParser.kt | 317 ++++++++++++++++++ 1 file changed, 317 insertions(+) create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt new file mode 100644 index 00000000..bfa60129 --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt @@ -0,0 +1,317 @@ +package org.koitharu.kotatsu.parsers.site.mangareader + +import androidx.collection.ArrayMap +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import org.json.JSONObject +import org.jsoup.nodes.Document +import org.jsoup.nodes.Element +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.PagedMangaParser +import org.koitharu.kotatsu.parsers.config.ConfigKey +import org.koitharu.kotatsu.parsers.model.* +import org.koitharu.kotatsu.parsers.util.* +import java.text.SimpleDateFormat +import java.util.* + +internal abstract class MangaReaderParser( + source: MangaSource, + pageSize: Int, + searchPageSize: Int +) : PagedMangaParser(source, pageSize, searchPageSize) { + + override val sortOrders: Set + get() = EnumSet.of(SortOrder.UPDATED, SortOrder.POPULARITY, SortOrder.ALPHABETICAL, SortOrder.NEWEST) + + protected val idLocale + get() = Locale("in", "ID") + + abstract val listUrl: String + abstract val tableMode: Boolean + open val chapterDateFormat = SimpleDateFormat("MMM d, yyyy", idLocale) + + private var tagCache: ArrayMap? = null + private val mutex = Mutex() + private var lastSearchPage = 1 + + override suspend fun getDetails(manga: Manga): Manga { + val docs = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml() + val chapters = docs.select("#chapterlist > ul > li").mapChapters { index, element -> + val url = element.selectFirst("a")?.attrAsRelativeUrl("href") ?: return@mapChapters null + MangaChapter( + id = generateUid(url), + name = element.selectFirst(".chapternum")?.text() ?: "Chapter ${index + 1}", + url = url, + number = index + 1, + scanlator = null, + uploadDate = chapterDateFormat.tryParse(element.selectFirst(".chapterdate")?.text()), + branch = null, + source = source + ) + } + return if (tableMode) parseInfoTable(docs, manga, chapters) else parseInfoList(docs, manga, chapters) + } + + open suspend fun parseInfoTable(docs: Document, manga: Manga, chapters: List): Manga { + val mangaInfo = docs.selectFirst("div.seriestucontent > div.seriestucontentr") + val mangaState = mangaInfo?.selectFirst(".infotable td:contains(Status)")?.lastElementSibling()?.let { + when (it.text()) { + "Ongoing" -> MangaState.ONGOING + "Completed" -> MangaState.FINISHED + else -> null + } + } + val tagMap = getOrCreateTagMap() + val tags = mangaInfo?.select(".seriestugenre > a")?.mapNotNullToSet { tagMap[it.text()] } + + return manga.copy( + description = mangaInfo?.selectFirst("div.entry-content")?.html(), + state = mangaState, + author = mangaInfo?.selectFirst(".infotable td:contains(Author)")?.lastElementSibling()?.text(), + isNsfw = docs.selectFirst(".restrictcontainer") != null, + tags = tags.orEmpty(), + chapters = chapters, + ) + } + + open suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { + val mangaState = docs.selectFirst(".info-left .tsinfo div:contains(Status)")?.lastElementChild()?.let { + when (it.text()) { + "Ongoing" -> MangaState.ONGOING + "Completed" -> MangaState.FINISHED + else -> null + } + } + val tagMap = getOrCreateTagMap() + val tags = docs.select(".info-right .mgen > a").mapNotNullToSet { tagMap[it.text()] } + + return manga.copy( + description = docs.selectFirst(".info-right div.entry-content > p")?.html(), + state = mangaState, + author = docs.selectFirst(".info-left .tsinfo div:contains(Author)")?.lastElementChild()?.text(), + isNsfw = docs.selectFirst(".info-right .alr") != null, + tags = tags, + chapters = chapters, + ) + } + + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder + ): List { + if (!query.isNullOrEmpty()) { + if (page > lastSearchPage) { + return emptyList() + } + + val url = buildString { + append("https://") + append(getDomain()) + append("/page/") + append(page) + append("/?s=") + append(query.urlEncoded()) + } + + val docs = context.httpGet(url).parseHtml() + lastSearchPage = docs.selectFirst(".pagination .next") + ?.previousElementSibling() + ?.text()?.toIntOrNull() ?: 1 + return parseMangaList(docs) + } + + val sortQuery = when (sortOrder) { + SortOrder.ALPHABETICAL -> "title" + SortOrder.NEWEST -> "latest" + SortOrder.POPULARITY -> "popular" + SortOrder.UPDATED -> "update" + else -> "" + } + val tagKey = "genre[]".urlEncoded() + val tagQuery = if (tags.isNullOrEmpty()) "" else tags.joinToString(separator = "&", prefix = "&") { "$tagKey=${it.key}" } + val url = buildString { + append("https://") + append(getDomain()) + append(listUrl) + append("/?order=") + append(sortQuery) + append(tagQuery) + append("&page=") + append(page) + } + + return parseMangaList(context.httpGet(url).parseHtml()) + } + + private fun parseMangaList(docs: Document): List { + return docs.select(".postbody .listupd .bs .bsx").mapNotNull { + val a = it.selectFirst("a") ?: return@mapNotNull null + val relativeUrl = a.attrAsRelativeUrl("href") + val rating = it.selectFirst(".numscore")?.text() + ?.toFloatOrNull()?.div(10) ?: RATING_UNKNOWN + + Manga( + id = generateUid(relativeUrl), + url = relativeUrl, + title = a.attr("title"), + altTitle = null, + publicUrl = a.attrAsAbsoluteUrl("href"), + rating = rating, + isNsfw = false, + coverUrl = it.selectFirst("img.ts-post-image")?.imageUrl().orEmpty(), + tags = emptySet(), + state = null, + author = null, + source = source + ) + } + } + + override suspend fun getPages(chapter: MangaChapter): List { + val chapterUrl = chapter.url.toAbsoluteUrl(getDomain()) + val docs = context.httpGet(chapterUrl).parseHtml() + val script = docs.selectFirstOrThrow("script:containsData(ts_reader)") + val images = JSONObject(script.data().substringAfter('(').substringBeforeLast(')')) + .getJSONArray("sources") + .getJSONObject(0) + .getJSONArray("images") + + val pages = ArrayList(images.length()) + for (i in 0 until images.length()) { + pages.add( + MangaPage( + id = generateUid(images.getString(i)), + url = images.getString(i), + referer = chapterUrl, + preview = null, + source = source + ) + ) + } + + return pages + } + + override suspend fun getTags(): Set { + return getOrCreateTagMap().values.toSet() + } + + private suspend fun getOrCreateTagMap(): Map = mutex.withLock { + tagCache?.let { return@withLock it } + val tagMap = ArrayMap() + + val url = listUrl.toAbsoluteUrl(getDomain()) + val tagElements = context.httpGet(url).parseHtml().select("ul.genrez > li") + for (el in tagElements) { + if (el.text().isEmpty()) continue + + tagMap[el.text()] = MangaTag( + title = el.text(), + key = el.selectFirst(".genre-item")?.attr("value") ?: continue, + source = source + ) + } + + return@withLock tagMap + } + + private fun Element.imageUrl(): String { + return attrAsAbsoluteUrlOrNull("src") + ?: attrAsAbsoluteUrlOrNull("data-cfsrc") + ?: "" + } + + @MangaSourceParser("MANHWALAND", "Manhwaland", "id") + class ManhwaLandParser(override val context: MangaLoaderContext) : MangaReaderParser(MangaSource.MANHWALAND, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("manhwaland.guru", null) + + override val listUrl: String + get() = "/series" + override val tableMode: Boolean + get() = false + } + + @MangaSourceParser("HEROXIA", "Heroxia", "id") + class HeroxiaParser(override val context: MangaLoaderContext) : MangaReaderParser(MangaSource.HEROXIA, pageSize = 25, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("heroxia.com", null) + + override val tableMode: Boolean + get() = true + override val listUrl: String + get() = "/manga" + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMMM d, yyyy", Locale.ENGLISH) + } + + + @MangaSourceParser("SEKAIKOMIK", "Sekaikomik", "id") + class SekaikomikParser(override val context: MangaLoaderContext) : MangaReaderParser(MangaSource.SEKAIKOMIK, pageSize = 20, searchPageSize = 100) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("sekaikomik.pro", null) + + override val listUrl: String + get() = "/manga" + override val tableMode: Boolean + get() = false + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMMM D, yyyy", idLocale) + } + + @MangaSourceParser("MANHWAINDO", "Manhwaindo", "id") + class ManhwaIndoParser(override val context: MangaLoaderContext) : MangaReaderParser(MangaSource.MANHWAINDO, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("manhwaindo.id", null) + + override val listUrl: String get() = "/series" + override val tableMode: Boolean get() = false + } + + @MangaSourceParser("MANHWALIST", "Manhwalist", "id") + class ManhwalistParser(override val context: MangaLoaderContext) : MangaReaderParser(MangaSource.MANHWALIST, pageSize = 24, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("manhwalist.com", null) + + override val listUrl: String = "/manga" + override val tableMode: Boolean get() = false + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMMM d, yyyy", Locale.ENGLISH) + } + + @MangaSourceParser("KIRYUU", "Kiryuu", "id") + class KiryuuParser(override val context: MangaLoaderContext) : MangaReaderParser(MangaSource.KIRYUU, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("kiryuu.id", null) + + override val listUrl: String + get() = "/manga" + override val tableMode: Boolean + get() = true + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMMM d, yyyy", idLocale) + } + + @MangaSourceParser("TURKTOON", "Turktoon", "tr") + class TurktoonParser(override val context: MangaLoaderContext) : MangaReaderParser(MangaSource.TURKTOON, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("turktoon.com", null) + + override val listUrl: String + get() = "/manga" + override val tableMode: Boolean + get() = false + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMMM d, yyyy", Locale("tr", "TR")) + + override suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { + val mangaState = docs.selectFirst(".info-left .tsinfo div:contains(Durum)")?.lastElementChild()?.let { + when (it.text()) { + "Devam Ediyor" -> MangaState.ONGOING + "Tamamlandı" -> MangaState.FINISHED + else -> null + } + } + + return super.parseInfoList(docs, manga, chapters).copy(state = mangaState) + } + } +} \ No newline at end of file From 42253e43b323818a660d925c86a2cb2f8de78cb1 Mon Sep 17 00:00:00 2001 From: vianh Date: Wed, 7 Dec 2022 00:25:22 +0700 Subject: [PATCH 017/121] Add more mangareader theme sources, fix cache tags --- .../site/mangareader/MangaReaderParser.kt | 40 ++++++++++++++++++- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt index bfa60129..fb946fa0 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt @@ -199,7 +199,7 @@ internal abstract class MangaReaderParser( return getOrCreateTagMap().values.toSet() } - private suspend fun getOrCreateTagMap(): Map = mutex.withLock { + protected suspend fun getOrCreateTagMap(): Map = mutex.withLock { tagCache?.let { return@withLock it } val tagMap = ArrayMap() @@ -210,11 +210,12 @@ internal abstract class MangaReaderParser( tagMap[el.text()] = MangaTag( title = el.text(), - key = el.selectFirst(".genre-item")?.attr("value") ?: continue, + key = el.selectFirst("input")?.attr("value") ?: continue, source = source ) } + tagCache = tagMap return@withLock tagMap } @@ -314,4 +315,39 @@ internal abstract class MangaReaderParser( return super.parseInfoList(docs, manga, chapters).copy(state = mangaState) } } + + @MangaSourceParser("WESTMANGA", "Westmanga", "id") + class WestmangaParser(override val context: MangaLoaderContext) : MangaReaderParser(MangaSource.WESTMANGA, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("westmanga.info", null) + + override val listUrl: String + get() = "/manga" + override val tableMode: Boolean + get() = true + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMMM d, yyyy", Locale.ENGLISH) + } + + @MangaSourceParser("TEMPESTFANSUB", "Tempestfansub", "tr") + class TempestfansubParser(override val context: MangaLoaderContext) : MangaReaderParser(MangaSource.TEMPESTFANSUB, pageSize = 40, searchPageSize = 40) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("manga.tempestfansub.com", null) + + override val listUrl: String get() = "/manga" + override val tableMode: Boolean get() = true + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMMM d, yyyy", Locale("tr", "TR")) + + override suspend fun parseInfoTable(docs: Document, manga: Manga, chapters: List): Manga { + val infoElement = docs.selectFirst("div.infox") + return manga.copy( + chapters = chapters, + description = infoElement?.selectFirst("div.entry-content")?.html(), + author = infoElement?.selectFirst(".flex-wrap div:contains(Artist)")?.lastElementSibling()?.text(), + tags = infoElement?.select(".wd-full .mgen > a") + ?.mapNotNullToSet { getOrCreateTagMap()[it.text()] } + .orEmpty(), + isNsfw = docs.selectFirst(".postbody .alr") != null, + ) + } + } } \ No newline at end of file From add70b47904180fce15d44d1172ad38f5a39e8f0 Mon Sep 17 00:00:00 2001 From: Koitharu Date: Fri, 9 Dec 2022 18:11:27 +0200 Subject: [PATCH 018/121] [ExHentai] Fix pagination --- build.gradle | 2 +- .../kotatsu/parsers/site/ExHentaiParser.kt | 19 +++++++++++++++++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/build.gradle b/build.gradle index 9cae532e..965d76fb 100644 --- a/build.gradle +++ b/build.gradle @@ -57,7 +57,7 @@ dependencies { implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-core:1.6.4' implementation 'com.squareup.okhttp3:okhttp:4.10.0' implementation 'com.squareup.okio:okio:3.2.0' - api 'org.jsoup:jsoup:1.15.2' + api 'org.jsoup:jsoup:1.15.3' implementation 'org.json:json:20220320' implementation 'androidx.collection:collection-ktx:1.2.0' diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ExHentaiParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ExHentaiParser.kt index f53a20f7..62190c0f 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ExHentaiParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ExHentaiParser.kt @@ -1,5 +1,8 @@ package org.koitharu.kotatsu.parsers.site +import androidx.collection.SparseArrayCompat +import androidx.collection.set +import okhttp3.HttpUrl.Companion.toHttpUrlOrNull import org.jsoup.nodes.Element import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaParserAuthProvider @@ -33,6 +36,7 @@ internal class ExHentaiParser( private val ratingPattern = Regex("-?[0-9]+px") private val authCookies = arrayOf("ipb_member_id", "ipb_pass_hash") private var updateDm = false + private val nextPages = SparseArrayCompat() override val isAuthorized: Boolean get() { @@ -56,6 +60,7 @@ internal class ExHentaiParser( context.cookieJar.insertCookies(DOMAIN_UNAUTHORIZED, "nw=1", "sl=dm_2") } + override suspend fun getListPage( page: Int, query: String?, @@ -66,8 +71,8 @@ internal class ExHentaiParser( val url = buildString { append("https://") append(getDomain()) - append("/?page=") - append(page) + append("/?next=") + append(nextPages.get(page, 0L)) if (!tags.isNullOrEmpty()) { var fCats = 0 for (tag in tags) { @@ -88,6 +93,7 @@ internal class ExHentaiParser( if (updateDm) { append("&inline_set=dm_e") } + append("&advsearch=1&f_sh=on") } val body = context.httpGet(url).parseHtml().body() val root = body.selectFirst("table.itg") @@ -99,6 +105,7 @@ internal class ExHentaiParser( return getListPage(page, query, tags, sortOrder) } updateDm = false + nextPages[page + 1] = getNextTimestamp(root) return root.children().mapNotNull { tr -> if (tr.childrenSize() != 2) return@mapNotNull null val (td1, td2) = tr.children() @@ -276,4 +283,12 @@ internal class ExHentaiParser( val num = className.drop(2).toIntOrNull(16) ?: return null return 2.0.pow(num).toInt().toString() } + + private fun getNextTimestamp(root: Element): Long { + return root.getElementById("unext") + ?.attrAsAbsoluteUrlOrNull("href") + ?.toHttpUrlOrNull() + ?.queryParameter("next") + ?.toLongOrNull() ?: 1 + } } \ No newline at end of file From 63676c21a8b3f90589f2879f2dae73f104bcc8b2 Mon Sep 17 00:00:00 2001 From: vianh Date: Sat, 10 Dec 2022 14:55:43 +0700 Subject: [PATCH 019/121] [MangaReader] fix chapters order --- .../kotatsu/parsers/site/mangareader/MangaReaderParser.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt index fb946fa0..fd3c9160 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt @@ -37,7 +37,7 @@ internal abstract class MangaReaderParser( override suspend fun getDetails(manga: Manga): Manga { val docs = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml() - val chapters = docs.select("#chapterlist > ul > li").mapChapters { index, element -> + val chapters = docs.select("#chapterlist > ul > li").reversed().mapChapters { index, element -> val url = element.selectFirst("a")?.attrAsRelativeUrl("href") ?: return@mapChapters null MangaChapter( id = generateUid(url), From f9b0e82473835a112b8b86ee66e1dafbfe7688ee Mon Sep 17 00:00:00 2001 From: vianh Date: Sun, 11 Dec 2022 00:59:53 +0700 Subject: [PATCH 020/121] [MangaReader] Add new source --- .../parsers/site/mangareader/MangaReaderParser.kt | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt index fd3c9160..9eb6c01a 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt @@ -350,4 +350,13 @@ internal abstract class MangaReaderParser( ) } } + + @MangaSourceParser("MANHWADESU", "ManhwaDesu", "id") + class ManhwadesuParser(override val context: MangaLoaderContext) : MangaReaderParser(MangaSource.MANHWADESU, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("manhwadesu.org", null) + + override val listUrl: String get() = "/komik" + override val tableMode: Boolean get() = false + } } \ No newline at end of file From efd039439857e205fe00f07c2a626dd517203b53 Mon Sep 17 00:00:00 2001 From: Zakhar Timoshenko Date: Sun, 25 Dec 2022 12:22:13 +0300 Subject: [PATCH 021/121] [MangaTale] New parser (closes #125) --- .../site/mangareader/MangaReaderParser.kt | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt index 9eb6c01a..13cf138c 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt @@ -359,4 +359,28 @@ internal abstract class MangaReaderParser( override val listUrl: String get() = "/komik" override val tableMode: Boolean get() = false } + + @MangaSourceParser("MANGATALE", "MangaTale", "id") + class MangaTaleParser(override val context: MangaLoaderContext) : MangaReaderParser(MangaSource.MANGATALE, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("mangatale.co", null) + + override val listUrl: String get() = "/manga" + override val tableMode: Boolean get() = false + + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMMM d, yyyy", Locale.ENGLISH) + + override suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { + val infoElement = docs.selectFirst("div.infox") + return manga.copy( + chapters = chapters, + description = infoElement?.selectFirst("div.entry-content")?.html(), + author = infoElement?.selectFirst(".flex-wrap div:contains(Author)")?.lastElementSibling()?.text(), + tags = infoElement?.select(".wd-full .mgen > a") + ?.mapNotNullToSet { getOrCreateTagMap()[it.text()] } + .orEmpty(), + isNsfw = docs.selectFirst(".postbody .alr") != null, + ) + } + } } \ No newline at end of file From c7c142f4a06dc5340f8103686df09ee3fe0e7ce1 Mon Sep 17 00:00:00 2001 From: Koitharu Date: Mon, 26 Dec 2022 17:45:35 +0200 Subject: [PATCH 022/121] [Remanga] Show bought chapters --- build.gradle | 4 +--- .../org/koitharu/kotatsu/parsers/site/RemangaParser.kt | 5 ++++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/build.gradle b/build.gradle index 965d76fb..b8d86236 100644 --- a/build.gradle +++ b/build.gradle @@ -1,5 +1,3 @@ -import tasks.ReportGenerateTask - plugins { id 'java-library' id 'org.jetbrains.kotlin.jvm' @@ -70,4 +68,4 @@ dependencies { testImplementation 'io.webfolder:quickjs:1.1.0' } -task generateTestsReport(type: ReportGenerateTask) \ No newline at end of file +tasks.register('generateTestsReport') \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/RemangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/RemangaParser.kt index a7e39e73..63f86686 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/RemangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/RemangaParser.kt @@ -140,7 +140,10 @@ internal class RemangaParser( ) }, chapters = chapters.mapChapters { i, jo -> - if (jo.getBooleanOrDefault("is_paid", false)) { + if ( + jo.getBooleanOrDefault("is_paid", false) && + !jo.getBooleanOrDefault("is_bought", false) + ) { return@mapChapters null } val id = jo.getLong("id") From 9ee1c21a67e7d8fcfb8419274bbab4fb00c01f5c Mon Sep 17 00:00:00 2001 From: Koitharu Date: Mon, 26 Dec 2022 18:23:15 +0200 Subject: [PATCH 023/121] [ExHentai] Fix pagination --- .../kotlin/org/koitharu/kotatsu/parsers/site/ExHentaiParser.kt | 2 +- src/test/kotlin/org/koitharu/kotatsu/parsers/MangaParserTest.kt | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ExHentaiParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ExHentaiParser.kt index 62190c0f..286372bc 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ExHentaiParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ExHentaiParser.kt @@ -105,7 +105,7 @@ internal class ExHentaiParser( return getListPage(page, query, tags, sortOrder) } updateDm = false - nextPages[page + 1] = getNextTimestamp(root) + nextPages[page + 1] = getNextTimestamp(body) return root.children().mapNotNull { tr -> if (tr.childrenSize() != 2) return@mapNotNull null val (td1, td2) = tr.children() diff --git a/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaParserTest.kt b/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaParserTest.kt index ea1f85eb..4abd1506 100644 --- a/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaParserTest.kt +++ b/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaParserTest.kt @@ -36,6 +36,8 @@ internal class MangaParserTest { val parser = source.newParser(context) val page1 = parser.getList(0, sortOrder = null, tags = null) val page2 = parser.getList(page1.size, sortOrder = null, tags = null) + assert(page1.isNotEmpty()) { "Page 1 is empty" } + assert(page2.isNotEmpty()) { "Page 2 is empty" } val intersection = page1.intersect(page2.toSet()) assert(intersection.isEmpty()) { "Pages are intersected by " + intersection.size From fb22eb5facd7d4f5926b7e9f32930d18bda80ae5 Mon Sep 17 00:00:00 2001 From: Zakhar Timoshenko Date: Wed, 28 Dec 2022 23:10:17 +0300 Subject: [PATCH 024/121] [Tempest Fan Sub] New parser (closes #56) --- .../site/mangareader/MangaReaderParser.kt | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt index 13cf138c..f3205606 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt @@ -383,4 +383,28 @@ internal abstract class MangaReaderParser( ) } } + + @MangaSourceParser("TEMPEST", "Tempest Manga", "tr") + class TempestParser(override val context: MangaLoaderContext) : MangaReaderParser(MangaSource.TEMPEST, pageSize = 30, searchPageSize = 30) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("manga.tempestfansub.com", null) + + override val listUrl: String get() = "/manga" + override val tableMode: Boolean get() = false + + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", Locale("tr")) + + override suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { + val infoElement = docs.selectFirst("div.infox") + return manga.copy( + chapters = chapters, + description = infoElement?.selectFirst("div.entry-content")?.html(), + author = infoElement?.selectFirst(".flex-wrap div:contains(Yapımcı)")?.lastElementSibling()?.text(), + tags = infoElement?.select(".wd-full .mgen > a") + ?.mapNotNullToSet { getOrCreateTagMap()[it.text()] } + .orEmpty(), + isNsfw = docs.selectFirst(".postbody .alr") != null, + ) + } + } } \ No newline at end of file From 238cc7529a8494bcae7b5f3e783db2416b04d153 Mon Sep 17 00:00:00 2001 From: Zakhar Timoshenko Date: Wed, 28 Dec 2022 23:26:26 +0300 Subject: [PATCH 025/121] [Asura Scans (tr)] New parser (closes #55) --- .../site/mangareader/MangaReaderParser.kt | 26 ++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt index f3205606..b85d6ddd 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt @@ -385,7 +385,7 @@ internal abstract class MangaReaderParser( } @MangaSourceParser("TEMPEST", "Tempest Manga", "tr") - class TempestParser(override val context: MangaLoaderContext) : MangaReaderParser(MangaSource.TEMPEST, pageSize = 30, searchPageSize = 30) { + class TempestParser(override val context: MangaLoaderContext) : MangaReaderParser(MangaSource.TEMPEST, pageSize = 40, searchPageSize = 40) { override val configKeyDomain: ConfigKey.Domain get() = ConfigKey.Domain("manga.tempestfansub.com", null) @@ -407,4 +407,28 @@ internal abstract class MangaReaderParser( ) } } + + @MangaSourceParser("ASURATR", "Asura Scans (tr)", "tr") + class AsuraTRParser(override val context: MangaLoaderContext) : MangaReaderParser(MangaSource.ASURATR, pageSize = 30, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("asurascanstr.com", null) + + override val listUrl: String get() = "/manga" + override val tableMode: Boolean get() = false + + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", Locale("tr")) + + override suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { + val infoElement = docs.selectFirst("div.infox") + return manga.copy( + chapters = chapters, + description = infoElement?.selectFirst("div.entry-content")?.html(), + author = infoElement?.selectFirst(".flex-wrap div:contains(Yazar)")?.lastElementSibling()?.text(), + tags = infoElement?.select(".wd-full .mgen > a") + ?.mapNotNullToSet { getOrCreateTagMap()[it.text()] } + .orEmpty(), + isNsfw = docs.selectFirst(".postbody .alr") != null, + ) + } + } } \ No newline at end of file From e89f06a35bfea8f81f7f5dabee7c5fbeb4aac0d8 Mon Sep 17 00:00:00 2001 From: Zakhar Timoshenko Date: Thu, 29 Dec 2022 11:09:25 +0300 Subject: [PATCH 026/121] [KomikTap] New parser (closes #40) (?) --- .../parsers/site/mangareader/MangaReaderParser.kt | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt index b85d6ddd..ee9e59a6 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt @@ -431,4 +431,16 @@ internal abstract class MangaReaderParser( ) } } + + @MangaSourceParser("KOMIKTAP", "KomikTap", "id") + class KomikTapParser(override val context: MangaLoaderContext) : MangaReaderParser(MangaSource.KOMIKTAP, pageSize = 15, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("92.87.6.124", null) + + override val listUrl: String + get() = "/manga" + override val tableMode: Boolean + get() = true + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", idLocale) + } } \ No newline at end of file From ad6e074cce79344b22c8ba557bfb02ee5597f18f Mon Sep 17 00:00:00 2001 From: Zakhar Timoshenko Date: Thu, 29 Dec 2022 18:12:22 +0300 Subject: [PATCH 027/121] [Asura Scans] New parser --- .../site/mangareader/MangaReaderParser.kt | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt index ee9e59a6..8f0446d0 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt @@ -441,6 +441,33 @@ internal abstract class MangaReaderParser( get() = "/manga" override val tableMode: Boolean get() = true + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", idLocale) } + + @MangaSourceParser("ASURASCANS", "Asura Scans", "en") + class AsuraScansParser(override val context: MangaLoaderContext) : MangaReaderParser(MangaSource.ASURASCANS, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("asurascans.com", null) + + override val listUrl: String + get() = "/manga" + override val tableMode: Boolean + get() = false + + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH) + + override suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { + val infoElement = docs.selectFirst("div.infox") + return manga.copy( + chapters = chapters, + description = infoElement?.selectFirst("div.entry-content")?.html(), + author = infoElement?.selectFirst(".flex-wrap div:contains(Author)")?.lastElementSibling()?.text(), + tags = infoElement?.select(".wd-full .mgen > a") + ?.mapNotNullToSet { getOrCreateTagMap()[it.text()] } + .orEmpty(), + isNsfw = docs.selectFirst(".postbody .alr") != null, + ) + } + } } \ No newline at end of file From 106a85aa80e8f5a31e7b13f52b1ebf67756c3927 Mon Sep 17 00:00:00 2001 From: Zakhar Timoshenko Date: Sat, 31 Dec 2022 19:03:06 +0300 Subject: [PATCH 028/121] [Lib] Authorization fixes --- .../parsers/site/rulib/MangaLibParser.kt | 28 +++++++++++++++++-- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/rulib/MangaLibParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/rulib/MangaLibParser.kt index 8c0bb874..c2d641f3 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/rulib/MangaLibParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/rulib/MangaLibParser.kt @@ -110,8 +110,10 @@ internal open class MangaLibParser( append(item.getInt("chapter_volume")) append("/c") append(item.getString("chapter_number")) - append('/') - append(item.optString("chapter_string")) + if (isAuthorized) { + append("?ui=") + append(getUID()) + } } val nameChapter = item.getStringOrNull("chapter_name") val volume = item.getInt("chapter_volume") @@ -233,13 +235,27 @@ internal open class MangaLibParser( } override suspend fun getUsername(): String { - val body = context.httpGet("https://${getDomain()}/messages").parseHtml().body() + val body = context.httpGet("https://${LibConst.LIB_SOCIAL_LINK}/messages").parseHtml().body() if (body.baseUri().endsWith("/login")) { throw AuthRequiredException(source) } return body.selectFirst(".profile-user__username")?.text() ?: body.parseFailed("Cannot find username") } + private suspend fun getUID(): String { + val url = "https://${getDomain()}/news/polzovatelskoe-soglasenie" + val body = context.httpGet(url).parseHtml().body() + val scripts = body.select("script") + for (script in scripts) { + val raw = script.html().trim() + if (raw.startsWith("window.__DATA")) { + val json = JSONObject(raw.substringAfter('=').substringBeforeLast(';')) + return json.getJSONObject("user").getInt("id").toString() + } + } + throw AuthRequiredException(source) + } + protected open fun isNsfw(doc: Document): Boolean { val sidebar = doc.body().run { selectFirst(".media-sidebar") ?: selectFirst(".media-info") @@ -285,4 +301,10 @@ internal open class MangaLibParser( @MangaSourceParser("MANGALIB", "MangaLib", "ru") class Impl(context: MangaLoaderContext) : MangaLibParser(context, MangaSource.MANGALIB) + + object LibConst { + + val LIB_SOCIAL_LINK = "lib.social" + + } } \ No newline at end of file From 04e219ff6921b415c8f5a8fa1012399c5acc2c55 Mon Sep 17 00:00:00 2001 From: Zakhar Timoshenko Date: Sat, 31 Dec 2022 19:41:36 +0300 Subject: [PATCH 029/121] Remove accidentally duplicated parser --- .../site/mangareader/MangaReaderParser.kt | 37 +++++++------------ 1 file changed, 13 insertions(+), 24 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt index 8f0446d0..badb4476 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt @@ -384,30 +384,6 @@ internal abstract class MangaReaderParser( } } - @MangaSourceParser("TEMPEST", "Tempest Manga", "tr") - class TempestParser(override val context: MangaLoaderContext) : MangaReaderParser(MangaSource.TEMPEST, pageSize = 40, searchPageSize = 40) { - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("manga.tempestfansub.com", null) - - override val listUrl: String get() = "/manga" - override val tableMode: Boolean get() = false - - override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", Locale("tr")) - - override suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { - val infoElement = docs.selectFirst("div.infox") - return manga.copy( - chapters = chapters, - description = infoElement?.selectFirst("div.entry-content")?.html(), - author = infoElement?.selectFirst(".flex-wrap div:contains(Yapımcı)")?.lastElementSibling()?.text(), - tags = infoElement?.select(".wd-full .mgen > a") - ?.mapNotNullToSet { getOrCreateTagMap()[it.text()] } - .orEmpty(), - isNsfw = docs.selectFirst(".postbody .alr") != null, - ) - } - } - @MangaSourceParser("ASURATR", "Asura Scans (tr)", "tr") class AsuraTRParser(override val context: MangaLoaderContext) : MangaReaderParser(MangaSource.ASURATR, pageSize = 30, searchPageSize = 10) { override val configKeyDomain: ConfigKey.Domain @@ -470,4 +446,17 @@ internal abstract class MangaReaderParser( ) } } + + @MangaSourceParser("MANGASUSU", "Mangasusu", "id") + class MangasusuParser(override val context: MangaLoaderContext) : MangaReaderParser(MangaSource.MANGASUSU, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("194.233.87.209", null) + + override val listUrl: String + get() = "/project" + override val tableMode: Boolean + get() = true + + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH) + } } \ No newline at end of file From c4acb9725f04713feedbf65cbea2c52b6e77512b Mon Sep 17 00:00:00 2001 From: Zakhar Timoshenko Date: Sat, 31 Dec 2022 22:28:46 +0300 Subject: [PATCH 030/121] [Lib] Fix auth part 2 --- .../parsers/site/rulib/MangaLibParser.kt | 33 ++++++++----------- 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/rulib/MangaLibParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/rulib/MangaLibParser.kt index c2d641f3..27872a76 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/rulib/MangaLibParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/rulib/MangaLibParser.kt @@ -98,6 +98,7 @@ internal open class MangaLibParser( if (line.startsWith("window.__DATA__")) { val json = JSONObject(line.substringAfter('=').substringBeforeLast(';')) val list = json.getJSONObject("chapters").getJSONArray("list") + val id = json.optJSONObject("user")?.getLong("id")?.toString() ?: "not" val total = list.length() chapters = ChaptersListBuilder(total) for (i in 0 until total) { @@ -105,14 +106,20 @@ internal open class MangaLibParser( val chapterId = item.getLong("chapter_id") val scanlator = item.getStringOrNull("username") val url = buildString { - append(manga.url) - append("/v") - append(item.getInt("chapter_volume")) - append("/c") - append(item.getString("chapter_number")) if (isAuthorized) { + append(manga.url) + append("/v") + append(item.getInt("chapter_volume")) + append("/c") + append(item.getString("chapter_number")) append("?ui=") - append(getUID()) + append(id) + } else { + append(manga.url) + append("/v") + append(item.getInt("chapter_volume")) + append("/c") + append(item.getString("chapter_number")) } } val nameChapter = item.getStringOrNull("chapter_name") @@ -242,20 +249,6 @@ internal open class MangaLibParser( return body.selectFirst(".profile-user__username")?.text() ?: body.parseFailed("Cannot find username") } - private suspend fun getUID(): String { - val url = "https://${getDomain()}/news/polzovatelskoe-soglasenie" - val body = context.httpGet(url).parseHtml().body() - val scripts = body.select("script") - for (script in scripts) { - val raw = script.html().trim() - if (raw.startsWith("window.__DATA")) { - val json = JSONObject(raw.substringAfter('=').substringBeforeLast(';')) - return json.getJSONObject("user").getInt("id").toString() - } - } - throw AuthRequiredException(source) - } - protected open fun isNsfw(doc: Document): Boolean { val sidebar = doc.body().run { selectFirst(".media-sidebar") ?: selectFirst(".media-info") From ef838068ff0d9632288dc9176845e30257309ca6 Mon Sep 17 00:00:00 2001 From: Zakhar Timoshenko Date: Sun, 1 Jan 2023 20:37:04 +0300 Subject: [PATCH 031/121] [MangahaTachi] Change url (closes #128) --- .../org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt index d69aee6e..5eb1e561 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt @@ -326,7 +326,7 @@ internal abstract class MadaraParser( @MangaSourceParser("MANGAHATACHI", "MangahaTachi", "ja") class MangahaTachi(context: MangaLoaderContext) : - MadaraParser(context, MangaSource.MANGAHATACHI, "mangahatachi.com") { + MadaraParser(context, MangaSource.MANGAHATACHI, "hachimanga.com") { override val sourceLocale: Locale = Locale.ENGLISH } From cf0073202377352c8cfc8f169d63bcc54719deab Mon Sep 17 00:00:00 2001 From: Koitharu Date: Thu, 5 Jan 2023 11:26:39 +0200 Subject: [PATCH 032/121] Misc parsers fixes --- build.gradle | 5 ++- buildSrc/src/main/resources/report.html | 31 +++++++++++++++++++ .../kotatsu/parsers/site/ComickFunParser.kt | 2 +- .../kotatsu/parsers/site/ExHentaiParser.kt | 9 ++++-- .../kotatsu/parsers/site/NudeMoonParser.kt | 5 ++- 5 files changed, 47 insertions(+), 5 deletions(-) diff --git a/build.gradle b/build.gradle index b8d86236..cbf87048 100644 --- a/build.gradle +++ b/build.gradle @@ -1,3 +1,5 @@ +import tasks.ReportGenerateTask + plugins { id 'java-library' id 'org.jetbrains.kotlin.jvm' @@ -68,4 +70,5 @@ dependencies { testImplementation 'io.webfolder:quickjs:1.1.0' } -tasks.register('generateTestsReport') \ No newline at end of file +//noinspection ConfigurationAvoidance +task generateTestsReport(type: ReportGenerateTask) \ No newline at end of file diff --git a/buildSrc/src/main/resources/report.html b/buildSrc/src/main/resources/report.html index 7e7280ba..b18a0dad 100644 --- a/buildSrc/src/main/resources/report.html +++ b/buildSrc/src/main/resources/report.html @@ -51,6 +51,22 @@ data-bs-toggle="modal" style="cursor: pointer;"> + {% elseif case.failure.type == 'java.net.SocketTimeoutException' or case.failure.type == + 'java.net.UnknownHostException' %} + + + + {% elseif case.failure.type == 'org.koitharu.kotatsu.parsers.exception.CloudFlareProtectedException' %} + + + + {% elseif case.failure.type == 'org.koitharu.kotatsu.parsers.exception.AuthRequiredException' %} + + + {% else %} @@ -88,6 +104,21 @@ + + + + + + + + + + + + + + + diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ComickFunParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ComickFunParser.kt index 06acadeb..ab7d471d 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ComickFunParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ComickFunParser.kt @@ -24,7 +24,7 @@ private const val CHAPTERS_LIMIT = 99999 @MangaSourceParser("COMICK_FUN", "ComicK") internal class ComickFunParser(override val context: MangaLoaderContext) : MangaParser(MangaSource.COMICK_FUN) { - override val configKeyDomain = ConfigKey.Domain("comick.fun", null) + override val configKeyDomain = ConfigKey.Domain("comick.app", null) override val sortOrders: Set = EnumSet.of( SortOrder.POPULARITY, diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ExHentaiParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ExHentaiParser.kt index 286372bc..13feeebf 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ExHentaiParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ExHentaiParser.kt @@ -58,9 +58,9 @@ internal class ExHentaiParser( init { context.cookieJar.insertCookies(DOMAIN_AUTHORIZED, "nw=1", "sl=dm_2") context.cookieJar.insertCookies(DOMAIN_UNAUTHORIZED, "nw=1", "sl=dm_2") + paginator.firstPage = 0 } - override suspend fun getListPage( page: Int, query: String?, @@ -68,11 +68,16 @@ internal class ExHentaiParser( sortOrder: SortOrder, ): List { var search = query?.urlEncoded().orEmpty() + val next = nextPages.get(page, 0L) + if (page > 0 && next == 0L) { + assert(false) { "Page timestamp not found" } + return emptyList() + } val url = buildString { append("https://") append(getDomain()) append("/?next=") - append(nextPages.get(page, 0L)) + append(next) if (!tags.isNullOrEmpty()) { var fCats = 0 for (tag in tags) { diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NudeMoonParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NudeMoonParser.kt index 67520445..0bde4f09 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NudeMoonParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NudeMoonParser.kt @@ -18,7 +18,10 @@ internal class NudeMoonParser( override val context: MangaLoaderContext, ) : MangaParser(MangaSource.NUDEMOON), MangaParserAuthProvider { - override val configKeyDomain = ConfigKey.Domain("nude-moon.net", null) + override val configKeyDomain = ConfigKey.Domain( + defaultValue = "nude-moon.org", + presetValues = arrayOf("nude-moon.org", "nude-moon.net"), + ) override val authUrl: String get() = "https://${getDomain()}/index.php" From e153463c354326722f6b55c7e46b4c251125f98b Mon Sep 17 00:00:00 2001 From: Koitharu Date: Tue, 24 Jan 2023 07:58:56 +0200 Subject: [PATCH 033/121] [Grouple] Fix authorization recognizing --- .../kotatsu/parsers/site/grouple/GroupleParser.kt | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt index 232db97b..370944c4 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt @@ -140,7 +140,7 @@ internal abstract class GroupleParser( } override suspend fun getDetails(manga: Manga): Manga { - val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain()), headers).parseHtml() + val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain()), headers).checkAuthRequired().parseHtml() val root = doc.body().getElementById("mangaBox")?.selectFirst("div.leftContent") ?: doc.parseFailed("Cannot find root") val dateFormat = SimpleDateFormat("dd.MM.yy", Locale.US) @@ -186,7 +186,9 @@ internal abstract class GroupleParser( } override suspend fun getPages(chapter: MangaChapter): List { - val doc = context.httpGet(chapter.url.toAbsoluteUrl(getDomain()) + "?mtr=1", headers).parseHtml() + val doc = context.httpGet(chapter.url.toAbsoluteUrl(getDomain()) + "?mtr=1", headers) + .checkAuthRequired() + .parseHtml() val scripts = doc.select("script") for (script in scripts) { val data = script.html() @@ -307,4 +309,12 @@ internal abstract class GroupleParser( private suspend fun tryHead(url: String, headers: Headers): Boolean = runCatching { context.httpHead(url, headers).isSuccessful }.getOrDefault(false) + + private fun Response.checkAuthRequired(): Response { + val lastPathSegment = request.url.pathSegments.lastOrNull() ?: return this + if (lastPathSegment == "login") { + throw AuthRequiredException(source) + } + return this + } } \ No newline at end of file From 918318252ebbec74e1d824351df4a395de2f2762 Mon Sep 17 00:00:00 2001 From: Koitharu Date: Tue, 24 Jan 2023 08:03:51 +0200 Subject: [PATCH 034/121] Fix coroutine cancellation handling in runCatching --- .../kotatsu/parsers/site/NetTruyenParser.kt | 7 +++++-- .../kotatsu/parsers/site/grouple/GroupleParser.kt | 2 +- .../org/koitharu/kotatsu/parsers/util/Result.kt | 13 +++++++++++++ .../koitharu/kotatsu/parsers/AuthCheckExtension.kt | 3 ++- 4 files changed, 21 insertions(+), 4 deletions(-) create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/util/Result.kt diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NetTruyenParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NetTruyenParser.kt index 29171eeb..afe8664f 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NetTruyenParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NetTruyenParser.kt @@ -4,7 +4,9 @@ import androidx.collection.ArrayMap import androidx.collection.ArraySet import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock -import org.koitharu.kotatsu.parsers.* +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.PagedMangaParser import org.koitharu.kotatsu.parsers.config.ConfigKey import org.koitharu.kotatsu.parsers.exception.NotFoundException import org.koitharu.kotatsu.parsers.model.* @@ -89,6 +91,7 @@ class NetTruyenParser(override val context: MangaLoaderContext) : val currentYear = calendar.get(Calendar.YEAR).toString().takeLast(2) "$relativeDate/$currentYear" } + 3 -> relativeDate else -> return 0L } @@ -124,7 +127,7 @@ class NetTruyenParser(override val context: MangaLoaderContext) : } val response = if (isSearching) { - val result = runCatching { context.httpGet(url) } + val result = runCatchingCancellable { context.httpGet(url) } val exception = result.exceptionOrNull() if (exception is NotFoundException) { return emptyList() diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt index 370944c4..006aac28 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt @@ -306,7 +306,7 @@ internal abstract class GroupleParser( return context.httpPost(url, payload, headers) } - private suspend fun tryHead(url: String, headers: Headers): Boolean = runCatching { + private suspend fun tryHead(url: String, headers: Headers): Boolean = runCatchingCancellable { context.httpHead(url, headers).isSuccessful }.getOrDefault(false) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Result.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Result.kt new file mode 100644 index 00000000..17732922 --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Result.kt @@ -0,0 +1,13 @@ +package org.koitharu.kotatsu.parsers.util + +import kotlinx.coroutines.CancellationException + +inline fun T.runCatchingCancellable(block: T.() -> R): Result { + return try { + Result.success(block()) + } catch (e: CancellationException) { + throw e + } catch (e: Throwable) { + Result.failure(e) + } +} \ No newline at end of file diff --git a/src/test/kotlin/org/koitharu/kotatsu/parsers/AuthCheckExtension.kt b/src/test/kotlin/org/koitharu/kotatsu/parsers/AuthCheckExtension.kt index d058e68e..0dfbe83f 100644 --- a/src/test/kotlin/org/koitharu/kotatsu/parsers/AuthCheckExtension.kt +++ b/src/test/kotlin/org/koitharu/kotatsu/parsers/AuthCheckExtension.kt @@ -4,6 +4,7 @@ import kotlinx.coroutines.test.runTest import org.junit.jupiter.api.extension.BeforeAllCallback import org.junit.jupiter.api.extension.ExtensionContext import org.koitharu.kotatsu.parsers.model.MangaSource +import org.koitharu.kotatsu.parsers.util.runCatchingCancellable class AuthCheckExtension : BeforeAllCallback { @@ -22,7 +23,7 @@ class AuthCheckExtension : BeforeAllCallback { } private fun checkAuthorization(source: MangaSource, parser: MangaParserAuthProvider) = runTest { - runCatching { + runCatchingCancellable { parser.getUsername() }.onSuccess { username -> println("Signed in to ${source.name} as $username") From e1d22b1110da6856ef5ae44db9e8ea62d7ad1337 Mon Sep 17 00:00:00 2001 From: Koitharu Date: Tue, 24 Jan 2023 08:57:19 +0200 Subject: [PATCH 035/121] [ExHentai] Option to hide suspicious content --- .../org/koitharu/kotatsu/parsers/config/ConfigKey.kt | 4 ++++ .../koitharu/kotatsu/parsers/site/ExHentaiParser.kt | 11 ++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/config/ConfigKey.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/config/ConfigKey.kt index c6c623d1..3ff6e276 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/config/ConfigKey.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/config/ConfigKey.kt @@ -10,4 +10,8 @@ sealed class ConfigKey( override val defaultValue: String, val presetValues: Array?, ) : ConfigKey("domain") + + class ShowSuspiciousContent( + override val defaultValue: Boolean, + ) : ConfigKey("show_suspicious") } \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ExHentaiParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ExHentaiParser.kt index 13feeebf..6432667f 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ExHentaiParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ExHentaiParser.kt @@ -37,6 +37,7 @@ internal class ExHentaiParser( private val authCookies = arrayOf("ipb_member_id", "ipb_pass_hash") private var updateDm = false private val nextPages = SparseArrayCompat() + private val suspiciousContentKey = ConfigKey.ShowSuspiciousContent(true) override val isAuthorized: Boolean get() { @@ -98,7 +99,10 @@ internal class ExHentaiParser( if (updateDm) { append("&inline_set=dm_e") } - append("&advsearch=1&f_sh=on") + append("&advsearch=1") + if (config[suspiciousContentKey]) { + append("&f_sh=on") + } } val body = context.httpGet(url).parseHtml().body() val root = body.selectFirst("table.itg") @@ -235,6 +239,11 @@ internal class ExHentaiParser( return username } + override fun onCreateConfig(keys: MutableCollection>) { + super.onCreateConfig(keys) + keys.add(suspiciousContentKey) + } + private fun isAuthorized(domain: String): Boolean { val cookies = context.cookieJar.getCookies(domain).mapToSet { x -> x.name } return authCookies.all { it in cookies } From e5a6b828538f64d27e16b67235416155f370cf23 Mon Sep 17 00:00:00 2001 From: Koitharu Date: Tue, 24 Jan 2023 09:11:09 +0200 Subject: [PATCH 036/121] [Grouple] Fix chapter link extraction --- .../koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt index 006aac28..492229a1 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt @@ -162,7 +162,7 @@ internal abstract class GroupleParser( isNsfw = root.select(".alert-warning").any { it.ownText().contains(NSFW_ALERT) }, chapters = root.selectFirst("div.chapters-link")?.selectFirst("table") ?.select("tr:has(td > a)")?.asReversed()?.mapChapters { i, tr -> - val a = tr.selectFirst("a") ?: return@mapChapters null + val a = tr.selectFirst("a.chapter-link") ?: return@mapChapters null val href = a.attrAsRelativeUrl("href") var translators = "" val translatorElement = a.attr("title") @@ -176,7 +176,7 @@ internal abstract class GroupleParser( name = tr.selectFirst("a")?.text().orEmpty().removePrefix(manga.title).trim(), number = i + 1, url = href, - uploadDate = dateFormat.tryParse(tr.selectFirst("td.d-none")?.text()), + uploadDate = dateFormat.tryParse(tr.selectFirst("td.date")?.text()), scanlator = translators, source = source, branch = null, From 7f630184c02207c4b58c0f5e11db4259444f628e Mon Sep 17 00:00:00 2001 From: Zakhar Timoshenko Date: Sun, 29 Jan 2023 20:54:52 +0300 Subject: [PATCH 037/121] [GroupLe] Fix pages loading --- .../koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt index 492229a1..04fb602d 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt @@ -192,7 +192,7 @@ internal abstract class GroupleParser( val scripts = doc.select("script") for (script in scripts) { val data = script.html() - val pos = data.indexOf("rm_h.initReader(") + val pos = data.indexOf("rm_h.readerInit( 0,") if (pos == -1) { continue } @@ -204,7 +204,7 @@ internal abstract class GroupleParser( continue } val ja = JSONArray("[$json]") - val pages = ja.getJSONArray(0) + val pages = ja.getJSONArray(1) val servers = ja.getJSONArray(3).mapJSON { it.getString("path") } val serversStr = servers.joinToString("|") return (0 until pages.length()).map { i -> From c28e2a72d57e2188b6ec8839d4a9b2fda6e8bffb Mon Sep 17 00:00:00 2001 From: Zakhar Timoshenko Date: Sun, 5 Feb 2023 10:09:53 +0300 Subject: [PATCH 038/121] [Remanga] Bypass blocking --- .../kotlin/org/koitharu/kotatsu/parsers/site/RemangaParser.kt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/RemangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/RemangaParser.kt index 63f86686..dcdcdab5 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/RemangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/RemangaParser.kt @@ -31,6 +31,10 @@ internal class RemangaParser( override val context: MangaLoaderContext, ) : PagedMangaParser(MangaSource.REMANGA, PAGE_SIZE), MangaParserAuthProvider { + override val headers = Headers.Builder() + .add("User-Agent", "Mozilla/5.0 (Android 13; Mobile; rv:68.0) Gecko/68.0 Firefox/109.0") + .build() + override val configKeyDomain = ConfigKey.Domain("remanga.org", null) override val authUrl: String get() = "https://${getDomain()}/user/login" From 176437531dd18ed5170338a50d39a97f353bf65f Mon Sep 17 00:00:00 2001 From: Saksham Shekher <95137948+OshekharO@users.noreply.github.com> Date: Sun, 5 Feb 2023 22:14:48 +0530 Subject: [PATCH 039/121] Add: CosmicScans (#142) * Add: CosmicScans * Fix info * Rem: nsfw * Mangasu url changed Cert issue needs to be fixed --- .../site/mangareader/MangaReaderParser.kt | 29 +++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt index badb4476..db515ce2 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt @@ -447,10 +447,35 @@ internal abstract class MangaReaderParser( } } + @MangaSourceParser("COSMICSCANS", "CosmicScans", "en") + class CosmicScansParser(override val context: MangaLoaderContext) : MangaReaderParser(MangaSource.COSMICSCANS, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("cosmicscans.com", null) + + override val listUrl: String + get() = "/manga" + override val tableMode: Boolean + get() = false + + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH) + + override suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { + val infoElement = docs.selectFirst("div.infox") + return manga.copy( + chapters = chapters, + description = infoElement?.selectFirst("div.entry-content")?.html(), + author = infoElement?.selectFirst(".flex-wrap div:contains(Author)")?.lastElementSibling()?.text(), + tags = infoElement?.select(".wd-full .mgen > a") + ?.mapNotNullToSet { getOrCreateTagMap()[it.text()] } + .orEmpty(), + ) + } + } + @MangaSourceParser("MANGASUSU", "Mangasusu", "id") class MangasusuParser(override val context: MangaLoaderContext) : MangaReaderParser(MangaSource.MANGASUSU, pageSize = 20, searchPageSize = 10) { override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("194.233.87.209", null) + get() = ConfigKey.Domain("92.84.132.251", null) override val listUrl: String get() = "/project" @@ -459,4 +484,4 @@ internal abstract class MangaReaderParser( override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH) } -} \ No newline at end of file +} From fc2781c83a564cc17dee40e3fd88c48844dd9122 Mon Sep 17 00:00:00 2001 From: Saksham Shekher <95137948+OshekharO@users.noreply.github.com> Date: Mon, 6 Feb 2023 13:43:02 +0530 Subject: [PATCH 040/121] Add: KomikManga --- .../parsers/site/mangareader/MangaReaderParser.kt | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt index db515ce2..dd14093c 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt @@ -484,4 +484,17 @@ internal abstract class MangaReaderParser( override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH) } + + @MangaSourceParser("KOMIKMANGA", "KomikManga", "id") + class KomikMangaParser(override val context: MangaLoaderContext) : MangaReaderParser(MangaSource.KOMIKMANGA, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("komikmanga.xyz", null) + + override val listUrl: String + get() = "/project" + override val tableMode: Boolean + get() = true + + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH) + } } From 9b78944f9b77792970b2b0a92bfade1abe8e69ac Mon Sep 17 00:00:00 2001 From: Saksham Shekher <95137948+OshekharO@users.noreply.github.com> Date: Mon, 6 Feb 2023 13:49:31 +0530 Subject: [PATCH 041/121] Fix: ManhwaList --- .../kotatsu/parsers/site/mangareader/MangaReaderParser.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt index dd14093c..fdd3881f 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt @@ -273,7 +273,7 @@ internal abstract class MangaReaderParser( @MangaSourceParser("MANHWALIST", "Manhwalist", "id") class ManhwalistParser(override val context: MangaLoaderContext) : MangaReaderParser(MangaSource.MANHWALIST, pageSize = 24, searchPageSize = 10) { override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("manhwalist.com", null) + get() = ConfigKey.Domain("manhwalist.in", null) override val listUrl: String = "/manga" override val tableMode: Boolean get() = false From 06ff8df2e4ffe61aebc7fae75e69d76a5aa50284 Mon Sep 17 00:00:00 2001 From: Saksham Shekher <95137948+OshekharO@users.noreply.github.com> Date: Mon, 6 Feb 2023 13:52:37 +0530 Subject: [PATCH 042/121] Dead: Heroxia --- .../parsers/site/mangareader/MangaReaderParser.kt | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt index fdd3881f..c878a56a 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt @@ -236,19 +236,6 @@ internal abstract class MangaReaderParser( get() = false } - @MangaSourceParser("HEROXIA", "Heroxia", "id") - class HeroxiaParser(override val context: MangaLoaderContext) : MangaReaderParser(MangaSource.HEROXIA, pageSize = 25, searchPageSize = 10) { - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("heroxia.com", null) - - override val tableMode: Boolean - get() = true - override val listUrl: String - get() = "/manga" - override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMMM d, yyyy", Locale.ENGLISH) - } - - @MangaSourceParser("SEKAIKOMIK", "Sekaikomik", "id") class SekaikomikParser(override val context: MangaLoaderContext) : MangaReaderParser(MangaSource.SEKAIKOMIK, pageSize = 20, searchPageSize = 100) { override val configKeyDomain: ConfigKey.Domain From 4799f54fd1e34385ecc9640e20339d4369967f9e Mon Sep 17 00:00:00 2001 From: Saksham Shekher <95137948+OshekharO@users.noreply.github.com> Date: Mon, 6 Feb 2023 13:57:30 +0530 Subject: [PATCH 043/121] Dead: All Top Manga --- .../kotatsu/parsers/site/madara/MadaraParser.kt | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt index 5eb1e561..6e070980 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt @@ -519,14 +519,6 @@ internal abstract class MadaraParser( } } - @MangaSourceParser("ALLTOPMANGA", "All Top Manga", "en") - class AllTopManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.ALLTOPMANGA, "alltopmanga.com") { - - override fun getFaviconUrl(): String { - return "https://${getDomain()}/wp-content/uploads/2021/12/cropped-Screenshot_4-removebg-preview-32x32.png" - } - } - @MangaSourceParser("MANGACV", "Manga Cv", "en") class MangaCv(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGACV, "mangacv.com") { @@ -554,4 +546,4 @@ internal abstract class MadaraParser( @MangaSourceParser("ZINMANGA", "ZINMANGA", "en") class Zinmanga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.ZINMANGA, "zinmanga.com") -} \ No newline at end of file +} From f68f0009a22268fae65dfa5877a9602d077af1e3 Mon Sep 17 00:00:00 2001 From: Saksham Shekher <95137948+OshekharO@users.noreply.github.com> Date: Mon, 6 Feb 2023 14:00:07 +0530 Subject: [PATCH 044/121] Dead: ManhwaChill --- .../org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt index 6e070980..e0759871 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt @@ -506,9 +506,6 @@ internal abstract class MadaraParser( override val isNsfwSource = true } - @MangaSourceParser("MANHWA_CHILL", "Manhwa Chill", "en") - class ManhwaChill(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANHWA_CHILL, "manhwachill.me") - @MangaSourceParser("TREE_MANGA", "Tree Manga", "en") class TreeManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.TREE_MANGA, "treemanga.com") { From cf7f55181f4acc17f4860240cb9c7be896d045da Mon Sep 17 00:00:00 2001 From: Saksham Shekher <95137948+OshekharO@users.noreply.github.com> Date: Mon, 6 Feb 2023 14:03:26 +0530 Subject: [PATCH 045/121] Dead: Manhwa --- .../org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt index e0759871..26648a56 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt @@ -429,9 +429,6 @@ internal abstract class MadaraParser( @MangaSourceParser("S2MANGA", "S2Manga", "en") class S2Manga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.S2MANGA, "s2manga.com") - @MangaSourceParser("MANHWA", "Manhwa", "en") - class Manhwa(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANHWA, "manhwa.pics") - @MangaSourceParser("SKY_MANGA", "Sky Manga", "en") class SkyManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.SKY_MANGA, "skymanga.xyz") { From 2dedaea0abf76a4c197bcf0317715a73b530a276 Mon Sep 17 00:00:00 2001 From: Saksham Shekher <95137948+OshekharO@users.noreply.github.com> Date: Mon, 6 Feb 2023 14:05:36 +0530 Subject: [PATCH 046/121] Dead: KingManga --- .../koitharu/kotatsu/parsers/site/madara/MadaraParser.kt | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt index 26648a56..b8b57824 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt @@ -316,14 +316,6 @@ internal abstract class MadaraParser( override val datePattern = "dd MMMM HH:mm" } - @MangaSourceParser("KINGMANGA", "KingManga", "en") - class KingManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.KINGMANGA, "king-manga.com") { - - override fun getFaviconUrl(): String { - return "https://${getDomain()}/wp-content/uploads/2022/05/cropped-logo-king-02-03-32x32.png" - } - } - @MangaSourceParser("MANGAHATACHI", "MangahaTachi", "ja") class MangahaTachi(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGAHATACHI, "hachimanga.com") { From 742533977f66f78a8efc3e1f29cbb39199f407b1 Mon Sep 17 00:00:00 2001 From: Saksham Shekher <95137948+OshekharO@users.noreply.github.com> Date: Mon, 6 Feb 2023 14:06:39 +0530 Subject: [PATCH 047/121] Dead: X2Manga --- .../org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt index b8b57824..d45d7a61 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt @@ -415,9 +415,6 @@ internal abstract class MadaraParser( override val datePattern = "MM/dd/yyyy" } - @MangaSourceParser("X2MANGA", "X2Manga", "en") - class X2Manga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.X2MANGA, "x2manga.com") - @MangaSourceParser("S2MANGA", "S2Manga", "en") class S2Manga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.S2MANGA, "s2manga.com") From 7a78a677402f9fe9e276a2a37093b7d8ad05843d Mon Sep 17 00:00:00 2001 From: Saksham Shekher <95137948+OshekharO@users.noreply.github.com> Date: Mon, 6 Feb 2023 14:08:40 +0530 Subject: [PATCH 048/121] Dead: MangaKik --- .../org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt index d45d7a61..25c0c8e9 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt @@ -390,9 +390,6 @@ internal abstract class MadaraParser( @MangaSourceParser("NEATMANGA", "NeatManga", "en") class NeatManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.NEATMANGA, "neatmanga.com") - @MangaSourceParser("MANGAKIK", "MangaKik", "en") - class MangaKik(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGAKIK, "mangakik.net") - @MangaSourceParser("MANHWACLAN", "ManhwaClan", "en") class ManhwaClan(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANHWACLAN, "manhwaclan.com") From 56a54d5eb4e48a98f8d455e6e56135eec113e7f0 Mon Sep 17 00:00:00 2001 From: Saksham Shekher <95137948+OshekharO@users.noreply.github.com> Date: Mon, 6 Feb 2023 14:10:12 +0530 Subject: [PATCH 049/121] Fix: NeatManga --- .../org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt index 25c0c8e9..a52923ba 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt @@ -388,7 +388,7 @@ internal abstract class MadaraParser( class ReadManhwa(context: MangaLoaderContext) : MadaraParser(context, MangaSource.READMANHWA, "readmanhwa.info") @MangaSourceParser("NEATMANGA", "NeatManga", "en") - class NeatManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.NEATMANGA, "neatmanga.com") + class NeatManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.NEATMANGA, "neatmangas.com") @MangaSourceParser("MANHWACLAN", "ManhwaClan", "en") class ManhwaClan(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANHWACLAN, "manhwaclan.com") From 5eda6f96b43936633dd39877e65f7e39f771287e Mon Sep 17 00:00:00 2001 From: Saksham Shekher <95137948+OshekharO@users.noreply.github.com> Date: Mon, 6 Feb 2023 14:10:54 +0530 Subject: [PATCH 050/121] Dead: ReadManhwa --- .../org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt index a52923ba..4bc8a0cb 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt @@ -384,9 +384,6 @@ internal abstract class MadaraParser( @MangaSourceParser("MANGA_KOMI", "MangaKomi", "en") class MangaKomi(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGA_KOMI, "mangakomi.io") - @MangaSourceParser("READMANHWA", "ReadManhwa", "en") - class ReadManhwa(context: MangaLoaderContext) : MadaraParser(context, MangaSource.READMANHWA, "readmanhwa.info") - @MangaSourceParser("NEATMANGA", "NeatManga", "en") class NeatManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.NEATMANGA, "neatmangas.com") From 31c26f16ac60fafbed7f9549d6e8c2d28290934d Mon Sep 17 00:00:00 2001 From: Saksham Shekher <95137948+OshekharO@users.noreply.github.com> Date: Mon, 6 Feb 2023 14:13:58 +0530 Subject: [PATCH 051/121] Dead: IsekaiScan Url changed to isekaiscan.to ( chapter not found in kotatsu ) --- .../koitharu/kotatsu/parsers/site/madara/MadaraParser.kt | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt index 4bc8a0cb..4fe9789e 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt @@ -373,14 +373,6 @@ internal abstract class MadaraParser( @MangaSourceParser("FREEMANGA", "FreeManga", "en") class FreeManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.FREEMANGA, "freemanga.me") - @MangaSourceParser("ISEKAISCAN", "IsekaiScan", "en") - class IsekaiScan(context: MangaLoaderContext) : MadaraParser(context, MangaSource.ISEKAISCAN, "isekaiscan.com") { - - override fun getFaviconUrl(): String { - return "https://${getDomain()}/wp-content/uploads/2021/10/isekai-scan-02-01-150x150.png" - } - } - @MangaSourceParser("MANGA_KOMI", "MangaKomi", "en") class MangaKomi(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGA_KOMI, "mangakomi.io") From 5fe546c94d032e72d963fbb43686a8b089b7ebfe Mon Sep 17 00:00:00 2001 From: Saksham Shekher <95137948+OshekharO@users.noreply.github.com> Date: Mon, 6 Feb 2023 14:16:40 +0530 Subject: [PATCH 052/121] Dead: MangaTX OT Removed as site is not being updated --- .../org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt index 4fe9789e..57ce1faf 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt @@ -348,9 +348,6 @@ internal abstract class MadaraParser( } } - @MangaSourceParser("MANGATX_OT", "MangaTx (ot)", "en") - class MangaTxOt(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGATX_OT, "manga-tx.com") - @MangaSourceParser("MANGALEK", "MangaLek", "ar") class MangaLek(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGALEK, "mangalek.com") { From 7253699529284967394846abc4a14aeabbe6e76d Mon Sep 17 00:00:00 2001 From: Saksham Shekher <95137948+OshekharO@users.noreply.github.com> Date: Mon, 6 Feb 2023 14:18:04 +0530 Subject: [PATCH 053/121] Dead: ReadManwha --- .../org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt index 57ce1faf..540f7d49 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt @@ -329,9 +329,6 @@ internal abstract class MadaraParser( @MangaSourceParser("MANGAROSIE", "MangaRosie", "en") class MangaRosie(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGAROSIE, "mangarosie.love") - @MangaSourceParser("READMANWHA", "ReadManwha", "en") - class ReadManwha(context: MangaLoaderContext) : MadaraParser(context, MangaSource.READMANWHA, "readmanwha.net") - @MangaSourceParser("MANGATX", "MangaTx", "en") class MangaTx(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGATX, "mangatx.com") From 841d8f05d96cc8c2cfb58ae7f247049b0372e060 Mon Sep 17 00:00:00 2001 From: Saksham Shekher <95137948+OshekharO@users.noreply.github.com> Date: Mon, 6 Feb 2023 14:19:49 +0530 Subject: [PATCH 054/121] Fix: MangaRosie, PianManga --- .../org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt index 540f7d49..d8c7a3c3 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt @@ -324,10 +324,10 @@ internal abstract class MadaraParser( } @MangaSourceParser("PIANMANGA", "PianManga", "en") - class PianManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.PIANMANGA, "pianmanga.com") + class PianManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.PIANMANGA, "pianmanga.me") @MangaSourceParser("MANGAROSIE", "MangaRosie", "en") - class MangaRosie(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGAROSIE, "mangarosie.love") + class MangaRosie(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGAROSIE, "mangarosie.in") @MangaSourceParser("MANGATX", "MangaTx", "en") class MangaTx(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGATX, "mangatx.com") From 029f554d6e8aafcd9b78d861ee414616b1731d0a Mon Sep 17 00:00:00 2001 From: Saksham Shekher <95137948+OshekharO@users.noreply.github.com> Date: Mon, 6 Feb 2023 14:37:30 +0530 Subject: [PATCH 055/121] Add: DragonTranslation --- .../site/mangareader/MangaReaderParser.kt | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt index c878a56a..cca0dc9e 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt @@ -371,6 +371,30 @@ internal abstract class MangaReaderParser( } } + @MangaSourceParser("DRAGONTRANSLATION", "DragonTranslation", "es") + class DragonTranslationParser(override val context: MangaLoaderContext) : MangaReaderParser(MangaSource.DRAGONTRANSLATION, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("dragontranslation.com", null) + + override val listUrl: String get() = "/manga" + override val tableMode: Boolean get() = false + + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMMM d, yyyy", Locale.ENGLISH) + + override suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { + val infoElement = docs.selectFirst("div.infox") + return manga.copy( + chapters = chapters, + description = infoElement?.selectFirst("div.entry-content")?.html(), + author = infoElement?.selectFirst(".flex-wrap div:contains(Author)")?.lastElementSibling()?.text(), + tags = infoElement?.select(".wd-full .mgen > a") + ?.mapNotNullToSet { getOrCreateTagMap()[it.text()] } + .orEmpty(), + isNsfw = docs.selectFirst(".postbody .alr") != null, + ) + } + } + @MangaSourceParser("ASURATR", "Asura Scans (tr)", "tr") class AsuraTRParser(override val context: MangaLoaderContext) : MangaReaderParser(MangaSource.ASURATR, pageSize = 30, searchPageSize = 10) { override val configKeyDomain: ConfigKey.Domain From 80a2a10e7162b8bfd5cbf91113dae300bfbd7698 Mon Sep 17 00:00:00 2001 From: Koitharu Date: Mon, 6 Feb 2023 17:30:17 +0200 Subject: [PATCH 056/121] [Remanga] Fix headers --- .../kotlin/org/koitharu/kotatsu/parsers/site/RemangaParser.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/RemangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/RemangaParser.kt index dcdcdab5..c1a27795 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/RemangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/RemangaParser.kt @@ -233,7 +233,7 @@ internal class RemangaParser( } ?: return null val jo = JSONObject(URLDecoder.decode(userCookie.value, Charsets.UTF_8.name())) val accessToken = jo.getStringOrNull("access_token") ?: return null - return Headers.headersOf("authorization", "bearer $accessToken") + return headers.newBuilder().add("authorization", "bearer $accessToken").build() } private fun copyCookies() { From c3b1556816d889e81d95e5a21df5a4e3f907da57 Mon Sep 17 00:00:00 2001 From: Koitharu Date: Mon, 6 Feb 2023 17:50:44 +0200 Subject: [PATCH 057/121] Add headers to CloudFlareException --- .../parsers/exception/CloudFlareProtectedException.kt | 2 ++ .../org/koitharu/kotatsu/parsers/CloudFlareInterceptor.kt | 8 ++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/exception/CloudFlareProtectedException.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/exception/CloudFlareProtectedException.kt index 75337a12..e06415e3 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/exception/CloudFlareProtectedException.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/exception/CloudFlareProtectedException.kt @@ -1,7 +1,9 @@ package org.koitharu.kotatsu.parsers.exception +import okhttp3.Headers import okio.IOException class CloudFlareProtectedException( val url: String, + val headers: Headers, ) : IOException("Protected by CloudFlare: $url") \ No newline at end of file diff --git a/src/test/kotlin/org/koitharu/kotatsu/parsers/CloudFlareInterceptor.kt b/src/test/kotlin/org/koitharu/kotatsu/parsers/CloudFlareInterceptor.kt index 5f9a64c3..957eb889 100644 --- a/src/test/kotlin/org/koitharu/kotatsu/parsers/CloudFlareInterceptor.kt +++ b/src/test/kotlin/org/koitharu/kotatsu/parsers/CloudFlareInterceptor.kt @@ -13,11 +13,15 @@ private const val SERVER_CLOUDFLARE = "cloudflare" class CloudFlareInterceptor : Interceptor { override fun intercept(chain: Interceptor.Chain): Response { - val response = chain.proceed(chain.request()) + val request = chain.request() + val response = chain.proceed(request) if (response.code == HTTP_FORBIDDEN || response.code == HTTP_UNAVAILABLE) { if (response.header(HEADER_SERVER)?.startsWith(SERVER_CLOUDFLARE) == true) { response.closeQuietly() - throw CloudFlareProtectedException(response.request.url.toString()) + throw CloudFlareProtectedException( + url = response.request.url.toString(), + headers = request.headers, + ) } } return response From d14168e3251d24b0c2ca94e807ffc6d22114e0f0 Mon Sep 17 00:00:00 2001 From: Koitharu Date: Mon, 6 Feb 2023 18:06:14 +0200 Subject: [PATCH 058/121] Make headers accessible --- src/main/kotlin/org/koitharu/kotatsu/parsers/MangaParser.kt | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaParser.kt index 0c2bc8fd..3d4047cd 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaParser.kt @@ -1,7 +1,6 @@ package org.koitharu.kotatsu.parsers import androidx.annotation.CallSuper -import androidx.annotation.VisibleForTesting import okhttp3.Headers import okhttp3.HttpUrl import org.jsoup.nodes.Element @@ -35,8 +34,7 @@ abstract class MangaParser @InternalParsersApi constructor(val source: MangaSour */ protected abstract val configKeyDomain: ConfigKey.Domain - @VisibleForTesting(otherwise = VisibleForTesting.PROTECTED) - internal open val headers: Headers? = null + open val headers: Headers? = null /** * Used as fallback if value of `sortOrder` passed to [getList] is null From 00abaea32415fae2f4ae27614b13136fd7949f4b Mon Sep 17 00:00:00 2001 From: Koitharu Date: Mon, 6 Feb 2023 18:56:11 +0200 Subject: [PATCH 059/121] [Remanga] Fix headers --- .../org/koitharu/kotatsu/parsers/site/RemangaParser.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/RemangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/RemangaParser.kt index c1a27795..48f4530e 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/RemangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/RemangaParser.kt @@ -227,12 +227,12 @@ internal class RemangaParser( return jo.getJSONObject("content").getString("username") } - private fun getApiHeaders(): Headers? { + private fun getApiHeaders(): Headers { val userCookie = context.cookieJar.getCookies(getDomain()).find { it.name == "user" - } ?: return null + } ?: return headers val jo = JSONObject(URLDecoder.decode(userCookie.value, Charsets.UTF_8.name())) - val accessToken = jo.getStringOrNull("access_token") ?: return null + val accessToken = jo.getStringOrNull("access_token") ?: return headers return headers.newBuilder().add("authorization", "bearer $accessToken").build() } From 5e23d52e8abdc10907ca61529a99709bf9a6b3a4 Mon Sep 17 00:00:00 2001 From: Saksham Shekher <95137948+OshekharO@users.noreply.github.com> Date: Tue, 7 Feb 2023 10:40:03 +0530 Subject: [PATCH 060/121] refactor: HachiManga --- .../koitharu/kotatsu/parsers/site/madara/MadaraParser.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt index d8c7a3c3..58bb835c 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt @@ -316,9 +316,9 @@ internal abstract class MadaraParser( override val datePattern = "dd MMMM HH:mm" } - @MangaSourceParser("MANGAHATACHI", "MangahaTachi", "ja") - class MangahaTachi(context: MangaLoaderContext) : - MadaraParser(context, MangaSource.MANGAHATACHI, "hachimanga.com") { + @MangaSourceParser("HachiManga", "HACHIMANGA", "ja") + class HachiManga(context: MangaLoaderContext) : + MadaraParser(context, MangaSource.HACHIMANGA, "hachimanga.com") { override val sourceLocale: Locale = Locale.ENGLISH } From aae37479251192af69f8d7c58f3ea0cf7ff05d2f Mon Sep 17 00:00:00 2001 From: Saksham Shekher <95137948+OshekharO@users.noreply.github.com> Date: Wed, 8 Feb 2023 08:27:17 +0530 Subject: [PATCH 061/121] Add: KomikIndo Closes #145 --- .../parsers/site/mangareader/MangaReaderParser.kt | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt index cca0dc9e..4892f507 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt @@ -496,6 +496,19 @@ internal abstract class MangaReaderParser( override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH) } + @MangaSourceParser("KOMIKINDO", "KomikIndo", "id") + class KomikIndoParser(override val context: MangaLoaderContext) : MangaReaderParser(MangaSource.KOMIKINDO, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("komikindo.co", null) + + override val listUrl: String + get() = "/project" + override val tableMode: Boolean + get() = true + + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", idLocale) + } + @MangaSourceParser("KOMIKMANGA", "KomikManga", "id") class KomikMangaParser(override val context: MangaLoaderContext) : MangaReaderParser(MangaSource.KOMIKMANGA, pageSize = 20, searchPageSize = 10) { override val configKeyDomain: ConfigKey.Domain From 7f852e793abde6bf270263bada0a15a78f2b5bc7 Mon Sep 17 00:00:00 2001 From: Saksham Shekher <95137948+OshekharO@users.noreply.github.com> Date: Wed, 8 Feb 2023 14:16:15 +0530 Subject: [PATCH 062/121] Add: BakaMan Thai Manga Provider --- .../koitharu/kotatsu/parsers/site/madara/MadaraParser.kt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt index 58bb835c..3885663a 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt @@ -408,6 +408,12 @@ internal abstract class MadaraParser( } } + @MangaSourceParser("BAKAMAN", "BakaMan", "th") + class BakaMan(context: MangaLoaderContext) : MadaraParser(context, MangaSource.BAKAMAN, "bakaman.net") { + + override val isNsfwSource = true + } + @MangaSourceParser("MANGA_DISTRICT", "Manga District", "en") class MangaDistrict(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGA_DISTRICT, "mangadistrict.com") { From 1e1756aa1799109117b5e16bae1583243aae27c2 Mon Sep 17 00:00:00 2001 From: Koitharu Date: Wed, 8 Feb 2023 10:13:22 +0200 Subject: [PATCH 063/121] Refactor web client --- .editorconfig | 4 +- .github/workflows/test-parsers.yml | 4 +- .../kotatsu/parsers/MangaLoaderContext.kt | 167 +-- .../koitharu/kotatsu/parsers/MangaParser.kt | 313 ++--- .../kotatsu/parsers/PagedMangaParser.kt | 81 +- .../kotatsu/parsers/model/MangaPage.kt | 83 +- .../parsers/network/OkHttpWebClient.kt | 107 ++ .../kotatsu/parsers/network/WebClient.kt | 68 ++ .../kotatsu/parsers/site/AnibelParser.kt | 358 +++--- .../kotatsu/parsers/site/BatoToParser.kt | 543 ++++----- .../kotatsu/parsers/site/BlogTruyenParser.kt | 469 ++++---- .../kotatsu/parsers/site/CloneMangaParser.kt | 179 +-- .../kotatsu/parsers/site/ComickFunParser.kt | 362 +++--- .../kotatsu/parsers/site/DesuMeParser.kt | 278 ++--- .../kotatsu/parsers/site/ExHentaiParser.kt | 534 ++++----- .../kotatsu/parsers/site/MangaDexParser.kt | 470 ++++---- .../kotatsu/parsers/site/MangaInUaParser.kt | 261 ++--- .../kotatsu/parsers/site/MangaOwlParser.kt | 187 --- .../kotatsu/parsers/site/MangaTownParser.kt | 384 +++--- .../kotatsu/parsers/site/Manhwa18Parser.kt | 318 ++--- .../kotatsu/parsers/site/NHentaiParser.kt | 322 +++--- .../kotatsu/parsers/site/NetTruyenParser.kt | 412 +++---- .../parsers/site/NicovideoSeigaParser.kt | 320 ++--- .../kotatsu/parsers/site/NineMangaParser.kt | 437 ++++--- .../kotatsu/parsers/site/NudeMoonParser.kt | 428 +++---- .../kotatsu/parsers/site/RemangaParser.kt | 480 ++++---- .../kotatsu/parsers/site/UnionMangasParser.kt | 306 ++--- .../parsers/site/grouple/GroupleParser.kt | 558 +++++---- .../parsers/site/grouple/MintMangaParser.kt | 20 +- .../parsers/site/grouple/ReadmangaParser.kt | 20 +- .../parsers/site/grouple/SelfMangaParser.kt | 14 +- .../parsers/site/madara/IsekaiScanEu.kt | 62 +- .../parsers/site/madara/Madara5Parser.kt | 342 +++--- .../parsers/site/madara/Madara6Parser.kt | 100 +- .../parsers/site/madara/MadaraParser.kt | 948 +++++++-------- .../parsers/site/madara/MangalinkParser.kt | 69 +- .../site/madara/MangasOriginesParser.kt | 71 +- .../parsers/site/madara/PrismaScansParser.kt | 75 +- .../parsers/site/madara/ReaperScansParser.kt | 140 +-- .../parsers/site/madara/TatakaeScansParser.kt | 82 +- .../site/mangareader/MangaReaderParser.kt | 1030 +++++++++-------- .../parsers/site/multichan/ChanParser.kt | 316 ++--- .../parsers/site/multichan/HenChanParser.kt | 96 +- .../parsers/site/multichan/MangaChanParser.kt | 6 +- .../parsers/site/multichan/YaoiChanParser.kt | 52 +- .../parsers/site/rulib/MangaLibParser.kt | 530 ++++----- .../kotatsu/parsers/util/FaviconParser.kt | 152 ++- .../kotatsu/parsers/util/MangaParserEnv.kt | 78 ++ .../koitharu/kotatsu/parsers/util/OkHttp.kt | 20 +- .../kotatsu/parsers/AuthCheckExtension.kt | 44 +- .../kotatsu/parsers/CloudFlareInterceptor.kt | 35 +- .../parsers/CommonHeadersInterceptor.kt | 47 + .../kotatsu/parsers/MangaLoaderContextMock.kt | 96 +- .../kotatsu/parsers/MangaParserTest.kt | 361 +++--- .../koitharu/kotatsu/parsers/MangaSources.kt | 2 +- .../kotatsu/parsers/UserAgentInterceptor.kt | 20 - 56 files changed, 6600 insertions(+), 6661 deletions(-) create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/network/OkHttpWebClient.kt create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/network/WebClient.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaOwlParser.kt create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/util/MangaParserEnv.kt create mode 100644 src/test/kotlin/org/koitharu/kotatsu/parsers/CommonHeadersInterceptor.kt delete mode 100644 src/test/kotlin/org/koitharu/kotatsu/parsers/UserAgentInterceptor.kt diff --git a/.editorconfig b/.editorconfig index 7f2b16d0..a352878e 100644 --- a/.editorconfig +++ b/.editorconfig @@ -6,8 +6,8 @@ end_of_line = lf indent_style = tab max_line_length = 120 tab_width = 4 -insert_final_newline = false -disabled_rules=no-wildcard-imports,no-unused-imports +insert_final_newline = true +disabled_rules = no-wildcard-imports, no-unused-imports [{*.kt,*.kts}] ij_kotlin_allow_trailing_comma = true diff --git a/.github/workflows/test-parsers.yml b/.github/workflows/test-parsers.yml index 4479e758..817a3941 100644 --- a/.github/workflows/test-parsers.yml +++ b/.github/workflows/test-parsers.yml @@ -4,7 +4,7 @@ on: workflow_dispatch: pull_request: paths: - - 'src/main/kotlin/org/koitharu/kotatsu/parsers/site/*' + - 'src/main/kotlin/org/koitharu/kotatsu/parsers/site/**' permissions: contents: read @@ -24,4 +24,4 @@ jobs: - uses: actions/upload-artifact@v3 with: name: Report - path: build/test-results-html/TEST-org.koitharu.kotatsu.parsers.MangaParserTest.htm \ No newline at end of file + path: build/test-results-html/TEST-org.koitharu.kotatsu.parsers.MangaParserTest.htm diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaLoaderContext.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaLoaderContext.kt index 04753783..1268be96 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaLoaderContext.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaLoaderContext.kt @@ -1,166 +1,29 @@ package org.koitharu.kotatsu.parsers -import okhttp3.* -import okhttp3.HttpUrl.Companion.toHttpUrl -import okhttp3.MediaType.Companion.toMediaType -import okhttp3.RequestBody.Companion.toRequestBody -import org.json.JSONObject -import org.jsoup.HttpStatusException +import okhttp3.CookieJar +import okhttp3.OkHttpClient import org.koitharu.kotatsu.parsers.config.MangaSourceConfig -import org.koitharu.kotatsu.parsers.exception.GraphQLException -import org.koitharu.kotatsu.parsers.exception.NotFoundException import org.koitharu.kotatsu.parsers.model.MangaSource -import org.koitharu.kotatsu.parsers.util.await -import org.koitharu.kotatsu.parsers.util.parseJson import java.util.* abstract class MangaLoaderContext { - protected abstract val httpClient: OkHttpClient + abstract val httpClient: OkHttpClient - abstract val cookieJar: CookieJar + abstract val cookieJar: CookieJar - /** - * Do a GET http request to specific url - * @param url - * @param headers an additional headers for request, may be null - */ - suspend fun httpGet(url: HttpUrl, headers: Headers? = null): Response { - val request = Request.Builder() - .get() - .url(url) - if (headers != null) { - request.headers(headers) - } - return httpClient.newCall(request.build()).await().ensureSuccess() - } + open fun encodeBase64(data: ByteArray): String = Base64.getEncoder().encodeToString(data) - suspend fun httpGet(url: String, headers: Headers? = null): Response { - return httpGet(url.toHttpUrl(), headers) - } + open fun decodeBase64(data: String): ByteArray = Base64.getDecoder().decode(data) + open fun getPreferredLocales(): List = listOf(Locale.getDefault()) - /** - * Do a HEAD http request to specific url - * @param url - * @param headers an additional headers for request, may be null - */ - suspend fun httpHead(url: String, headers: Headers? = null): Response { - val request = Request.Builder() - .head() - .url(url) - if (headers != null) { - request.headers(headers) - } - return httpClient.newCall(request.build()).await().ensureSuccess() - } + /** + * Execute JavaScript code and return result + * @param script JavaScript source code + * @return execution result as string, may be null + */ + abstract suspend fun evaluateJs(script: String): String? - /** - * Do a POST http request to specific url with `multipart/form-data` payload - * @param url - * @param form payload as key=>value map - * @param headers an additional headers for request, may be null - */ - suspend fun httpPost( - url: String, - form: Map, - headers: Headers? = null, - ): Response { - val body = FormBody.Builder() - form.forEach { (k, v) -> - body.addEncoded(k, v) - } - val request = Request.Builder() - .post(body.build()) - .url(url) - if (headers != null) { - request.headers(headers) - } - return httpClient.newCall(request.build()).await().ensureSuccess() - } - - /** - * Do a POST http request to specific url with `multipart/form-data` payload - * @param url - * @param payload payload as `key=value` string with `&` separator - * @param headers an additional headers for request, may be null - */ - suspend fun httpPost( - url: String, - payload: String, - headers: Headers?, - ): Response { - val body = FormBody.Builder() - payload.split('&').forEach { - val pos = it.indexOf('=') - if (pos != -1) { - val k = it.substring(0, pos) - val v = it.substring(pos + 1) - body.addEncoded(k, v) - } - } - val request = Request.Builder() - .post(body.build()) - .url(url) - if (headers != null) { - request.headers(headers) - } - return httpClient.newCall(request.build()).await().ensureSuccess() - } - - /** - * Do a GraphQL request to specific url - * @param endpoint an url - * @param query GraphQL request payload - */ - suspend fun graphQLQuery(endpoint: String, query: String): JSONObject { - val body = JSONObject() - body.put("operationName", null as Any?) - body.put("variables", JSONObject()) - body.put("query", "{$query}") - val mediaType = "application/json; charset=utf-8".toMediaType() - val requestBody = body.toString().toRequestBody(mediaType) - val request = Request.Builder() - .post(requestBody) - .url(endpoint) - val json = httpClient.newCall(request.build()).await().ensureSuccess().parseJson() - json.optJSONArray("errors")?.let { - if (it.length() != 0) { - throw GraphQLException(it) - } - } - return json - } - - open fun encodeBase64(data: ByteArray): String = Base64.getEncoder().encodeToString(data) - - open fun decodeBase64(data: String): ByteArray = Base64.getDecoder().decode(data) - - open fun getPreferredLocales(): List = listOf(Locale.getDefault()) - - /** - * Execute JavaScript code and return result - * @param script JavaScript source code - * @return execution result as string, may be null - */ - abstract suspend fun evaluateJs(script: String): String? - - abstract fun getConfig(source: MangaSource): MangaSourceConfig - - private fun Response.ensureSuccess(): Response { - val exception: Exception? = when (code) { // Catch some error codes, not all - 404 -> NotFoundException(message, request.url.toString()) - in 500..599 -> HttpStatusException(message, code, request.url.toString()) - else -> null - } - if (exception != null) { - runCatching { - close() - }.onFailure { - exception.addSuppressed(it) - } - throw exception - } - return this - } -} \ No newline at end of file + abstract fun getConfig(source: MangaSource): MangaSourceConfig +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaParser.kt index 3d4047cd..435976f1 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaParser.kt @@ -2,197 +2,134 @@ package org.koitharu.kotatsu.parsers import androidx.annotation.CallSuper import okhttp3.Headers -import okhttp3.HttpUrl -import org.jsoup.nodes.Element import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.exception.ParseException import org.koitharu.kotatsu.parsers.model.* +import org.koitharu.kotatsu.parsers.network.OkHttpWebClient +import org.koitharu.kotatsu.parsers.network.WebClient import org.koitharu.kotatsu.parsers.util.FaviconParser +import org.koitharu.kotatsu.parsers.util.domain import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl import java.util.* -abstract class MangaParser @InternalParsersApi constructor(val source: MangaSource) { - - protected abstract val context: MangaLoaderContext - - /** - * Supported [SortOrder] variants. Must not be empty. - * - * For better performance use [EnumSet] for more than one item. - */ - abstract val sortOrders: Set - - val config by lazy { context.getConfig(source) } - - open val sourceLocale: Locale? - get() = source.locale?.let { Locale(it) } - - /** - * Provide default domain and available alternatives, if any. - * - * Never hardcode domain in requests, use [getDomain] instead. - */ - protected abstract val configKeyDomain: ConfigKey.Domain - - open val headers: Headers? = null - - /** - * Used as fallback if value of `sortOrder` passed to [getList] is null - */ - protected open val defaultSortOrder: SortOrder - get() { - val supported = sortOrders - return SortOrder.values().first { it in supported } - } - - /** - * Parse list of manga by specified criteria - * - * @param offset starting from 0 and used for pagination. - * Note than passed value may not be divisible by internal page size, so you should adjust it manually. - * @param query search query, may be null or empty if no search needed - * @param tags genres for filtering, values from [getTags] and [Manga.tags]. May be null or empty - * @param sortOrder one of [sortOrders] or null for default value - */ - @JvmSynthetic - @InternalParsersApi - abstract suspend fun getList( - offset: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List - - /** - * Parse list of manga with search by text query - * - * @param offset starting from 0 and used for pagination. - * @param query search query - */ - open suspend fun getList(offset: Int, query: String): List { - return getList(offset, query, null, defaultSortOrder) - } - - /** - * Parse list of manga by specified criteria - * - * @param offset starting from 0 and used for pagination. - * Note than passed value may not be divisible by internal page size, so you should adjust it manually. - * @param tags genres for filtering, values from [getTags] and [Manga.tags]. May be null or empty - * @param sortOrder one of [sortOrders] or null for default value - */ - open suspend fun getList(offset: Int, tags: Set?, sortOrder: SortOrder?): List { - return getList(offset, null, tags, sortOrder ?: defaultSortOrder) - } - - /** - * Parse details for [Manga]: chapters list, description, large cover, etc. - * Must return the same manga, may change any fields excepts id, url and source - * @see Manga.copy - */ - abstract suspend fun getDetails(manga: Manga): Manga - - /** - * Parse pages list for specified chapter. - * @see MangaPage for details - */ - abstract suspend fun getPages(chapter: MangaChapter): List - - /** - * Fetch direct link to the page image. - */ - open suspend fun getPageUrl(page: MangaPage): String = page.url.toAbsoluteUrl(getDomain()) - - /** - * Fetch available tags (genres) for source - */ - abstract suspend fun getTags(): Set - - /** - * Returns direct link to the website favicon - */ - @Deprecated( - message = "Use parseFavicons() to get multiple favicons with different size", - replaceWith = ReplaceWith("parseFavicons()"), - ) - open fun getFaviconUrl() = "https://${getDomain()}/favicon.ico" - - /** - * Parse favicons from the main page of the source`s website - */ - open suspend fun getFavicons(): Favicons { - return FaviconParser(context, getDomain(), headers).parseFavicons() - } - - @CallSuper - open fun onCreateConfig(keys: MutableCollection>) { - keys.add(configKeyDomain) - } - - /* Utils */ - - fun getDomain(): String { - return config[configKeyDomain] - } - - fun getDomain(subdomain: String): String { - val domain = getDomain() - return subdomain + "." + domain.removePrefix("www.") - } - - fun urlBuilder(): HttpUrl.Builder { - return HttpUrl.Builder() - .scheme("https") - .host(getDomain()) - } - - /** - * Create a unique id for [Manga]/[MangaChapter]/[MangaPage]. - * @param url must be relative url, without a domain - * @see [Manga.id] - * @see [MangaChapter.id] - * @see [MangaPage.id] - */ - @InternalParsersApi - protected fun generateUid(url: String): Long { - var h = 1125899906842597L - source.name.forEach { c -> - h = 31 * h + c.code - } - url.forEach { c -> - h = 31 * h + c.code - } - return h - } - - /** - * Create a unique id for [Manga]/[MangaChapter]/[MangaPage]. - * @param id an internal identifier - * @see [Manga.id] - * @see [MangaChapter.id] - * @see [MangaPage.id] - */ - @InternalParsersApi - protected fun generateUid(id: Long): Long { - var h = 1125899906842597L - source.name.forEach { c -> - h = 31 * h + c.code - } - h = 31 * h + id - return h - } - - @InternalParsersApi - protected fun Element.parseFailed(message: String? = null): Nothing { - throw ParseException(message, ownerDocument()?.location() ?: baseUri(), null) - } - - @InternalParsersApi - protected fun Set?.oneOrThrowIfMany(): MangaTag? { - return when { - isNullOrEmpty() -> null - size == 1 -> first() - else -> throw IllegalArgumentException("Multiple genres are not supported by this source") - } - } -} \ No newline at end of file +abstract class MangaParser @InternalParsersApi constructor( + @property:InternalParsersApi val context: MangaLoaderContext, + val source: MangaSource, +) { + + /** + * Supported [SortOrder] variants. Must not be empty. + * + * For better performance use [EnumSet] for more than one item. + */ + abstract val sortOrders: Set + + val config by lazy { context.getConfig(source) } + + open val sourceLocale: Locale? + get() = source.locale?.let { Locale(it) } + + /** + * Provide default domain and available alternatives, if any. + * + * Never hardcode domain in requests, use [getDomain] instead. + */ + @InternalParsersApi + abstract val configKeyDomain: ConfigKey.Domain + + open val headers: Headers? = null + + /** + * Used as fallback if value of `sortOrder` passed to [getList] is null + */ + protected open val defaultSortOrder: SortOrder + get() { + val supported = sortOrders + return SortOrder.values().first { it in supported } + } + + protected val webClient: WebClient = OkHttpWebClient(context.httpClient, source) + + /** + * Parse list of manga by specified criteria + * + * @param offset starting from 0 and used for pagination. + * Note than passed value may not be divisible by internal page size, so you should adjust it manually. + * @param query search query, may be null or empty if no search needed + * @param tags genres for filtering, values from [getTags] and [Manga.tags]. May be null or empty + * @param sortOrder one of [sortOrders] or null for default value + */ + @JvmSynthetic + @InternalParsersApi + abstract suspend fun getList( + offset: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List + + /** + * Parse list of manga with search by text query + * + * @param offset starting from 0 and used for pagination. + * @param query search query + */ + open suspend fun getList(offset: Int, query: String): List { + return getList(offset, query, null, defaultSortOrder) + } + + /** + * Parse list of manga by specified criteria + * + * @param offset starting from 0 and used for pagination. + * Note than passed value may not be divisible by internal page size, so you should adjust it manually. + * @param tags genres for filtering, values from [getTags] and [Manga.tags]. May be null or empty + * @param sortOrder one of [sortOrders] or null for default value + */ + open suspend fun getList(offset: Int, tags: Set?, sortOrder: SortOrder?): List { + return getList(offset, null, tags, sortOrder ?: defaultSortOrder) + } + + /** + * Parse details for [Manga]: chapters list, description, large cover, etc. + * Must return the same manga, may change any fields excepts id, url and source + * @see Manga.copy + */ + abstract suspend fun getDetails(manga: Manga): Manga + + /** + * Parse pages list for specified chapter. + * @see MangaPage for details + */ + abstract suspend fun getPages(chapter: MangaChapter): List + + /** + * Fetch direct link to the page image. + */ + open suspend fun getPageUrl(page: MangaPage): String = page.url.toAbsoluteUrl(domain) + + /** + * Fetch available tags (genres) for source + */ + abstract suspend fun getTags(): Set + + /** + * Returns direct link to the website favicon + */ + @Deprecated( + message = "Use parseFavicons() to get multiple favicons with different size", + replaceWith = ReplaceWith("parseFavicons()"), + ) + open fun getFaviconUrl() = "https://${domain}/favicon.ico" + + /** + * Parse favicons from the main page of the source`s website + */ + open suspend fun getFavicons(): Favicons { + return FaviconParser(webClient, domain).parseFavicons() + } + + @CallSuper + open fun onCreateConfig(keys: MutableCollection>) { + keys.add(configKeyDomain) + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/PagedMangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/PagedMangaParser.kt index 7040bce4..ea18f848 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/PagedMangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/PagedMangaParser.kt @@ -8,43 +8,44 @@ import org.koitharu.kotatsu.parsers.util.Paginator @InternalParsersApi abstract class PagedMangaParser( - source: MangaSource, - pageSize: Int, - searchPageSize: Int = pageSize, -) : MangaParser(source) { - - protected val paginator = Paginator(pageSize) - protected val searchPaginator = Paginator(searchPageSize) - - override suspend fun getList(offset: Int, query: String): List { - return getList(searchPaginator, offset, query, null, defaultSortOrder) - } - - override suspend fun getList(offset: Int, tags: Set?, sortOrder: SortOrder?): List { - return getList(paginator, offset, null, tags, sortOrder ?: defaultSortOrder) - } - - @InternalParsersApi - @Deprecated("You should use getListPage for PagedMangaParser", level = DeprecationLevel.HIDDEN) - final override suspend fun getList( - offset: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List = throw UnsupportedOperationException("You should use getListPage for PagedMangaParser") - - abstract suspend fun getListPage(page: Int, query: String?, tags: Set?, sortOrder: SortOrder): List - - private suspend fun getList( - paginator: Paginator, - offset: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - val page = paginator.getPage(offset) - val list = getListPage(page, query, tags, sortOrder) - paginator.onListReceived(offset, page, list.size) - return list - } -} \ No newline at end of file + context: MangaLoaderContext, + source: MangaSource, + pageSize: Int, + searchPageSize: Int = pageSize, +) : MangaParser(context, source) { + + protected val paginator = Paginator(pageSize) + protected val searchPaginator = Paginator(searchPageSize) + + override suspend fun getList(offset: Int, query: String): List { + return getList(searchPaginator, offset, query, null, defaultSortOrder) + } + + override suspend fun getList(offset: Int, tags: Set?, sortOrder: SortOrder?): List { + return getList(paginator, offset, null, tags, sortOrder ?: defaultSortOrder) + } + + @InternalParsersApi + @Deprecated("You should use getListPage for PagedMangaParser", level = DeprecationLevel.HIDDEN) + final override suspend fun getList( + offset: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List = throw UnsupportedOperationException("You should use getListPage for PagedMangaParser") + + abstract suspend fun getListPage(page: Int, query: String?, tags: Set?, sortOrder: SortOrder): List + + private suspend fun getList( + paginator: Paginator, + offset: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + val page = paginator.getPage(offset) + val list = getListPage(page, query, tags, sortOrder) + paginator.onListReceived(offset, page, list.size) + return list + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/model/MangaPage.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/model/MangaPage.kt index 4cc66d29..0c5102a0 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/model/MangaPage.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/model/MangaPage.kt @@ -3,50 +3,49 @@ package org.koitharu.kotatsu.parsers.model import org.koitharu.kotatsu.parsers.MangaParser class MangaPage( - /** - * Unique identifier for manga - */ - val id: Long, - /** - * Relative url to page (**without** a domain) or any other uri. - * Used principally in parsers. - * May contain link to image or html page. - * @see MangaParser.getPageUrl - */ - val url: String, - /** - * Absolute link to the chapter or website home page. - * Used in Referer header - */ - val referer: String, - /** - * Absolute url of the small page image if exists, null otherwise - */ - val preview: String?, - val source: MangaSource, + /** + * Unique identifier for manga + */ + val id: Long, + /** + * Relative url to page (**without** a domain) or any other uri. + * Used principally in parsers. + * May contain link to image or html page. + * @see MangaParser.getPageUrl + */ + val url: String, + /** + * Absolute link to the chapter or website home page. + * Used in Referer header + */ + @Deprecated("Referer header should be added based on source domain") + val referer: String, + /** + * Absolute url of the small page image if exists, null otherwise + */ + val preview: String?, + val source: MangaSource, ) { - override fun equals(other: Any?): Boolean { - if (this === other) return true - if (javaClass != other?.javaClass) return false + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (javaClass != other?.javaClass) return false - other as MangaPage + other as MangaPage - if (id != other.id) return false - if (url != other.url) return false - if (referer != other.referer) return false - if (preview != other.preview) return false - if (source != other.source) return false + if (id != other.id) return false + if (url != other.url) return false + if (referer != other.referer) return false + if (preview != other.preview) return false + return source == other.source + } - return true - } - - override fun hashCode(): Int { - var result = id.hashCode() - result = 31 * result + url.hashCode() - result = 31 * result + referer.hashCode() - result = 31 * result + (preview?.hashCode() ?: 0) - result = 31 * result + source.hashCode() - return result - } -} \ No newline at end of file + override fun hashCode(): Int { + var result = id.hashCode() + result = 31 * result + url.hashCode() + result = 31 * result + referer.hashCode() + result = 31 * result + (preview?.hashCode() ?: 0) + result = 31 * result + source.hashCode() + return result + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/network/OkHttpWebClient.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/network/OkHttpWebClient.kt new file mode 100644 index 00000000..7d61ba1e --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/network/OkHttpWebClient.kt @@ -0,0 +1,107 @@ +package org.koitharu.kotatsu.parsers.network + +import okhttp3.* +import okhttp3.MediaType.Companion.toMediaType +import okhttp3.RequestBody.Companion.toRequestBody +import org.json.JSONObject +import org.jsoup.HttpStatusException +import org.koitharu.kotatsu.parsers.exception.GraphQLException +import org.koitharu.kotatsu.parsers.exception.NotFoundException +import org.koitharu.kotatsu.parsers.model.MangaSource +import org.koitharu.kotatsu.parsers.util.await +import org.koitharu.kotatsu.parsers.util.parseJson + +class OkHttpWebClient( + private val httpClient: OkHttpClient, + private val mangaSource: MangaSource, +) : WebClient { + + override suspend fun httpGet(url: HttpUrl): Response { + val request = Request.Builder() + .get() + .url(url) + .addTags() + return httpClient.newCall(request.build()).await().ensureSuccess() + } + + override suspend fun httpHead(url: HttpUrl): Response { + val request = Request.Builder() + .head() + .url(url) + .addTags() + return httpClient.newCall(request.build()).await().ensureSuccess() + } + + override suspend fun httpPost(url: HttpUrl, form: Map): Response { + val body = FormBody.Builder() + form.forEach { (k, v) -> + body.addEncoded(k, v) + } + val request = Request.Builder() + .post(body.build()) + .url(url) + .addTags() + return httpClient.newCall(request.build()).await().ensureSuccess() + } + + override suspend fun httpPost(url: HttpUrl, payload: String): Response { + val body = FormBody.Builder() + payload.split('&').forEach { + val pos = it.indexOf('=') + if (pos != -1) { + val k = it.substring(0, pos) + val v = it.substring(pos + 1) + body.addEncoded(k, v) + } + } + val request = Request.Builder() + .post(body.build()) + .url(url) + .addTags() + return httpClient.newCall(request.build()).await().ensureSuccess() + } + + override suspend fun graphQLQuery(endpoint: String, query: String): JSONObject { + val body = JSONObject() + body.put("operationName", null as Any?) + body.put("variables", JSONObject()) + body.put("query", "{$query}") + val mediaType = "application/json; charset=utf-8".toMediaType() + val requestBody = body.toString().toRequestBody(mediaType) + val request = Request.Builder() + .post(requestBody) + .url(endpoint) + .addTags() + val json = httpClient.newCall(request.build()).await().ensureSuccess().parseJson() + json.optJSONArray("errors")?.let { + if (it.length() != 0) { + throw GraphQLException(it) + } + } + return json + } + + private fun Request.Builder.addTags(): Request.Builder { + if (mangaSource != null) { + tag(MangaSource::class.java, mangaSource) + } + return this + } + + private fun Response.ensureSuccess(): Response { + val exception: Exception? = when (code) { // Catch some error codes, not all + 404 -> NotFoundException(message, request.url.toString()) + in 500..599 -> HttpStatusException(message, code, request.url.toString()) + else -> null + } + if (exception != null) { + runCatching { + close() + }.onFailure { + exception.addSuppressed(it) + } + throw exception + } + return this + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/network/WebClient.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/network/WebClient.kt new file mode 100644 index 00000000..93770c4f --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/network/WebClient.kt @@ -0,0 +1,68 @@ +package org.koitharu.kotatsu.parsers.network + +import okhttp3.HttpUrl +import okhttp3.HttpUrl.Companion.toHttpUrl +import okhttp3.Response +import org.json.JSONObject + +interface WebClient { + + /** + * Do a GET http request to specific url + * @param url + */ + suspend fun httpGet(url: String): Response = httpGet(url.toHttpUrl()) + + /** + * Do a GET http request to specific url + * @param url + */ + suspend fun httpGet(url: HttpUrl): Response + + /** + * Do a HEAD http request to specific url + * @param url + */ + suspend fun httpHead(url: String): Response = httpHead(url.toHttpUrl()) + + /** + * Do a HEAD http request to specific url + * @param url + */ + suspend fun httpHead(url: HttpUrl): Response + + /** + * Do a POST http request to specific url with `multipart/form-data` payload + * @param url + * @param form payload as key=>value map + */ + suspend fun httpPost(url: String, form: Map): Response = httpPost(url.toHttpUrl(), form) + + /** + * Do a POST http request to specific url with `multipart/form-data` payload + * @param url + * @param form payload as key=>value map + */ + suspend fun httpPost(url: HttpUrl, form: Map): Response + + /** + * Do a POST http request to specific url with `multipart/form-data` payload + * @param url + * @param payload payload as `key=value` string with `&` separator + */ + suspend fun httpPost(url: String, payload: String): Response = httpPost(url.toHttpUrl(), payload) + + /** + * Do a POST http request to specific url with `multipart/form-data` payload + * @param url + * @param payload payload as `key=value` string with `&` separator + */ + suspend fun httpPost(url: HttpUrl, payload: String): Response + + /** + * Do a GraphQL request to specific url + * @param endpoint an url + * @param query GraphQL request payload + */ + suspend fun graphQLQuery(endpoint: String, query: String): JSONObject +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/AnibelParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/AnibelParser.kt index 8436ee1d..a01f96be 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/AnibelParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/AnibelParser.kt @@ -8,6 +8,9 @@ import org.koitharu.kotatsu.parsers.MangaParser import org.koitharu.kotatsu.parsers.MangaSourceParser import org.koitharu.kotatsu.parsers.config.ConfigKey import org.koitharu.kotatsu.parsers.model.* +import org.koitharu.kotatsu.parsers.util.domain +import org.koitharu.kotatsu.parsers.util.generateUid +import org.koitharu.kotatsu.parsers.util.getDomain import org.koitharu.kotatsu.parsers.util.json.mapJSON import org.koitharu.kotatsu.parsers.util.json.mapJSONIndexed import org.koitharu.kotatsu.parsers.util.json.stringIterator @@ -15,38 +18,38 @@ import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl import java.util.* @MangaSourceParser("ANIBEL", "Anibel", "be") -internal class AnibelParser(override val context: MangaLoaderContext) : MangaParser(MangaSource.ANIBEL) { +internal class AnibelParser(context: MangaLoaderContext) : MangaParser(context, MangaSource.ANIBEL) { - override val configKeyDomain = ConfigKey.Domain("anibel.net", null) + override val configKeyDomain = ConfigKey.Domain("anibel.net", null) - override val sortOrders: Set = EnumSet.of( - SortOrder.NEWEST, - ) + override val sortOrders: Set = EnumSet.of( + SortOrder.NEWEST, + ) - override fun getFaviconUrl(): String { - return "https://cdn.${getDomain()}/favicons/favicon.png" - } + override fun getFaviconUrl(): String { + return "https://cdn.${domain}/favicons/favicon.png" + } - override suspend fun getList( - offset: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - if (!query.isNullOrEmpty()) { - return if (offset == 0) { - search(query) - } else { - emptyList() - } - } - val filters = tags?.takeUnless { it.isEmpty() }?.joinToString( - separator = ",", - prefix = "genres: [", - postfix = "]", - ) { "\"${it.key}\"" }.orEmpty() - val array = apiCall( - """ + override suspend fun getList( + offset: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + if (!query.isNullOrEmpty()) { + return if (offset == 0) { + search(query) + } else { + emptyList() + } + } + val filters = tags?.takeUnless { it.isEmpty() }?.joinToString( + separator = ",", + prefix = "genres: [", + postfix = "]", + ) { "\"${it.key}\"" }.orEmpty() + val array = apiCall( + """ getMediaList(offset: $offset, limit: 20, mediaType: manga, filters: {$filters}) { docs { mediaId @@ -63,37 +66,37 @@ internal class AnibelParser(override val context: MangaLoaderContext) : MangaPar } } """.trimIndent(), - ).getJSONObject("getMediaList").getJSONArray("docs") - return array.mapJSON { jo -> - val mediaId = jo.getString("mediaId") - val title = jo.getJSONObject("title") - val href = "${jo.getString("mediaType")}/${jo.getString("slug")}" - Manga( - id = generateUid(mediaId), - title = title.getString("be"), - coverUrl = jo.getString("poster").removePrefix("/cdn") - .toAbsoluteUrl(getDomain("cdn")) + "?width=200&height=280", - altTitle = title.getString("alt").takeUnless(String::isEmpty), - author = null, - isNsfw = false, - rating = jo.getDouble("rating").toFloat() / 10f, - url = href, - publicUrl = "https://${getDomain()}/$href", - tags = jo.getJSONArray("genres").mapToTags(), - state = when (jo.getString("status")) { - "ongoing" -> MangaState.ONGOING - "finished" -> MangaState.FINISHED - else -> null - }, - source = source, - ) - } - } + ).getJSONObject("getMediaList").getJSONArray("docs") + return array.mapJSON { jo -> + val mediaId = jo.getString("mediaId") + val title = jo.getJSONObject("title") + val href = "${jo.getString("mediaType")}/${jo.getString("slug")}" + Manga( + id = generateUid(mediaId), + title = title.getString("be"), + coverUrl = jo.getString("poster").removePrefix("/cdn") + .toAbsoluteUrl(getDomain("cdn")) + "?width=200&height=280", + altTitle = title.getString("alt").takeUnless(String::isEmpty), + author = null, + isNsfw = false, + rating = jo.getDouble("rating").toFloat() / 10f, + url = href, + publicUrl = "https://${domain}/$href", + tags = jo.getJSONArray("genres").mapToTags(), + state = when (jo.getString("status")) { + "ongoing" -> MangaState.ONGOING + "finished" -> MangaState.FINISHED + else -> null + }, + source = source, + ) + } + } - override suspend fun getDetails(manga: Manga): Manga { - val (type, slug) = manga.url.split('/') - val details = apiCall( - """ + override suspend fun getDetails(manga: Manga): Manga { + val (type, slug) = manga.url.split('/') + val details = apiCall( + """ media(mediaType: $type, slug: "$slug") { mediaId title { @@ -109,52 +112,52 @@ internal class AnibelParser(override val context: MangaLoaderContext) : MangaPar genres } """.trimIndent(), - ).getJSONObject("media") - val title = details.getJSONObject("title") - val poster = details.getString("poster").removePrefix("/cdn") - .toAbsoluteUrl(getDomain("cdn")) - val chapters = apiCall( - """ + ).getJSONObject("media") + val title = details.getJSONObject("title") + val poster = details.getString("poster").removePrefix("/cdn") + .toAbsoluteUrl(getDomain("cdn")) + val chapters = apiCall( + """ chapters(mediaId: "${details.getString("mediaId")}") { id chapter released } """.trimIndent(), - ).getJSONArray("chapters") - return manga.copy( - title = title.getString("be"), - altTitle = title.getString("alt"), - coverUrl = "$poster?width=200&height=280", - largeCoverUrl = poster, - description = details.getJSONObject("description").getString("be"), - rating = details.getDouble("rating").toFloat() / 10f, - tags = details.getJSONArray("genres").mapToTags(), - state = when (details.getString("status")) { - "ongoing" -> MangaState.ONGOING - "finished" -> MangaState.FINISHED - else -> null - }, - chapters = chapters.mapJSON { jo -> - val number = jo.getInt("chapter") - MangaChapter( - id = generateUid(jo.getString("id")), - name = "Глава $number", - number = number, - url = "${manga.url}/read/$number", - scanlator = null, - uploadDate = jo.getLong("released"), - branch = null, - source = source, - ) - }, - ) - } + ).getJSONArray("chapters") + return manga.copy( + title = title.getString("be"), + altTitle = title.getString("alt"), + coverUrl = "$poster?width=200&height=280", + largeCoverUrl = poster, + description = details.getJSONObject("description").getString("be"), + rating = details.getDouble("rating").toFloat() / 10f, + tags = details.getJSONArray("genres").mapToTags(), + state = when (details.getString("status")) { + "ongoing" -> MangaState.ONGOING + "finished" -> MangaState.FINISHED + else -> null + }, + chapters = chapters.mapJSON { jo -> + val number = jo.getInt("chapter") + MangaChapter( + id = generateUid(jo.getString("id")), + name = "Глава $number", + number = number, + url = "${manga.url}/read/$number", + scanlator = null, + uploadDate = jo.getLong("released"), + branch = null, + source = source, + ) + }, + ) + } - override suspend fun getPages(chapter: MangaChapter): List { - val (_, slug, _, number) = chapter.url.split('/') - val chapterJson = apiCall( - """ + override suspend fun getPages(chapter: MangaChapter): List { + val (_, slug, _, number) = chapter.url.split('/') + val chapterJson = apiCall( + """ chapter(slug: "$slug", chapter: $number) { id images { @@ -163,35 +166,35 @@ internal class AnibelParser(override val context: MangaLoaderContext) : MangaPar } } """.trimIndent(), - ).getJSONObject("chapter") - val pages = chapterJson.getJSONArray("images") - val chapterUrl = "https://${getDomain()}/${chapter.url}" - return pages.mapJSONIndexed { i, jo -> - MangaPage( - id = generateUid("${chapter.url}/$i"), - url = jo.getString("large"), - referer = chapterUrl, - preview = jo.getString("thumbnail"), - source = source, - ) - } - } + ).getJSONObject("chapter") + val pages = chapterJson.getJSONArray("images") + val chapterUrl = "https://${domain}/${chapter.url}" + return pages.mapJSONIndexed { i, jo -> + MangaPage( + id = generateUid("${chapter.url}/$i"), + url = jo.getString("large"), + referer = chapterUrl, + preview = jo.getString("thumbnail"), + source = source, + ) + } + } - override suspend fun getTags(): Set { - val json = apiCall( - """ + override suspend fun getTags(): Set { + val json = apiCall( + """ getFilters(mediaType: manga) { genres } """.trimIndent(), - ) - val array = json.getJSONObject("getFilters").getJSONArray("genres") - return array.mapToTags() - } + ) + val array = json.getJSONObject("getFilters").getJSONArray("genres") + return array.mapToTags() + } - private suspend fun search(query: String): List { - val json = apiCall( - """ + private suspend fun search(query: String): List { + val json = apiCall( + """ search(query: "$query", limit: 40) { id title { @@ -203,64 +206,65 @@ internal class AnibelParser(override val context: MangaLoaderContext) : MangaPar type } """.trimIndent(), - ) - val array = json.getJSONArray("search") - return array.mapJSON { jo -> - val mediaId = jo.getString("id") - val title = jo.getJSONObject("title") - val href = "${jo.getString("type").lowercase()}/${jo.getString("url")}" - Manga( - id = generateUid(mediaId), - title = title.getString("be"), - coverUrl = jo.getString("poster").removePrefix("/cdn") - .toAbsoluteUrl(getDomain("cdn")) + "?width=200&height=280", - altTitle = title.getString("en").takeUnless(String::isEmpty), - author = null, - isNsfw = false, - rating = RATING_UNKNOWN, - url = href, - publicUrl = "https://${getDomain()}/$href", - tags = emptySet(), - state = null, - source = source, - ) - } - } + ) + val array = json.getJSONArray("search") + return array.mapJSON { jo -> + val mediaId = jo.getString("id") + val title = jo.getJSONObject("title") + val href = "${jo.getString("type").lowercase()}/${jo.getString("url")}" + Manga( + id = generateUid(mediaId), + title = title.getString("be"), + coverUrl = jo.getString("poster").removePrefix("/cdn") + .toAbsoluteUrl(getDomain("cdn")) + "?width=200&height=280", + altTitle = title.getString("en").takeUnless(String::isEmpty), + author = null, + isNsfw = false, + rating = RATING_UNKNOWN, + url = href, + publicUrl = "https://${domain}/$href", + tags = emptySet(), + state = null, + source = source, + ) + } + } - private suspend fun apiCall(request: String): JSONObject { - return context.graphQLQuery("https://api.${getDomain()}/graphql", request) - .getJSONObject("data") - } + private suspend fun apiCall(request: String): JSONObject { + return webClient.graphQLQuery("https://api.${domain}/graphql", request) + .getJSONObject("data") + } - private fun JSONArray.mapToTags(): Set { + private fun JSONArray.mapToTags(): Set { - fun toTitle(slug: String): String { - val builder = StringBuilder(slug) - var capitalize = true - for ((i, c) in builder.withIndex()) { - when { - c == '-' -> { - builder.setCharAt(i, ' ') - } - capitalize -> { - builder.setCharAt(i, c.uppercaseChar()) - capitalize = false - } - } - } - return builder.toString() - } + fun toTitle(slug: String): String { + val builder = StringBuilder(slug) + var capitalize = true + for ((i, c) in builder.withIndex()) { + when { + c == '-' -> { + builder.setCharAt(i, ' ') + } + + capitalize -> { + builder.setCharAt(i, c.uppercaseChar()) + capitalize = false + } + } + } + return builder.toString() + } - val result = ArraySet(length()) - stringIterator().forEach { - result.add( - MangaTag( - title = toTitle(it), - key = it, - source = source, - ), - ) - } - return result - } -} \ No newline at end of file + val result = ArraySet(length()) + stringIterator().forEach { + result.add( + MangaTag( + title = toTitle(it), + key = it, + source = source, + ), + ) + } + return result + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BatoToParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BatoToParser.kt index a366c1d1..f162a680 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BatoToParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BatoToParser.kt @@ -19,292 +19,293 @@ import javax.crypto.spec.IvParameterSpec import javax.crypto.spec.SecretKeySpec @MangaSourceParser("BATOTO", "Bato.To") -internal class BatoToParser(override val context: MangaLoaderContext) : PagedMangaParser( - source = MangaSource.BATOTO, - pageSize = 60, - searchPageSize = 20, +internal class BatoToParser(context: MangaLoaderContext) : PagedMangaParser( + context = context, + source = MangaSource.BATOTO, + pageSize = 60, + searchPageSize = 20, ) { - override val sortOrders: Set = EnumSet.of( - SortOrder.NEWEST, - SortOrder.UPDATED, - SortOrder.POPULARITY, - SortOrder.ALPHABETICAL, - ) + override val sortOrders: Set = EnumSet.of( + SortOrder.NEWEST, + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.ALPHABETICAL, + ) - override val configKeyDomain = ConfigKey.Domain( - "bato.to", - arrayOf("bato.to", "mto.to", "mangatoto.com", "battwo.com", "batotwo.com", "comiko.net", "batotoo.com"), - ) + override val configKeyDomain = ConfigKey.Domain( + "bato.to", + arrayOf("bato.to", "mto.to", "mangatoto.com", "battwo.com", "batotwo.com", "comiko.net", "batotoo.com"), + ) - override suspend fun getListPage( - page: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - if (!query.isNullOrEmpty()) { - return search(page, query) - } - @Suppress("NON_EXHAUSTIVE_WHEN_STATEMENT") - val url = buildString { - append("https://") - append(getDomain()) - append("/browse?sort=") - when (sortOrder) { - SortOrder.UPDATED, - -> append("update.za") + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + if (!query.isNullOrEmpty()) { + return search(page, query) + } + @Suppress("NON_EXHAUSTIVE_WHEN_STATEMENT") + val url = buildString { + append("https://") + append(domain) + append("/browse?sort=") + when (sortOrder) { + SortOrder.UPDATED, + -> append("update.za") - SortOrder.POPULARITY -> append("views_a.za") - SortOrder.NEWEST -> append("create.za") - SortOrder.ALPHABETICAL -> append("title.az") - } - if (!tags.isNullOrEmpty()) { - append("&genres=") - appendAll(tags, ",") { it.key } - } - append("&page=") - append(page) - } - return parseList(url, page) - } + SortOrder.POPULARITY -> append("views_a.za") + SortOrder.NEWEST -> append("create.za") + SortOrder.ALPHABETICAL -> append("title.az") + } + if (!tags.isNullOrEmpty()) { + append("&genres=") + appendAll(tags, ",") { it.key } + } + append("&page=") + append(page) + } + return parseList(url, page) + } - override suspend fun getDetails(manga: Manga): Manga { - val root = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml() - .requireElementById("mainer") - val details = root.selectFirstOrThrow(".detail-set") - val attrs = details.selectFirst(".attr-main")?.select(".attr-item")?.associate { - it.child(0).text().trim() to it.child(1) - }.orEmpty() - return manga.copy( - title = root.selectFirst("h3.item-title")?.text() ?: manga.title, - isNsfw = !root.selectFirst("alert")?.getElementsContainingOwnText("NSFW").isNullOrEmpty(), - largeCoverUrl = details.selectFirst("img[src]")?.absUrl("src"), - description = details.getElementById("limit-height-body-summary") - ?.selectFirst(".limit-html") - ?.html(), - tags = manga.tags + attrs["Genres:"]?.parseTags().orEmpty(), - state = when (attrs["Release status:"]?.text()) { - "Ongoing" -> MangaState.ONGOING - "Completed" -> MangaState.FINISHED - else -> manga.state - }, - author = attrs["Authors:"]?.text()?.trim() ?: manga.author, - chapters = root.selectFirst(".episode-list") - ?.selectFirst(".main") - ?.children() - ?.reversed() - ?.mapChapters { i, div -> - div.parseChapter(i) - }.orEmpty(), - ) - } + override suspend fun getDetails(manga: Manga): Manga { + val root = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + .requireElementById("mainer") + val details = root.selectFirstOrThrow(".detail-set") + val attrs = details.selectFirst(".attr-main")?.select(".attr-item")?.associate { + it.child(0).text().trim() to it.child(1) + }.orEmpty() + return manga.copy( + title = root.selectFirst("h3.item-title")?.text() ?: manga.title, + isNsfw = !root.selectFirst("alert")?.getElementsContainingOwnText("NSFW").isNullOrEmpty(), + largeCoverUrl = details.selectFirst("img[src]")?.absUrl("src"), + description = details.getElementById("limit-height-body-summary") + ?.selectFirst(".limit-html") + ?.html(), + tags = manga.tags + attrs["Genres:"]?.parseTags().orEmpty(), + state = when (attrs["Release status:"]?.text()) { + "Ongoing" -> MangaState.ONGOING + "Completed" -> MangaState.FINISHED + else -> manga.state + }, + author = attrs["Authors:"]?.text()?.trim() ?: manga.author, + chapters = root.selectFirst(".episode-list") + ?.selectFirst(".main") + ?.children() + ?.reversed() + ?.mapChapters { i, div -> + div.parseChapter(i) + }.orEmpty(), + ) + } - override suspend fun getPages(chapter: MangaChapter): List { - val fullUrl = chapter.url.toAbsoluteUrl(getDomain()) - val scripts = context.httpGet(fullUrl).parseHtml().select("script") - for (script in scripts) { - val scriptSrc = script.html() - val p = scriptSrc.indexOf("const imgHttpLis =") - if (p == -1) continue - val start = scriptSrc.indexOf('[', p) - val end = scriptSrc.indexOf(';', start) - if (start == -1 || end == -1) { - continue - } - val images = JSONArray(scriptSrc.substring(start, end)) - val batoPass = scriptSrc.substringBetweenFirst("batoPass =", ";")?.trim(' ', '"', '\n') - ?: script.parseFailed("Cannot find batoPass") - val batoWord = scriptSrc.substringBetweenFirst("batoWord =", ";")?.trim(' ', '"', '\n') - ?: script.parseFailed("Cannot find batoWord") - val password = context.evaluateJs(batoPass)?.removeSurrounding('"') - ?: script.parseFailed("Cannot evaluate batoPass") - val args = JSONArray(decryptAES(batoWord, password)) - val result = ArrayList(images.length()) - repeat(images.length()) { i -> - val url = images.getString(i) - result += MangaPage( - id = generateUid(url), - url = url + "?" + args.getString(i), - referer = fullUrl, - preview = null, - source = source, - ) - } - return result - } - throw ParseException("Cannot find images list", fullUrl) - } + override suspend fun getPages(chapter: MangaChapter): List { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val scripts = webClient.httpGet(fullUrl).parseHtml().select("script") + for (script in scripts) { + val scriptSrc = script.html() + val p = scriptSrc.indexOf("const imgHttpLis =") + if (p == -1) continue + val start = scriptSrc.indexOf('[', p) + val end = scriptSrc.indexOf(';', start) + if (start == -1 || end == -1) { + continue + } + val images = JSONArray(scriptSrc.substring(start, end)) + val batoPass = scriptSrc.substringBetweenFirst("batoPass =", ";")?.trim(' ', '"', '\n') + ?: script.parseFailed("Cannot find batoPass") + val batoWord = scriptSrc.substringBetweenFirst("batoWord =", ";")?.trim(' ', '"', '\n') + ?: script.parseFailed("Cannot find batoWord") + val password = context.evaluateJs(batoPass)?.removeSurrounding('"') + ?: script.parseFailed("Cannot evaluate batoPass") + val args = JSONArray(decryptAES(batoWord, password)) + val result = ArrayList(images.length()) + repeat(images.length()) { i -> + val url = images.getString(i) + result += MangaPage( + id = generateUid(url), + url = url + "?" + args.getString(i), + referer = fullUrl, + preview = null, + source = source, + ) + } + return result + } + throw ParseException("Cannot find images list", fullUrl) + } - override suspend fun getTags(): Set { - val scripts = context.httpGet( - "https://${getDomain()}/browse", - ).parseHtml().selectOrThrow("script") - for (script in scripts) { - val genres = script.html().substringBetweenFirst("const _genres =", ";") ?: continue - val jo = JSONObject(genres) - val result = ArraySet(jo.length()) - jo.keys().forEach { key -> - val item = jo.getJSONObject(key) - result += MangaTag( - title = item.getString("text").toTitleCase(), - key = item.getString("file"), - source = source, - ) - } - return result - } - throw ParseException("Cannot find gernes list", scripts[0].baseUri()) - } + override suspend fun getTags(): Set { + val scripts = webClient.httpGet( + "https://${domain}/browse", + ).parseHtml().selectOrThrow("script") + for (script in scripts) { + val genres = script.html().substringBetweenFirst("const _genres =", ";") ?: continue + val jo = JSONObject(genres) + val result = ArraySet(jo.length()) + jo.keys().forEach { key -> + val item = jo.getJSONObject(key) + result += MangaTag( + title = item.getString("text").toTitleCase(), + key = item.getString("file"), + source = source, + ) + } + return result + } + throw ParseException("Cannot find gernes list", scripts[0].baseUri()) + } - override fun getFaviconUrl(): String = "https://styles.amarkcdn.com/img/batoto/favicon.ico?v0" + override fun getFaviconUrl(): String = "https://styles.amarkcdn.com/img/batoto/favicon.ico?v0" - private suspend fun search(page: Int, query: String): List { - val url = buildString { - append("https://") - append(getDomain()) - append("/search?word=") - append(query.replace(' ', '+')) - append("&page=") - append(page) - } - return parseList(url, page) - } + private suspend fun search(page: Int, query: String): List { + val url = buildString { + append("https://") + append(domain) + append("/search?word=") + append(query.replace(' ', '+')) + append("&page=") + append(page) + } + return parseList(url, page) + } - private fun getActivePage(body: Element): Int = body.select("nav ul.pagination > li.page-item.active") - .lastOrNull() - ?.text() - ?.toIntOrNull() ?: body.parseFailed("Cannot determine current page") + private fun getActivePage(body: Element): Int = body.select("nav ul.pagination > li.page-item.active") + .lastOrNull() + ?.text() + ?.toIntOrNull() ?: body.parseFailed("Cannot determine current page") - private suspend fun parseList(url: String, page: Int): List { - val body = context.httpGet(url).parseHtml().body() - if (body.selectFirst(".browse-no-matches") != null) { - return emptyList() - } - val activePage = getActivePage(body) - if (activePage != page) { - return emptyList() - } - val root = body.requireElementById("series-list") - return root.children().map { div -> - val a = div.selectFirstOrThrow("a") - val href = a.attrAsRelativeUrl("href") - val title = div.selectFirstOrThrow(".item-title").text() - Manga( - id = generateUid(href), - title = title, - altTitle = div.selectFirst(".item-alias")?.text()?.takeUnless { it == title }, - url = href, - publicUrl = a.absUrl("href"), - rating = RATING_UNKNOWN, - isNsfw = false, - coverUrl = div.selectFirst("img[src]")?.absUrl("src").orEmpty(), - largeCoverUrl = null, - description = null, - tags = div.selectFirst(".item-genre")?.parseTags().orEmpty(), - state = null, - author = null, - source = source, - ) - } - } + private suspend fun parseList(url: String, page: Int): List { + val body = webClient.httpGet(url).parseHtml().body() + if (body.selectFirst(".browse-no-matches") != null) { + return emptyList() + } + val activePage = getActivePage(body) + if (activePage != page) { + return emptyList() + } + val root = body.requireElementById("series-list") + return root.children().map { div -> + val a = div.selectFirstOrThrow("a") + val href = a.attrAsRelativeUrl("href") + val title = div.selectFirstOrThrow(".item-title").text() + Manga( + id = generateUid(href), + title = title, + altTitle = div.selectFirst(".item-alias")?.text()?.takeUnless { it == title }, + url = href, + publicUrl = a.absUrl("href"), + rating = RATING_UNKNOWN, + isNsfw = false, + coverUrl = div.selectFirst("img[src]")?.absUrl("src").orEmpty(), + largeCoverUrl = null, + description = null, + tags = div.selectFirst(".item-genre")?.parseTags().orEmpty(), + state = null, + author = null, + source = source, + ) + } + } - private fun Element.parseTags() = children().mapToSet { span -> - val text = span.ownText() - MangaTag( - title = text.toTitleCase(), - key = text.lowercase(Locale.ENGLISH).replace(' ', '_'), - source = source, - ) - } + private fun Element.parseTags() = children().mapToSet { span -> + val text = span.ownText() + MangaTag( + title = text.toTitleCase(), + key = text.lowercase(Locale.ENGLISH).replace(' ', '_'), + source = source, + ) + } - private fun Element.parseChapter(index: Int): MangaChapter? { - val a = selectFirst("a.chapt") ?: return null - val extra = selectFirst(".extra") - val href = a.attrAsRelativeUrl("href") - return MangaChapter( - id = generateUid(href), - name = a.text(), - number = index + 1, - url = href, - scanlator = extra?.getElementsByAttributeValueContaining("href", "/group/")?.text(), - uploadDate = runCatching { - parseChapterDate(extra?.select("i")?.lastOrNull()?.ownText()) - }.getOrDefault(0), - branch = null, - source = source, - ) - } + private fun Element.parseChapter(index: Int): MangaChapter? { + val a = selectFirst("a.chapt") ?: return null + val extra = selectFirst(".extra") + val href = a.attrAsRelativeUrl("href") + return MangaChapter( + id = generateUid(href), + name = a.text(), + number = index + 1, + url = href, + scanlator = extra?.getElementsByAttributeValueContaining("href", "/group/")?.text(), + uploadDate = runCatching { + parseChapterDate(extra?.select("i")?.lastOrNull()?.ownText()) + }.getOrDefault(0), + branch = null, + source = source, + ) + } - private fun parseChapterDate(date: String?): Long { - if (date.isNullOrEmpty()) { - return 0 - } - val value = date.substringBefore(' ').toInt() - val field = when { - "sec" in date -> Calendar.SECOND - "min" in date -> Calendar.MINUTE - "hour" in date -> Calendar.HOUR - "day" in date -> Calendar.DAY_OF_MONTH - "week" in date -> Calendar.WEEK_OF_YEAR - "month" in date -> Calendar.MONTH - "year" in date -> Calendar.YEAR - else -> return 0 - } - val calendar = Calendar.getInstance() - calendar.add(field, -value) - return calendar.timeInMillis - } + private fun parseChapterDate(date: String?): Long { + if (date.isNullOrEmpty()) { + return 0 + } + val value = date.substringBefore(' ').toInt() + val field = when { + "sec" in date -> Calendar.SECOND + "min" in date -> Calendar.MINUTE + "hour" in date -> Calendar.HOUR + "day" in date -> Calendar.DAY_OF_MONTH + "week" in date -> Calendar.WEEK_OF_YEAR + "month" in date -> Calendar.MONTH + "year" in date -> Calendar.YEAR + else -> return 0 + } + val calendar = Calendar.getInstance() + calendar.add(field, -value) + return calendar.timeInMillis + } - private fun decryptAES(encrypted: String, password: String): String { - val cipherData = context.decodeBase64(encrypted) - val saltData = cipherData.copyOfRange(8, 16) - val (key, iv) = generateKeyAndIV( - keyLength = 32, - ivLength = 16, - iterations = 1, - salt = saltData, - password = password.toByteArray(StandardCharsets.UTF_8), - md = MessageDigest.getInstance("MD5"), - ) - val encryptedData = cipherData.copyOfRange(16, cipherData.size) - val cipher = Cipher.getInstance("AES/CBC/PKCS5Padding") - cipher.init(Cipher.DECRYPT_MODE, key, iv) - return cipher.doFinal(encryptedData).toString(Charsets.UTF_8) - } + private fun decryptAES(encrypted: String, password: String): String { + val cipherData = context.decodeBase64(encrypted) + val saltData = cipherData.copyOfRange(8, 16) + val (key, iv) = generateKeyAndIV( + keyLength = 32, + ivLength = 16, + iterations = 1, + salt = saltData, + password = password.toByteArray(StandardCharsets.UTF_8), + md = MessageDigest.getInstance("MD5"), + ) + val encryptedData = cipherData.copyOfRange(16, cipherData.size) + val cipher = Cipher.getInstance("AES/CBC/PKCS5Padding") + cipher.init(Cipher.DECRYPT_MODE, key, iv) + return cipher.doFinal(encryptedData).toString(Charsets.UTF_8) + } - @Suppress("SameParameterValue") - private fun generateKeyAndIV( - keyLength: Int, - ivLength: Int, - iterations: Int, - salt: ByteArray, - password: ByteArray, - md: MessageDigest, - ): Pair { - val digestLength = md.digestLength - val requiredLength = (keyLength + ivLength + digestLength - 1) / digestLength * digestLength - val generatedData = ByteArray(requiredLength) - var generatedLength = 0 - md.reset() - while (generatedLength < keyLength + ivLength) { - if (generatedLength > 0) { - md.update(generatedData, generatedLength - digestLength, digestLength) - } - md.update(password) - md.update(salt, 0, 8) - md.digest(generatedData, generatedLength, digestLength) - repeat(iterations - 1) { - md.update(generatedData, generatedLength, digestLength) - md.digest(generatedData, generatedLength, digestLength) - } - generatedLength += digestLength - } + @Suppress("SameParameterValue") + private fun generateKeyAndIV( + keyLength: Int, + ivLength: Int, + iterations: Int, + salt: ByteArray, + password: ByteArray, + md: MessageDigest, + ): Pair { + val digestLength = md.digestLength + val requiredLength = (keyLength + ivLength + digestLength - 1) / digestLength * digestLength + val generatedData = ByteArray(requiredLength) + var generatedLength = 0 + md.reset() + while (generatedLength < keyLength + ivLength) { + if (generatedLength > 0) { + md.update(generatedData, generatedLength - digestLength, digestLength) + } + md.update(password) + md.update(salt, 0, 8) + md.digest(generatedData, generatedLength, digestLength) + repeat(iterations - 1) { + md.update(generatedData, generatedLength, digestLength) + md.digest(generatedData, generatedLength, digestLength) + } + generatedLength += digestLength + } - return SecretKeySpec(generatedData.copyOfRange(0, keyLength), "AES") to IvParameterSpec( - if (ivLength > 0) { - generatedData.copyOfRange(keyLength, keyLength + ivLength) - } else byteArrayOf(), - ) - } -} \ No newline at end of file + return SecretKeySpec(generatedData.copyOfRange(0, keyLength), "AES") to IvParameterSpec( + if (ivLength > 0) { + generatedData.copyOfRange(keyLength, keyLength + ivLength) + } else byteArrayOf(), + ) + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BlogTruyenParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BlogTruyenParser.kt index 42cff91d..05b7b22d 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BlogTruyenParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BlogTruyenParser.kt @@ -14,240 +14,241 @@ import org.koitharu.kotatsu.parsers.model.* import org.koitharu.kotatsu.parsers.util.* import java.text.SimpleDateFormat import java.util.* -import kotlin.collections.HashSet @MangaSourceParser("BLOGTRUYEN", "BlogTruyen", "vi") -class BlogTruyenParser(override val context: MangaLoaderContext) : - PagedMangaParser(MangaSource.BLOGTRUYEN, pageSize = 20) { - - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("blogtruyen.vn", null) - - override val sortOrders: Set - get() = EnumSet.of(SortOrder.UPDATED) - - private val mutex = Mutex() - private val dateFormat = SimpleDateFormat("dd/MM/yyyy HH:mm", Locale.US) - private var cacheTags: ArrayMap? = null - - override suspend fun getDetails(manga: Manga): Manga { - val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml() - val descriptionElement = doc.selectFirstOrThrow("div.description") - val statusText = descriptionElement - .selectFirst("p:contains(Trạng thái) > span.color-red") - ?.text() - - val state = when (statusText) { - "Đang tiến hành" -> MangaState.ONGOING - "Đã hoàn thành" -> MangaState.FINISHED - else -> null - } - - val rating = doc.selectFirst("span.total-vote")?.attr("ng-init")?.let { text -> - val like = text.substringAfter("TotalLike=") - .substringBefore(';') - .toIntOrNull() ?: return@let RATING_UNKNOWN - val dislike = text.substringAfter("TotalDisLike=") - .toIntOrNull() ?: return@let RATING_UNKNOWN - - when { - like == 0 && dislike == 0 -> RATING_UNKNOWN - else -> like.toFloat() / (like + dislike) - } - } - - val tagMap = getOrCreateTagMap() - val tags = descriptionElement.select("p > span.category").mapNotNullToSet { - val tagName = it.selectFirst("a")?.text()?.trim() ?: return@mapNotNullToSet null - tagMap[tagName] - } - - return manga.copy( - tags = tags, - author = descriptionElement.selectFirst("p:contains(Tác giả) > a")?.text(), - description = doc.selectFirst(".detail .content")?.html(), - chapters = parseChapterList(doc), - largeCoverUrl = doc.selectLast("div.thumbnail > img")?.imageUrl(), - state = state, - rating = rating ?: RATING_UNKNOWN, - isNsfw = doc.getElementById("warningCategory") != null - ) - } - - private fun parseChapterList(doc: Document): List { - val chapterList = doc.select("#list-chapters > p") - return chapterList.asReversed().mapChapters { index, element -> - val titleElement = element.selectFirst("span.title > a") ?: return@mapChapters null - val name = titleElement.text() - val relativeUrl = titleElement.attrAsRelativeUrl("href") - val id = relativeUrl.substringAfter('/').substringBefore('/') - val uploadDate = dateFormat.tryParse(element.select("span.publishedDate").text()) - MangaChapter( - id = generateUid(id), - name = name, - number = index + 1, - url = relativeUrl, - scanlator = null, - uploadDate = uploadDate, - branch = null, - source = source - ) - } - } - - override suspend fun getListPage( - page: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - return when { - !query.isNullOrEmpty() -> { - val searchUrl = "https://${getDomain()}/timkiem/nangcao/1/0/-1/-1?txt=${query.urlEncoded()}&p=$page" - val searchContent = context.httpGet(searchUrl).parseHtml() - .selectFirst("section.list-manga-bycate > div.list") - parseMangaList(searchContent) - } - - !tags.isNullOrEmpty() -> { - val tag = tags.oneOrThrowIfMany()!! - val categoryAjax = "https://${getDomain()}/ajax/Category/AjaxLoadMangaByCategory?id=${tag.key}&orderBy=5&p=$page" - val listContent = context.httpGet(categoryAjax).parseHtml().selectFirst("div.list") - parseMangaList(listContent) - } - else -> getNormalList(page) - } - } - - private suspend fun getNormalList(page: Int): List { - val pageLink = "https://${getDomain()}/page-$page" - val doc = context.httpGet(pageLink).parseHtml() - val listElements = doc.selectFirstOrThrow("section.list-mainpage.listview") - .select("div.bg-white.storyitem") - - return listElements.mapNotNull { - val linkTag = it.selectFirst("div.fl-l > a") ?: return@mapNotNull null - val relativeUrl = linkTag.attrAsRelativeUrl("href") - val tagMap = getOrCreateTagMap() - val tags = it.select("footer > div.category > a").mapNotNullToSet { a -> - tagMap[a.text()] - } - - Manga( - id = generateUid(relativeUrl), - title = linkTag.attr("title"), - altTitle = null, - description = it.selectFirst("p.al-j.break.line-height-15")?.text(), - url = relativeUrl, - publicUrl = relativeUrl.toAbsoluteUrl(getDomain()), - coverUrl = linkTag.selectLast("img")?.imageUrl().orEmpty(), - source = source, - tags = tags, - isNsfw = false, - rating = RATING_UNKNOWN, - author = null, - state = null, - ) - } - } - - private fun parseMangaList(listElement: Element?): List { - listElement ?: return emptyList() - - return listElement.select("span.tiptip[data-tiptip]").mapNotNull { - val mangaInfo = listElement.getElementById(it.attr("data-tiptip")) ?: return@mapNotNull null - val a = it.selectFirst("a") ?: return@mapNotNull null - val relativeUrl = a.attrAsRelativeUrl("href") - Manga( - id = generateUid(relativeUrl), - title = a.text(), - altTitle = null, - description = mangaInfo.select("div.al-j.fs-12").text(), - url = relativeUrl, - publicUrl = relativeUrl.toAbsoluteUrl(getDomain()), - coverUrl = mangaInfo.selectFirst("div > img.img")?.imageUrl().orEmpty(), - isNsfw = false, - rating = RATING_UNKNOWN, - tags = emptySet(), - author = null, - state = null, - source = source, - ) - } - } - - override suspend fun getPages(chapter: MangaChapter): List { - fun generateImageId(index: Int) = generateUid("${chapter.url}/$index") - - val doc = context.httpGet(chapter.url.toAbsoluteUrl(getDomain())).parseHtml() - val pages = ArrayList() - val referer = chapter.url.toAbsoluteUrl(getDomain()) - doc.select("#content > img").forEach { img -> - pages.add( - MangaPage( - id = generateImageId(pages.size), - url = img.imageUrl(), - referer = referer, - preview = null, - source = source, - ) - ) - } - - // Some chapters use js script to render images - val script = doc.selectLast("#content > script") - if (script != null && script.data().contains("listImageCaption")) { - val imagesStr = script.data().substringBefore(';').substringAfterLast('=').trim() - val imageArr = JSONArray(imagesStr) - for (i in 0 until imageArr.length()) { - val imageUrl = imageArr.getJSONObject(i).getString("url") - pages.add( - MangaPage( - id = generateImageId(pages.size), - url = imageUrl, - referer = referer, - preview = null, - source = source - ) - ) - } - } - - return pages - } - - override suspend fun getTags(): Set { - val map = getOrCreateTagMap() - val tags = HashSet(map.size) - for (entry in map) { - tags.add(entry.value) - } - - return tags - } - - - private suspend fun getOrCreateTagMap(): ArrayMap = mutex.withLock { - cacheTags?.let { return@withLock it } - val doc = context.httpGet("/timkiem/nangcao".toAbsoluteUrl(getDomain())).parseHtml() - val tagItems = doc.select("li[data-id]") - val tagMap = ArrayMap(tagItems.size) - for (tag in tagItems) { - val title = tag.text().trim() - tagMap[tag.text().trim()] = MangaTag( - title = title, - key = tag.attr("data-id"), - source = source - ) - } - - cacheTags = tagMap - tagMap - } - - private fun Element.imageUrl(): String { - return attrAsAbsoluteUrlOrNull("src") - ?: attrAsAbsoluteUrlOrNull("data-cfsrc") - ?: "" - } +class BlogTruyenParser(context: MangaLoaderContext) : + PagedMangaParser(context, MangaSource.BLOGTRUYEN, pageSize = 20) { + + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("blogtruyen.vn", null) + + override val sortOrders: Set + get() = EnumSet.of(SortOrder.UPDATED) + + private val mutex = Mutex() + private val dateFormat = SimpleDateFormat("dd/MM/yyyy HH:mm", Locale.US) + private var cacheTags: ArrayMap? = null + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val descriptionElement = doc.selectFirstOrThrow("div.description") + val statusText = descriptionElement + .selectFirst("p:contains(Trạng thái) > span.color-red") + ?.text() + + val state = when (statusText) { + "Đang tiến hành" -> MangaState.ONGOING + "Đã hoàn thành" -> MangaState.FINISHED + else -> null + } + + val rating = doc.selectFirst("span.total-vote")?.attr("ng-init")?.let { text -> + val like = text.substringAfter("TotalLike=") + .substringBefore(';') + .toIntOrNull() ?: return@let RATING_UNKNOWN + val dislike = text.substringAfter("TotalDisLike=") + .toIntOrNull() ?: return@let RATING_UNKNOWN + + when { + like == 0 && dislike == 0 -> RATING_UNKNOWN + else -> like.toFloat() / (like + dislike) + } + } + + val tagMap = getOrCreateTagMap() + val tags = descriptionElement.select("p > span.category").mapNotNullToSet { + val tagName = it.selectFirst("a")?.text()?.trim() ?: return@mapNotNullToSet null + tagMap[tagName] + } + + return manga.copy( + tags = tags, + author = descriptionElement.selectFirst("p:contains(Tác giả) > a")?.text(), + description = doc.selectFirst(".detail .content")?.html(), + chapters = parseChapterList(doc), + largeCoverUrl = doc.selectLast("div.thumbnail > img")?.imageUrl(), + state = state, + rating = rating ?: RATING_UNKNOWN, + isNsfw = doc.getElementById("warningCategory") != null, + ) + } + + private fun parseChapterList(doc: Document): List { + val chapterList = doc.select("#list-chapters > p") + return chapterList.asReversed().mapChapters { index, element -> + val titleElement = element.selectFirst("span.title > a") ?: return@mapChapters null + val name = titleElement.text() + val relativeUrl = titleElement.attrAsRelativeUrl("href") + val id = relativeUrl.substringAfter('/').substringBefore('/') + val uploadDate = dateFormat.tryParse(element.select("span.publishedDate").text()) + MangaChapter( + id = generateUid(id), + name = name, + number = index + 1, + url = relativeUrl, + scanlator = null, + uploadDate = uploadDate, + branch = null, + source = source, + ) + } + } + + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + return when { + !query.isNullOrEmpty() -> { + val searchUrl = "https://${domain}/timkiem/nangcao/1/0/-1/-1?txt=${query.urlEncoded()}&p=$page" + val searchContent = webClient.httpGet(searchUrl).parseHtml() + .selectFirst("section.list-manga-bycate > div.list") + parseMangaList(searchContent) + } + + !tags.isNullOrEmpty() -> { + val tag = tags.oneOrThrowIfMany()!! + val categoryAjax = + "https://${domain}/ajax/Category/AjaxLoadMangaByCategory?id=${tag.key}&orderBy=5&p=$page" + val listContent = webClient.httpGet(categoryAjax).parseHtml().selectFirst("div.list") + parseMangaList(listContent) + } + + else -> getNormalList(page) + } + } + + private suspend fun getNormalList(page: Int): List { + val pageLink = "https://${domain}/page-$page" + val doc = webClient.httpGet(pageLink).parseHtml() + val listElements = doc.selectFirstOrThrow("section.list-mainpage.listview") + .select("div.bg-white.storyitem") + + return listElements.mapNotNull { + val linkTag = it.selectFirst("div.fl-l > a") ?: return@mapNotNull null + val relativeUrl = linkTag.attrAsRelativeUrl("href") + val tagMap = getOrCreateTagMap() + val tags = it.select("footer > div.category > a").mapNotNullToSet { a -> + tagMap[a.text()] + } + + Manga( + id = generateUid(relativeUrl), + title = linkTag.attr("title"), + altTitle = null, + description = it.selectFirst("p.al-j.break.line-height-15")?.text(), + url = relativeUrl, + publicUrl = relativeUrl.toAbsoluteUrl(domain), + coverUrl = linkTag.selectLast("img")?.imageUrl().orEmpty(), + source = source, + tags = tags, + isNsfw = false, + rating = RATING_UNKNOWN, + author = null, + state = null, + ) + } + } + + private fun parseMangaList(listElement: Element?): List { + listElement ?: return emptyList() + + return listElement.select("span.tiptip[data-tiptip]").mapNotNull { + val mangaInfo = listElement.getElementById(it.attr("data-tiptip")) ?: return@mapNotNull null + val a = it.selectFirst("a") ?: return@mapNotNull null + val relativeUrl = a.attrAsRelativeUrl("href") + Manga( + id = generateUid(relativeUrl), + title = a.text(), + altTitle = null, + description = mangaInfo.select("div.al-j.fs-12").text(), + url = relativeUrl, + publicUrl = relativeUrl.toAbsoluteUrl(domain), + coverUrl = mangaInfo.selectFirst("div > img.img")?.imageUrl().orEmpty(), + isNsfw = false, + rating = RATING_UNKNOWN, + tags = emptySet(), + author = null, + state = null, + source = source, + ) + } + } + + override suspend fun getPages(chapter: MangaChapter): List { + fun generateImageId(index: Int) = generateUid("${chapter.url}/$index") + + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + val pages = ArrayList() + val referer = chapter.url.toAbsoluteUrl(domain) + doc.select("#content > img").forEach { img -> + pages.add( + MangaPage( + id = generateImageId(pages.size), + url = img.imageUrl(), + referer = referer, + preview = null, + source = source, + ), + ) + } + + // Some chapters use js script to render images + val script = doc.selectLast("#content > script") + if (script != null && script.data().contains("listImageCaption")) { + val imagesStr = script.data().substringBefore(';').substringAfterLast('=').trim() + val imageArr = JSONArray(imagesStr) + for (i in 0 until imageArr.length()) { + val imageUrl = imageArr.getJSONObject(i).getString("url") + pages.add( + MangaPage( + id = generateImageId(pages.size), + url = imageUrl, + referer = referer, + preview = null, + source = source, + ), + ) + } + } + + return pages + } + + override suspend fun getTags(): Set { + val map = getOrCreateTagMap() + val tags = HashSet(map.size) + for (entry in map) { + tags.add(entry.value) + } + + return tags + } + + + private suspend fun getOrCreateTagMap(): ArrayMap = mutex.withLock { + cacheTags?.let { return@withLock it } + val doc = webClient.httpGet("/timkiem/nangcao".toAbsoluteUrl(domain)).parseHtml() + val tagItems = doc.select("li[data-id]") + val tagMap = ArrayMap(tagItems.size) + for (tag in tagItems) { + val title = tag.text().trim() + tagMap[tag.text().trim()] = MangaTag( + title = title, + key = tag.attr("data-id"), + source = source, + ) + } + + cacheTags = tagMap + tagMap + } + + private fun Element.imageUrl(): String { + return attrAsAbsoluteUrlOrNull("src") + ?: attrAsAbsoluteUrlOrNull("data-cfsrc") + ?: "" + } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/CloneMangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/CloneMangaParser.kt index a4d0f797..9bdea84f 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/CloneMangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/CloneMangaParser.kt @@ -1,104 +1,107 @@ package org.koitharu.kotatsu.parsers.site -import org.koitharu.kotatsu.parsers.* +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.PagedMangaParser import org.koitharu.kotatsu.parsers.config.ConfigKey import org.koitharu.kotatsu.parsers.model.* import org.koitharu.kotatsu.parsers.util.* import java.util.* @MangaSourceParser("CLONEMANGA", "CloneManga", "en") -internal class CloneMangaParser(override val context: MangaLoaderContext) : PagedMangaParser( - MangaSource.CLONEMANGA, - pageSize = 1, +internal class CloneMangaParser(context: MangaLoaderContext) : PagedMangaParser( + context, + MangaSource.CLONEMANGA, + pageSize = 1, ) { - override val sortOrders: Set = Collections.singleton( - SortOrder.POPULARITY, - ) + override val sortOrders: Set = Collections.singleton( + SortOrder.POPULARITY, + ) - override val configKeyDomain = ConfigKey.Domain("manga.clone-army.org", null) + override val configKeyDomain = ConfigKey.Domain("manga.clone-army.org", null) - override fun getFaviconUrl(): String { - return "https://pbs.twimg.com/profile_images/458758466346029056/Ys93EANp_400x400.png" - } + override fun getFaviconUrl(): String { + return "https://pbs.twimg.com/profile_images/458758466346029056/Ys93EANp_400x400.png" + } - override suspend fun getListPage( - page: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - if (query != null || page > 1) { - return emptyList() - } - val link = "https://${getDomain()}/viewer_landing.php" - val doc = context.httpGet(link).parseHtml() - val mangas = doc.getElementsByClass("comicPreviewContainer") - return mangas.mapNotNull { item -> - val attr = item.getElementsByClass("comicPreview").attr("style") - val href = item.selectFirst("a")?.attrAsAbsoluteUrl("href") ?: return@mapNotNull null - val cover = attr.substring(attr.indexOf("site/themes"), attr.indexOf(")")) - Manga( - id = generateUid(href), - title = item.selectFirst("h3")?.text() ?: return@mapNotNull null, - coverUrl = "https://${getDomain()}/$cover", - altTitle = null, - author = "Dan Kim", - rating = RATING_UNKNOWN, - url = href, - isNsfw = false, - tags = emptySet(), - state = null, - publicUrl = href.toAbsoluteUrl(getDomain()), - source = source, - ) - } - } + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + if (query != null || page > 1) { + return emptyList() + } + val link = "https://${domain}/viewer_landing.php" + val doc = webClient.httpGet(link).parseHtml() + val mangas = doc.getElementsByClass("comicPreviewContainer") + return mangas.mapNotNull { item -> + val attr = item.getElementsByClass("comicPreview").attr("style") + val href = item.selectFirst("a")?.attrAsAbsoluteUrl("href") ?: return@mapNotNull null + val cover = attr.substring(attr.indexOf("site/themes"), attr.indexOf(")")) + Manga( + id = generateUid(href), + title = item.selectFirst("h3")?.text() ?: return@mapNotNull null, + coverUrl = "https://${domain}/$cover", + altTitle = null, + author = "Dan Kim", + rating = RATING_UNKNOWN, + url = href, + isNsfw = false, + tags = emptySet(), + state = null, + publicUrl = href.toAbsoluteUrl(domain), + source = source, + ) + } + } - override suspend fun getDetails(manga: Manga): Manga { - val doc = context.httpGet(manga.publicUrl).parseHtml() - val series = doc.location() - val numChapters = Regex( - pattern = "&page=(.*)&lang=", - ).findAll( - input = doc.getElementsByTag("script")[3].toString(), - ) - .elementAt(3).destructured.component1() - .toInt() - val chapters = ArrayList() - for (i in 0..numChapters) { - val chapter = MangaChapter( - id = generateUid("$series&page=$i"), - name = "Chapter ${i + 1}", - number = i + 1, - url = "$series&page=$i", - scanlator = null, - branch = null, - uploadDate = 0L, - source = MangaSource.DUMMY, - ) - chapters.add(chapter) - } - return manga.copy(chapters = chapters) - } + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.publicUrl).parseHtml() + val series = doc.location() + val numChapters = Regex( + pattern = "&page=(.*)&lang=", + ).findAll( + input = doc.getElementsByTag("script")[3].toString(), + ) + .elementAt(3).destructured.component1() + .toInt() + val chapters = ArrayList() + for (i in 0..numChapters) { + val chapter = MangaChapter( + id = generateUid("$series&page=$i"), + name = "Chapter ${i + 1}", + number = i + 1, + url = "$series&page=$i", + scanlator = null, + branch = null, + uploadDate = 0L, + source = MangaSource.DUMMY, + ) + chapters.add(chapter) + } + return manga.copy(chapters = chapters) + } - override suspend fun getPages(chapter: MangaChapter): List { - val doc = context.httpGet(chapter.url.toAbsoluteUrl(getDomain())).parseHtml() - val imgUrl = doc.getElementsByClass("subsectionContainer")[0] - .selectFirst("img") - ?.attrAsAbsoluteUrlOrNull("src") ?: doc.parseFailed("Something broken") - return listOf( - MangaPage( - id = generateUid(imgUrl), - url = imgUrl, - referer = imgUrl, - preview = null, - source = MangaSource.DUMMY, - ), - ) - } + override suspend fun getPages(chapter: MangaChapter): List { + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + val imgUrl = doc.getElementsByClass("subsectionContainer")[0] + .selectFirst("img") + ?.attrAsAbsoluteUrlOrNull("src") ?: doc.parseFailed("Something broken") + return listOf( + MangaPage( + id = generateUid(imgUrl), + url = imgUrl, + referer = imgUrl, + preview = null, + source = MangaSource.DUMMY, + ), + ) + } - override suspend fun getTags(): Set { - return emptySet() - } -} \ No newline at end of file + override suspend fun getTags(): Set { + return emptySet() + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ComickFunParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ComickFunParser.kt index ab7d471d..fdb94512 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ComickFunParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ComickFunParser.kt @@ -22,195 +22,195 @@ private const val PAGE_SIZE = 20 private const val CHAPTERS_LIMIT = 99999 @MangaSourceParser("COMICK_FUN", "ComicK") -internal class ComickFunParser(override val context: MangaLoaderContext) : MangaParser(MangaSource.COMICK_FUN) { +internal class ComickFunParser(context: MangaLoaderContext) : MangaParser(context, MangaSource.COMICK_FUN) { - override val configKeyDomain = ConfigKey.Domain("comick.app", null) + override val configKeyDomain = ConfigKey.Domain("comick.app", null) - override val sortOrders: Set = EnumSet.of( - SortOrder.POPULARITY, - SortOrder.UPDATED, - SortOrder.RATING, - ) + override val sortOrders: Set = EnumSet.of( + SortOrder.POPULARITY, + SortOrder.UPDATED, + SortOrder.RATING, + ) - @Volatile - private var cachedTags: SparseArrayCompat? = null + @Volatile + private var cachedTags: SparseArrayCompat? = null - override suspend fun getList( - offset: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - val domain = getDomain() - val url = buildString { - append("https://api.") - append(domain) - append("/search?tachiyomi=true") - if (!query.isNullOrEmpty()) { - if (offset > 0) { - return emptyList() - } - append("&q=") - append(query.urlEncoded()) - } else { - append("&limit=") - append(PAGE_SIZE) - append("&page=") - append((offset / PAGE_SIZE) + 1) - if (!tags.isNullOrEmpty()) { - append("&genres=") - appendAll(tags, "&genres=", MangaTag::key) - } - append("&sort=") // view, uploaded, rating, follow, user_follow_count - append( - when (sortOrder) { - SortOrder.POPULARITY -> "view" - SortOrder.RATING -> "rating" - else -> "uploaded" - }, - ) - } - } - val ja = context.httpGet(url).parseJsonArray() - val tagsMap = cachedTags ?: loadTags() - return ja.mapJSON { jo -> - val slug = jo.getString("slug") - Manga( - id = generateUid(slug), - title = jo.getString("title"), - altTitle = null, - url = slug, - publicUrl = "https://$domain/comic/$slug", - rating = jo.getDoubleOrDefault("rating", -10.0).toFloat() / 10f, - isNsfw = false, - coverUrl = jo.getString("cover_url"), - largeCoverUrl = null, - description = jo.getStringOrNull("desc"), - tags = jo.selectGenres("genres", tagsMap), - state = runCatching { - if (jo.getBoolean("translation_completed")) { - MangaState.FINISHED - } else { - MangaState.ONGOING - } - }.getOrNull(), - author = null, - source = source, - ) - } - } + override suspend fun getList( + offset: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + val domain = domain + val url = buildString { + append("https://api.") + append(domain) + append("/search?tachiyomi=true") + if (!query.isNullOrEmpty()) { + if (offset > 0) { + return emptyList() + } + append("&q=") + append(query.urlEncoded()) + } else { + append("&limit=") + append(PAGE_SIZE) + append("&page=") + append((offset / PAGE_SIZE) + 1) + if (!tags.isNullOrEmpty()) { + append("&genres=") + appendAll(tags, "&genres=", MangaTag::key) + } + append("&sort=") // view, uploaded, rating, follow, user_follow_count + append( + when (sortOrder) { + SortOrder.POPULARITY -> "view" + SortOrder.RATING -> "rating" + else -> "uploaded" + }, + ) + } + } + val ja = webClient.httpGet(url).parseJsonArray() + val tagsMap = cachedTags ?: loadTags() + return ja.mapJSON { jo -> + val slug = jo.getString("slug") + Manga( + id = generateUid(slug), + title = jo.getString("title"), + altTitle = null, + url = slug, + publicUrl = "https://$domain/comic/$slug", + rating = jo.getDoubleOrDefault("rating", -10.0).toFloat() / 10f, + isNsfw = false, + coverUrl = jo.getString("cover_url"), + largeCoverUrl = null, + description = jo.getStringOrNull("desc"), + tags = jo.selectGenres("genres", tagsMap), + state = runCatching { + if (jo.getBoolean("translation_completed")) { + MangaState.FINISHED + } else { + MangaState.ONGOING + } + }.getOrNull(), + author = null, + source = source, + ) + } + } - override suspend fun getDetails(manga: Manga): Manga { - val domain = getDomain() - val url = "https://api.$domain/comic/${manga.url}?tachiyomi=true" - val jo = context.httpGet(url).parseJson() - val comic = jo.getJSONObject("comic") - return manga.copy( - title = comic.getString("title"), - altTitle = null, // TODO - isNsfw = jo.getBoolean("matureContent") || comic.getBoolean("hentai"), - description = comic.getStringOrNull("parsed") ?: comic.getString("desc"), - tags = manga.tags + jo.getJSONArray("genres").mapJSONToSet { - MangaTag( - title = it.getString("name"), - key = it.getString("slug"), - source = source, - ) - }, - author = jo.getJSONArray("artists").optJSONObject(0)?.getString("name"), - chapters = getChapters(comic.getLong("id")), - ) - } + override suspend fun getDetails(manga: Manga): Manga { + val domain = domain + val url = "https://api.$domain/comic/${manga.url}?tachiyomi=true" + val jo = webClient.httpGet(url).parseJson() + val comic = jo.getJSONObject("comic") + return manga.copy( + title = comic.getString("title"), + altTitle = null, // TODO + isNsfw = jo.getBoolean("matureContent") || comic.getBoolean("hentai"), + description = comic.getStringOrNull("parsed") ?: comic.getString("desc"), + tags = manga.tags + jo.getJSONArray("genres").mapJSONToSet { + MangaTag( + title = it.getString("name"), + key = it.getString("slug"), + source = source, + ) + }, + author = jo.getJSONArray("artists").optJSONObject(0)?.getString("name"), + chapters = getChapters(comic.getLong("id")), + ) + } - override suspend fun getPages(chapter: MangaChapter): List { - val jo = context.httpGet( - "https://api.${getDomain()}/chapter/${chapter.url}?tachiyomi=true", - ).parseJson().getJSONObject("chapter") - val referer = "https://${getDomain()}/" - return jo.getJSONArray("images").mapJSON { - val url = it.getString("url") - MangaPage( - id = generateUid(url), - url = url, - referer = referer, - preview = null, - source = source, - ) - } - } + override suspend fun getPages(chapter: MangaChapter): List { + val jo = webClient.httpGet( + "https://api.${domain}/chapter/${chapter.url}?tachiyomi=true", + ).parseJson().getJSONObject("chapter") + val referer = "https://${domain}/" + return jo.getJSONArray("images").mapJSON { + val url = it.getString("url") + MangaPage( + id = generateUid(url), + url = url, + referer = referer, + preview = null, + source = source, + ) + } + } - override suspend fun getTags(): Set { - val sparseArray = cachedTags ?: loadTags() - val set = ArraySet(sparseArray.size()) - for (i in 0 until sparseArray.size()) { - set.add(sparseArray.valueAt(i)) - } - return set - } + override suspend fun getTags(): Set { + val sparseArray = cachedTags ?: loadTags() + val set = ArraySet(sparseArray.size()) + for (i in 0 until sparseArray.size()) { + set.add(sparseArray.valueAt(i)) + } + return set + } - private suspend fun loadTags(): SparseArrayCompat { - val ja = context.httpGet("https://api.${getDomain()}/genre").parseJsonArray() - val tags = SparseArrayCompat(ja.length()) - for (jo in ja.JSONIterator()) { - tags.append( - jo.getInt("id"), - MangaTag( - title = jo.getString("name"), - key = jo.getString("slug"), - source = source, - ), - ) - } - cachedTags = tags - return tags - } + private suspend fun loadTags(): SparseArrayCompat { + val ja = webClient.httpGet("https://api.${domain}/genre").parseJsonArray() + val tags = SparseArrayCompat(ja.length()) + for (jo in ja.JSONIterator()) { + tags.append( + jo.getInt("id"), + MangaTag( + title = jo.getString("name"), + key = jo.getString("slug"), + source = source, + ), + ) + } + cachedTags = tags + return tags + } - private suspend fun getChapters(id: Long): List { - val ja = context.httpGet( - url = "https://api.${getDomain()}/comic/$id/chapter?tachiyomi=true&limit=$CHAPTERS_LIMIT", - ).parseJson().getJSONArray("chapters") - val dateFormat = SimpleDateFormat("yyyy-MM-dd") - val counters = HashMap() - return ja.mapReversed { jo -> - val locale = Locale.forLanguageTag(jo.getString("lang")) - var number = counters[locale] ?: 0 - number++ - counters[locale] = number - MangaChapter( - id = generateUid(jo.getLong("id")), - name = buildString { - jo.getStringOrNull("vol")?.let { append("Vol ").append(it).append(' ') } - jo.getStringOrNull("chap")?.let { append("Chap ").append(it) } - jo.getStringOrNull("title")?.let { append(": ").append(it) } - }, - number = number, - url = jo.getString("hid"), - scanlator = jo.optJSONArray("group_name")?.optString(0), - uploadDate = dateFormat.tryParse(jo.getString("created_at").substringBefore('T')), - branch = locale.getDisplayName(locale).toTitleCase(locale), - source = source, - ) - } - } + private suspend fun getChapters(id: Long): List { + val ja = webClient.httpGet( + url = "https://api.${domain}/comic/$id/chapter?tachiyomi=true&limit=$CHAPTERS_LIMIT", + ).parseJson().getJSONArray("chapters") + val dateFormat = SimpleDateFormat("yyyy-MM-dd") + val counters = HashMap() + return ja.mapReversed { jo -> + val locale = Locale.forLanguageTag(jo.getString("lang")) + var number = counters[locale] ?: 0 + number++ + counters[locale] = number + MangaChapter( + id = generateUid(jo.getLong("id")), + name = buildString { + jo.getStringOrNull("vol")?.let { append("Vol ").append(it).append(' ') } + jo.getStringOrNull("chap")?.let { append("Chap ").append(it) } + jo.getStringOrNull("title")?.let { append(": ").append(it) } + }, + number = number, + url = jo.getString("hid"), + scanlator = jo.optJSONArray("group_name")?.optString(0), + uploadDate = dateFormat.tryParse(jo.getString("created_at").substringBefore('T')), + branch = locale.getDisplayName(locale).toTitleCase(locale), + source = source, + ) + } + } - private inline fun JSONArray.mapReversed(block: (JSONObject) -> R): List { - val len = length() - val destination = ArrayList(len) - for (i in (0 until len).reversed()) { - val jo = getJSONObject(i) - destination.add(block(jo)) - } - return destination - } + private inline fun JSONArray.mapReversed(block: (JSONObject) -> R): List { + val len = length() + val destination = ArrayList(len) + for (i in (0 until len).reversed()) { + val jo = getJSONObject(i) + destination.add(block(jo)) + } + return destination + } - private fun JSONObject.selectGenres(name: String, tags: SparseArrayCompat): Set { - val array = optJSONArray(name) ?: return emptySet() - val res = ArraySet(array.length()) - for (i in 0 until array.length()) { - val id = array.getInt(i) - val tag = tags.get(id) ?: continue - res.add(tag) - } - return res - } -} \ No newline at end of file + private fun JSONObject.selectGenres(name: String, tags: SparseArrayCompat): Set { + val array = optJSONArray(name) ?: return emptySet() + val res = ArraySet(array.length()) + for (i in 0 until array.length()) { + val id = array.getInt(i) + val tag = tags.get(id) ?: continue + res.add(tag) + } + return res + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/DesuMeParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/DesuMeParser.kt index 3efe8716..3b76baed 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/DesuMeParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/DesuMeParser.kt @@ -13,149 +13,149 @@ import org.koitharu.kotatsu.parsers.util.json.mapJSONToSet import java.util.* @MangaSourceParser("DESUME", "Desu.me", "ru") -internal class DesuMeParser(override val context: MangaLoaderContext) : PagedMangaParser(MangaSource.DESUME, 20) { +internal class DesuMeParser(context: MangaLoaderContext) : PagedMangaParser(context, MangaSource.DESUME, 20) { - override val configKeyDomain = ConfigKey.Domain("desu.me", null) + override val configKeyDomain = ConfigKey.Domain("desu.me", null) - override val sortOrders: Set = EnumSet.of( - SortOrder.UPDATED, - SortOrder.POPULARITY, - SortOrder.NEWEST, - SortOrder.ALPHABETICAL, - ) + override val sortOrders: Set = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.NEWEST, + SortOrder.ALPHABETICAL, + ) - override suspend fun getListPage( - page: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - if (query != null && page != searchPaginator.firstPage) { - return emptyList() - } - val domain = getDomain() - val url = buildString { - append("https://") - append(domain) - append("/manga/api/?limit=20&order=") - append(getSortKey(sortOrder)) - append("&page=") - append(page) - if (!tags.isNullOrEmpty()) { - append("&genres=") - appendAll(tags, ",") { it.key } - } - if (query != null) { - append("&search=") - append(query) - } - } - val json = context.httpGet(url).parseJson().getJSONArray("response") - ?: throw ParseException("Invalid response", url) - val total = json.length() - val list = ArrayList(total) - for (i in 0 until total) { - val jo = json.getJSONObject(i) - val cover = jo.getJSONObject("image") - val id = jo.getLong("id") - list += Manga( - url = "/manga/api/$id", - publicUrl = jo.getString("url"), - source = MangaSource.DESUME, - title = jo.getString("russian"), - altTitle = jo.getString("name"), - coverUrl = cover.getString("preview"), - largeCoverUrl = cover.getString("original"), - state = when { - jo.getInt("ongoing") == 1 -> MangaState.ONGOING - else -> null - }, - rating = jo.getDouble("score").toFloat().coerceIn(0f, 1f), - id = generateUid(id), - isNsfw = false, - tags = emptySet(), - author = null, - description = jo.getString("description"), - ) - } - return list - } + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + if (query != null && page != searchPaginator.firstPage) { + return emptyList() + } + val domain = domain + val url = buildString { + append("https://") + append(domain) + append("/manga/api/?limit=20&order=") + append(getSortKey(sortOrder)) + append("&page=") + append(page) + if (!tags.isNullOrEmpty()) { + append("&genres=") + appendAll(tags, ",") { it.key } + } + if (query != null) { + append("&search=") + append(query) + } + } + val json = webClient.httpGet(url).parseJson().getJSONArray("response") + ?: throw ParseException("Invalid response", url) + val total = json.length() + val list = ArrayList(total) + for (i in 0 until total) { + val jo = json.getJSONObject(i) + val cover = jo.getJSONObject("image") + val id = jo.getLong("id") + list += Manga( + url = "/manga/api/$id", + publicUrl = jo.getString("url"), + source = MangaSource.DESUME, + title = jo.getString("russian"), + altTitle = jo.getString("name"), + coverUrl = cover.getString("preview"), + largeCoverUrl = cover.getString("original"), + state = when { + jo.getInt("ongoing") == 1 -> MangaState.ONGOING + else -> null + }, + rating = jo.getDouble("score").toFloat().coerceIn(0f, 1f), + id = generateUid(id), + isNsfw = false, + tags = emptySet(), + author = null, + description = jo.getString("description"), + ) + } + return list + } - override suspend fun getDetails(manga: Manga): Manga { - val url = manga.url.toAbsoluteUrl(getDomain()) - val json = context.httpGet(url).parseJson().getJSONObject("response") - ?: throw ParseException("Invalid response", url) - val baseChapterUrl = manga.url + "/chapter/" - val chaptersList = json.getJSONObject("chapters").getJSONArray("list") - val totalChapters = chaptersList.length() - return manga.copy( - tags = json.getJSONArray("genres").mapJSONToSet { - MangaTag( - key = it.getString("text"), - title = it.getString("russian").toTitleCase(), - source = manga.source, - ) - }, - publicUrl = json.getString("url"), - description = json.getString("description"), - chapters = chaptersList.mapJSONIndexed { i, it -> - val chid = it.getLong("id") - val volChap = "Том " + it.optString("vol", "0") + ". " + "Глава " + it.optString("ch", "0") - val title = it.optString("title", "null").takeUnless { it == "null" } - MangaChapter( - id = generateUid(chid), - source = manga.source, - url = "$baseChapterUrl$chid", - uploadDate = it.getLong("date") * 1000, - name = if (title.isNullOrEmpty()) volChap else "$volChap: $title", - number = totalChapters - i, - scanlator = null, - branch = null, - ) - }.reversed(), - ) - } + override suspend fun getDetails(manga: Manga): Manga { + val url = manga.url.toAbsoluteUrl(domain) + val json = webClient.httpGet(url).parseJson().getJSONObject("response") + ?: throw ParseException("Invalid response", url) + val baseChapterUrl = manga.url + "/chapter/" + val chaptersList = json.getJSONObject("chapters").getJSONArray("list") + val totalChapters = chaptersList.length() + return manga.copy( + tags = json.getJSONArray("genres").mapJSONToSet { + MangaTag( + key = it.getString("text"), + title = it.getString("russian").toTitleCase(), + source = manga.source, + ) + }, + publicUrl = json.getString("url"), + description = json.getString("description"), + chapters = chaptersList.mapJSONIndexed { i, it -> + val chid = it.getLong("id") + val volChap = "Том " + it.optString("vol", "0") + ". " + "Глава " + it.optString("ch", "0") + val title = it.optString("title", "null").takeUnless { it == "null" } + MangaChapter( + id = generateUid(chid), + source = manga.source, + url = "$baseChapterUrl$chid", + uploadDate = it.getLong("date") * 1000, + name = if (title.isNullOrEmpty()) volChap else "$volChap: $title", + number = totalChapters - i, + scanlator = null, + branch = null, + ) + }.reversed(), + ) + } - override suspend fun getPages(chapter: MangaChapter): List { - val fullUrl = chapter.url.toAbsoluteUrl(getDomain()) - val json = context.httpGet(fullUrl) - .parseJson() - .getJSONObject("response") ?: throw ParseException("Invalid response", fullUrl) - return json.getJSONObject("pages").getJSONArray("list").mapJSON { jo -> - MangaPage( - id = generateUid(jo.getLong("id")), - referer = fullUrl, - preview = null, - source = chapter.source, - url = jo.getString("img"), - ) - } - } + override suspend fun getPages(chapter: MangaChapter): List { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val json = webClient.httpGet(fullUrl) + .parseJson() + .getJSONObject("response") ?: throw ParseException("Invalid response", fullUrl) + return json.getJSONObject("pages").getJSONArray("list").mapJSON { jo -> + MangaPage( + id = generateUid(jo.getLong("id")), + referer = fullUrl, + preview = null, + source = chapter.source, + url = jo.getString("img"), + ) + } + } - override suspend fun getTags(): Set { - val doc = context.httpGet("https://${getDomain()}/manga/").parseHtml() - val root = doc.body().requireElementById("animeFilter") - .selectFirstOrThrow(".catalog-genres") - return root.select("li").mapToSet { - val input = it.selectFirstOrThrow("input") - MangaTag( - source = source, - key = input.attr("data-genre-slug").ifEmpty { - it.parseFailed("data-genre-slug is empty") - }, - title = input.attr("data-genre-name").toTitleCase().ifEmpty { - it.parseFailed("data-genre-name is empty") - }, - ) - } - } + override suspend fun getTags(): Set { + val doc = webClient.httpGet("https://${domain}/manga/").parseHtml() + val root = doc.body().requireElementById("animeFilter") + .selectFirstOrThrow(".catalog-genres") + return root.select("li").mapToSet { + val input = it.selectFirstOrThrow("input") + MangaTag( + source = source, + key = input.attr("data-genre-slug").ifEmpty { + it.parseFailed("data-genre-slug is empty") + }, + title = input.attr("data-genre-name").toTitleCase().ifEmpty { + it.parseFailed("data-genre-name is empty") + }, + ) + } + } - private fun getSortKey(sortOrder: SortOrder) = - when (sortOrder) { - SortOrder.ALPHABETICAL -> "name" - SortOrder.POPULARITY -> "popular" - SortOrder.UPDATED -> "updated" - SortOrder.NEWEST -> "id" - else -> "updated" - } -} \ No newline at end of file + private fun getSortKey(sortOrder: SortOrder) = + when (sortOrder) { + SortOrder.ALPHABETICAL -> "name" + SortOrder.POPULARITY -> "popular" + SortOrder.UPDATED -> "updated" + SortOrder.NEWEST -> "id" + else -> "updated" + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ExHentaiParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ExHentaiParser.kt index 6432667f..86be7938 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ExHentaiParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ExHentaiParser.kt @@ -20,289 +20,289 @@ private const val DOMAIN_AUTHORIZED = "exhentai.org" @MangaSourceParser("EXHENTAI", "ExHentai") internal class ExHentaiParser( - override val context: MangaLoaderContext, -) : PagedMangaParser(MangaSource.EXHENTAI, pageSize = 25), MangaParserAuthProvider { + context: MangaLoaderContext, +) : PagedMangaParser(context, MangaSource.EXHENTAI, pageSize = 25), MangaParserAuthProvider { - override val sortOrders: Set = Collections.singleton( - SortOrder.NEWEST, - ) + override val sortOrders: Set = Collections.singleton( + SortOrder.NEWEST, + ) - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain(if (isAuthorized) DOMAIN_AUTHORIZED else DOMAIN_UNAUTHORIZED, null) + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain(if (isAuthorized) DOMAIN_AUTHORIZED else DOMAIN_UNAUTHORIZED, null) - override val authUrl: String - get() = "https://${getDomain()}/bounce_login.php" + override val authUrl: String + get() = "https://${domain}/bounce_login.php" - private val ratingPattern = Regex("-?[0-9]+px") - private val authCookies = arrayOf("ipb_member_id", "ipb_pass_hash") - private var updateDm = false - private val nextPages = SparseArrayCompat() - private val suspiciousContentKey = ConfigKey.ShowSuspiciousContent(true) + private val ratingPattern = Regex("-?[0-9]+px") + private val authCookies = arrayOf("ipb_member_id", "ipb_pass_hash") + private var updateDm = false + private val nextPages = SparseArrayCompat() + private val suspiciousContentKey = ConfigKey.ShowSuspiciousContent(true) - override val isAuthorized: Boolean - get() { - val authorized = isAuthorized(DOMAIN_UNAUTHORIZED) - if (authorized) { - if (!isAuthorized(DOMAIN_AUTHORIZED)) { - context.cookieJar.copyCookies( - DOMAIN_UNAUTHORIZED, - DOMAIN_AUTHORIZED, - authCookies, - ) - context.cookieJar.insertCookies(DOMAIN_AUTHORIZED, "yay=louder") - } - return true - } - return false - } + override val isAuthorized: Boolean + get() { + val authorized = isAuthorized(DOMAIN_UNAUTHORIZED) + if (authorized) { + if (!isAuthorized(DOMAIN_AUTHORIZED)) { + context.cookieJar.copyCookies( + DOMAIN_UNAUTHORIZED, + DOMAIN_AUTHORIZED, + authCookies, + ) + context.cookieJar.insertCookies(DOMAIN_AUTHORIZED, "yay=louder") + } + return true + } + return false + } - init { - context.cookieJar.insertCookies(DOMAIN_AUTHORIZED, "nw=1", "sl=dm_2") - context.cookieJar.insertCookies(DOMAIN_UNAUTHORIZED, "nw=1", "sl=dm_2") - paginator.firstPage = 0 - } + init { + context.cookieJar.insertCookies(DOMAIN_AUTHORIZED, "nw=1", "sl=dm_2") + context.cookieJar.insertCookies(DOMAIN_UNAUTHORIZED, "nw=1", "sl=dm_2") + paginator.firstPage = 0 + } - override suspend fun getListPage( - page: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - var search = query?.urlEncoded().orEmpty() - val next = nextPages.get(page, 0L) - if (page > 0 && next == 0L) { - assert(false) { "Page timestamp not found" } - return emptyList() - } - val url = buildString { - append("https://") - append(getDomain()) - append("/?next=") - append(next) - if (!tags.isNullOrEmpty()) { - var fCats = 0 - for (tag in tags) { - tag.key.toIntOrNull()?.let { fCats = fCats or it } ?: run { - search += tag.key + " " - } - } - if (fCats != 0) { - append("&f_cats=") - append(1023 - fCats) - } - } - if (search.isNotEmpty()) { - append("&f_search=") - append(search.trim().replace(' ', '+')) - } - // by unknown reason cookie "sl=dm_2" is ignored, so, we should request it again - if (updateDm) { - append("&inline_set=dm_e") - } - append("&advsearch=1") - if (config[suspiciousContentKey]) { - append("&f_sh=on") - } - } - val body = context.httpGet(url).parseHtml().body() - val root = body.selectFirst("table.itg") - ?.selectFirst("tbody") - ?: if (updateDm) { - body.parseFailed("Cannot find root") - } else { - updateDm = true - return getListPage(page, query, tags, sortOrder) - } - updateDm = false - nextPages[page + 1] = getNextTimestamp(body) - return root.children().mapNotNull { tr -> - if (tr.childrenSize() != 2) return@mapNotNull null - val (td1, td2) = tr.children() - val glink = td2.selectFirstOrThrow("div.glink") - val a = glink.parents().select("a").first() ?: glink.parseFailed("link not found") - val href = a.attrAsRelativeUrl("href") - val tagsDiv = glink.nextElementSibling() ?: glink.parseFailed("tags div not found") - val mainTag = td2.selectFirst("div.cn")?.let { div -> - MangaTag( - title = div.text().toTitleCase(), - key = tagIdByClass(div.classNames()) ?: return@let null, - source = source, - ) - } - Manga( - id = generateUid(href), - title = glink.text().cleanupTitle(), - altTitle = null, - url = href, - publicUrl = a.absUrl("href"), - rating = td2.selectFirst("div.ir")?.parseRating() ?: RATING_UNKNOWN, - isNsfw = true, - coverUrl = td1.selectFirst("img")?.absUrl("src").orEmpty(), - tags = setOfNotNull(mainTag), - state = null, - author = tagsDiv.getElementsContainingOwnText("artist:").first() - ?.nextElementSibling()?.text(), - source = source, - ) - } - } + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + var search = query?.urlEncoded().orEmpty() + val next = nextPages.get(page, 0L) + if (page > 0 && next == 0L) { + assert(false) { "Page timestamp not found" } + return emptyList() + } + val url = buildString { + append("https://") + append(domain) + append("/?next=") + append(next) + if (!tags.isNullOrEmpty()) { + var fCats = 0 + for (tag in tags) { + tag.key.toIntOrNull()?.let { fCats = fCats or it } ?: run { + search += tag.key + " " + } + } + if (fCats != 0) { + append("&f_cats=") + append(1023 - fCats) + } + } + if (search.isNotEmpty()) { + append("&f_search=") + append(search.trim().replace(' ', '+')) + } + // by unknown reason cookie "sl=dm_2" is ignored, so, we should request it again + if (updateDm) { + append("&inline_set=dm_e") + } + append("&advsearch=1") + if (config[suspiciousContentKey]) { + append("&f_sh=on") + } + } + val body = webClient.httpGet(url).parseHtml().body() + val root = body.selectFirst("table.itg") + ?.selectFirst("tbody") + ?: if (updateDm) { + body.parseFailed("Cannot find root") + } else { + updateDm = true + return getListPage(page, query, tags, sortOrder) + } + updateDm = false + nextPages[page + 1] = getNextTimestamp(body) + return root.children().mapNotNull { tr -> + if (tr.childrenSize() != 2) return@mapNotNull null + val (td1, td2) = tr.children() + val glink = td2.selectFirstOrThrow("div.glink") + val a = glink.parents().select("a").first() ?: glink.parseFailed("link not found") + val href = a.attrAsRelativeUrl("href") + val tagsDiv = glink.nextElementSibling() ?: glink.parseFailed("tags div not found") + val mainTag = td2.selectFirst("div.cn")?.let { div -> + MangaTag( + title = div.text().toTitleCase(), + key = tagIdByClass(div.classNames()) ?: return@let null, + source = source, + ) + } + Manga( + id = generateUid(href), + title = glink.text().cleanupTitle(), + altTitle = null, + url = href, + publicUrl = a.absUrl("href"), + rating = td2.selectFirst("div.ir")?.parseRating() ?: RATING_UNKNOWN, + isNsfw = true, + coverUrl = td1.selectFirst("img")?.absUrl("src").orEmpty(), + tags = setOfNotNull(mainTag), + state = null, + author = tagsDiv.getElementsContainingOwnText("artist:").first() + ?.nextElementSibling()?.text(), + source = source, + ) + } + } - override suspend fun getDetails(manga: Manga): Manga { - val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml() - val root = doc.body().selectFirstOrThrow("div.gm") - val cover = root.getElementById("gd1")?.children()?.first() - val title = root.getElementById("gd2") - val taglist = root.getElementById("taglist") - val tabs = doc.body().selectFirst("table.ptt")?.selectFirst("tr") - return manga.copy( - title = title?.getElementById("gn")?.text()?.cleanupTitle() ?: manga.title, - altTitle = title?.getElementById("gj")?.text()?.cleanupTitle() ?: manga.altTitle, - publicUrl = doc.baseUri().ifEmpty { manga.publicUrl }, - rating = root.getElementById("rating_label")?.text() - ?.substringAfterLast(' ') - ?.toFloatOrNull() - ?.div(5f) ?: manga.rating, - largeCoverUrl = cover?.styleValueOrNull("background")?.cssUrl(), - description = taglist?.select("tr")?.joinToString("
") { tr -> - val (tc, td) = tr.children() - val subtags = td.select("a").joinToString { it.html() } - "${tc.html()} $subtags" - }, - chapters = tabs?.select("a")?.findLast { a -> - a.text().toIntOrNull() != null - }?.let { a -> - val count = a.text().toInt() - val chapters = ChaptersListBuilder(count) - for (i in 1..count) { - val url = "${manga.url}?p=${i - 1}" - chapters += MangaChapter( - id = generateUid(url), - name = "${manga.title} #$i", - number = i, - url = url, - uploadDate = 0L, - source = source, - scanlator = null, - branch = null, - ) - } - chapters.toList() - }, - ) - } + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val root = doc.body().selectFirstOrThrow("div.gm") + val cover = root.getElementById("gd1")?.children()?.first() + val title = root.getElementById("gd2") + val taglist = root.getElementById("taglist") + val tabs = doc.body().selectFirst("table.ptt")?.selectFirst("tr") + return manga.copy( + title = title?.getElementById("gn")?.text()?.cleanupTitle() ?: manga.title, + altTitle = title?.getElementById("gj")?.text()?.cleanupTitle() ?: manga.altTitle, + publicUrl = doc.baseUri().ifEmpty { manga.publicUrl }, + rating = root.getElementById("rating_label")?.text() + ?.substringAfterLast(' ') + ?.toFloatOrNull() + ?.div(5f) ?: manga.rating, + largeCoverUrl = cover?.styleValueOrNull("background")?.cssUrl(), + description = taglist?.select("tr")?.joinToString("
") { tr -> + val (tc, td) = tr.children() + val subtags = td.select("a").joinToString { it.html() } + "${tc.html()} $subtags" + }, + chapters = tabs?.select("a")?.findLast { a -> + a.text().toIntOrNull() != null + }?.let { a -> + val count = a.text().toInt() + val chapters = ChaptersListBuilder(count) + for (i in 1..count) { + val url = "${manga.url}?p=${i - 1}" + chapters += MangaChapter( + id = generateUid(url), + name = "${manga.title} #$i", + number = i, + url = url, + uploadDate = 0L, + source = source, + scanlator = null, + branch = null, + ) + } + chapters.toList() + }, + ) + } - override suspend fun getPages(chapter: MangaChapter): List { - val doc = context.httpGet(chapter.url.toAbsoluteUrl(getDomain())).parseHtml() - val root = doc.body().requireElementById("gdt") - return root.select("a").map { a -> - val url = a.attrAsRelativeUrl("href") - MangaPage( - id = generateUid(url), - url = url, - referer = a.absUrl("href"), - preview = null, - source = source, - ) - } - } + override suspend fun getPages(chapter: MangaChapter): List { + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + val root = doc.body().requireElementById("gdt") + return root.select("a").map { a -> + val url = a.attrAsRelativeUrl("href") + MangaPage( + id = generateUid(url), + url = url, + referer = a.absUrl("href"), + preview = null, + source = source, + ) + } + } - override suspend fun getPageUrl(page: MangaPage): String { - val doc = context.httpGet(page.url.toAbsoluteUrl(getDomain())).parseHtml() - return doc.body().requireElementById("img").attrAsAbsoluteUrl("src") - } + override suspend fun getPageUrl(page: MangaPage): String { + val doc = webClient.httpGet(page.url.toAbsoluteUrl(domain)).parseHtml() + return doc.body().requireElementById("img").attrAsAbsoluteUrl("src") + } - override suspend fun getTags(): Set { - val doc = context.httpGet("https://${getDomain()}").parseHtml() - val root = doc.body().requireElementById("searchbox").selectFirstOrThrow("table") - return root.select("div.cs").mapNotNullToSet { div -> - val id = div.id().substringAfterLast('_').toIntOrNull() - ?: return@mapNotNullToSet null - MangaTag( - title = div.text().toTitleCase(), - key = id.toString(), - source = source, - ) - } - } + override suspend fun getTags(): Set { + val doc = webClient.httpGet("https://${domain}").parseHtml() + val root = doc.body().requireElementById("searchbox").selectFirstOrThrow("table") + return root.select("div.cs").mapNotNullToSet { div -> + val id = div.id().substringAfterLast('_').toIntOrNull() + ?: return@mapNotNullToSet null + MangaTag( + title = div.text().toTitleCase(), + key = id.toString(), + source = source, + ) + } + } - override suspend fun getUsername(): String { - val doc = context.httpGet("https://forums.$DOMAIN_UNAUTHORIZED/").parseHtml().body() - val username = doc.getElementById("userlinks") - ?.getElementsByAttributeValueContaining("href", "showuser=") - ?.firstOrNull() - ?.ownText() - ?: if (doc.getElementById("userlinksguest") != null) { - throw AuthRequiredException(source) - } else { - doc.parseFailed() - } - return username - } + override suspend fun getUsername(): String { + val doc = webClient.httpGet("https://forums.$DOMAIN_UNAUTHORIZED/").parseHtml().body() + val username = doc.getElementById("userlinks") + ?.getElementsByAttributeValueContaining("href", "showuser=") + ?.firstOrNull() + ?.ownText() + ?: if (doc.getElementById("userlinksguest") != null) { + throw AuthRequiredException(source) + } else { + doc.parseFailed() + } + return username + } - override fun onCreateConfig(keys: MutableCollection>) { - super.onCreateConfig(keys) - keys.add(suspiciousContentKey) - } + override fun onCreateConfig(keys: MutableCollection>) { + super.onCreateConfig(keys) + keys.add(suspiciousContentKey) + } - private fun isAuthorized(domain: String): Boolean { - val cookies = context.cookieJar.getCookies(domain).mapToSet { x -> x.name } - return authCookies.all { it in cookies } - } + private fun isAuthorized(domain: String): Boolean { + val cookies = context.cookieJar.getCookies(domain).mapToSet { x -> x.name } + return authCookies.all { it in cookies } + } - private fun Element.parseRating(): Float { - return runCatching { - val style = requireNotNull(attr("style")) - val (v1, v2) = ratingPattern.find(style)!!.destructured - var p1 = v1.dropLast(2).toInt() - val p2 = v2.dropLast(2).toInt() - if (p2 != -1) { - p1 += 8 - } - (80 - p1) / 80f - }.getOrDefault(RATING_UNKNOWN) - } + private fun Element.parseRating(): Float { + return runCatching { + val style = requireNotNull(attr("style")) + val (v1, v2) = ratingPattern.find(style)!!.destructured + var p1 = v1.dropLast(2).toInt() + val p2 = v2.dropLast(2).toInt() + if (p2 != -1) { + p1 += 8 + } + (80 - p1) / 80f + }.getOrDefault(RATING_UNKNOWN) + } - private fun String.cleanupTitle(): String { - val result = StringBuilder(length) - var skip = false - for (c in this) { - when { - c == '[' -> skip = true - c == ']' -> skip = false - c.isWhitespace() && result.isEmpty() -> continue - !skip -> result.append(c) - } - } - while (result.lastOrNull()?.isWhitespace() == true) { - result.deleteCharAt(result.lastIndex) - } - return result.toString() - } + private fun String.cleanupTitle(): String { + val result = StringBuilder(length) + var skip = false + for (c in this) { + when { + c == '[' -> skip = true + c == ']' -> skip = false + c.isWhitespace() && result.isEmpty() -> continue + !skip -> result.append(c) + } + } + while (result.lastOrNull()?.isWhitespace() == true) { + result.deleteCharAt(result.lastIndex) + } + return result.toString() + } - private fun String.cssUrl(): String? { - val fromIndex = indexOf("url(") - if (fromIndex == -1) { - return null - } - val toIndex = indexOf(')', startIndex = fromIndex) - return if (toIndex == -1) { - null - } else { - substring(fromIndex + 4, toIndex).trim() - } - } + private fun String.cssUrl(): String? { + val fromIndex = indexOf("url(") + if (fromIndex == -1) { + return null + } + val toIndex = indexOf(')', startIndex = fromIndex) + return if (toIndex == -1) { + null + } else { + substring(fromIndex + 4, toIndex).trim() + } + } - private fun tagIdByClass(classNames: Collection): String? { - val className = classNames.find { x -> x.startsWith("ct") } ?: return null - val num = className.drop(2).toIntOrNull(16) ?: return null - return 2.0.pow(num).toInt().toString() - } + private fun tagIdByClass(classNames: Collection): String? { + val className = classNames.find { x -> x.startsWith("ct") } ?: return null + val num = className.drop(2).toIntOrNull(16) ?: return null + return 2.0.pow(num).toInt().toString() + } - private fun getNextTimestamp(root: Element): Long { - return root.getElementById("unext") - ?.attrAsAbsoluteUrlOrNull("href") - ?.toHttpUrlOrNull() - ?.queryParameter("next") - ?.toLongOrNull() ?: 1 - } -} \ No newline at end of file + private fun getNextTimestamp(root: Element): Long { + return root.getElementById("unext") + ?.attrAsAbsoluteUrlOrNull("href") + ?.toHttpUrlOrNull() + ?.queryParameter("next") + ?.toLongOrNull() ?: 1 + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaDexParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaDexParser.kt index a5632c46..ed236b15 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaDexParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaDexParser.kt @@ -22,254 +22,254 @@ private const val CHAPTERS_MAX_PAGE_SIZE = 500 private const val CHAPTERS_PARALLELISM = 3 private const val CHAPTERS_MAX_COUNT = 10_000 // strange api behavior, looks like a bug private const val CONTENT_RATING = - "contentRating[]=safe&contentRating[]=suggestive&contentRating[]=erotica&contentRating[]=pornographic" + "contentRating[]=safe&contentRating[]=suggestive&contentRating[]=erotica&contentRating[]=pornographic" private const val LOCALE_FALLBACK = "en" @MangaSourceParser("MANGADEX", "MangaDex") -internal class MangaDexParser(override val context: MangaLoaderContext) : MangaParser(MangaSource.MANGADEX) { +internal class MangaDexParser(context: MangaLoaderContext) : MangaParser(context, MangaSource.MANGADEX) { - override val configKeyDomain = ConfigKey.Domain("mangadex.org", null) + override val configKeyDomain = ConfigKey.Domain("mangadex.org", null) - override val sortOrders: EnumSet = EnumSet.of( - SortOrder.UPDATED, - SortOrder.ALPHABETICAL, - SortOrder.NEWEST, - SortOrder.POPULARITY, - ) + override val sortOrders: EnumSet = EnumSet.of( + SortOrder.UPDATED, + SortOrder.ALPHABETICAL, + SortOrder.NEWEST, + SortOrder.POPULARITY, + ) - override suspend fun getList( - offset: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - val domain = getDomain() - val url = buildString { - append("https://api.") - append(domain) - append("/manga?limit=") - append(PAGE_SIZE) - append("&offset=") - append(offset) - append("&includes[]=cover_art&includes[]=author&includes[]=artist&") - tags?.forEach { tag -> - append("includedTags[]=") - append(tag.key) - append('&') - } - if (!query.isNullOrEmpty()) { - append("title=") - append(query.urlEncoded()) - append('&') - } - append(CONTENT_RATING) - append("&order") - append( - when (sortOrder) { - SortOrder.UPDATED, - -> "[latestUploadedChapter]=desc" + override suspend fun getList( + offset: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + val domain = domain + val url = buildString { + append("https://api.") + append(domain) + append("/manga?limit=") + append(PAGE_SIZE) + append("&offset=") + append(offset) + append("&includes[]=cover_art&includes[]=author&includes[]=artist&") + tags?.forEach { tag -> + append("includedTags[]=") + append(tag.key) + append('&') + } + if (!query.isNullOrEmpty()) { + append("title=") + append(query.urlEncoded()) + append('&') + } + append(CONTENT_RATING) + append("&order") + append( + when (sortOrder) { + SortOrder.UPDATED, + -> "[latestUploadedChapter]=desc" - SortOrder.ALPHABETICAL -> "[title]=asc" - SortOrder.NEWEST -> "[createdAt]=desc" - SortOrder.POPULARITY -> "[followedCount]=desc" - else -> "[followedCount]=desc" - }, - ) - } - val json = context.httpGet(url).parseJson().getJSONArray("data") - return json.mapJSON { jo -> - val id = jo.getString("id") - val attrs = jo.getJSONObject("attributes") - val relations = jo.getJSONArray("relationships").associateByKey("type") - val cover = relations["cover_art"] - ?.getJSONObject("attributes") - ?.getString("fileName") - ?.let { - "https://uploads.$domain/covers/$id/$it" - } - Manga( - id = generateUid(id), - title = requireNotNull(attrs.getJSONObject("title").selectByLocale()) { - "Title should not be null" - }, - altTitle = attrs.optJSONObject("altTitles")?.selectByLocale(), - url = id, - publicUrl = "https://$domain/title/$id", - rating = RATING_UNKNOWN, - isNsfw = attrs.getStringOrNull("contentRating") == "erotica", - coverUrl = cover?.plus(".256.jpg").orEmpty(), - largeCoverUrl = cover, - description = attrs.optJSONObject("description")?.selectByLocale(), - tags = attrs.getJSONArray("tags").mapJSONToSet { tag -> - MangaTag( - title = tag.getJSONObject("attributes") - .getJSONObject("name") - .firstStringValue() - .toTitleCase(), - key = tag.getString("id"), - source = source, - ) - }, - state = when (jo.getStringOrNull("status")) { - "ongoing" -> MangaState.ONGOING - "completed" -> MangaState.FINISHED - else -> null - }, - author = (relations["author"] ?: relations["artist"]) - ?.getJSONObject("attributes") - ?.getStringOrNull("name"), - source = source, - ) - } - } + SortOrder.ALPHABETICAL -> "[title]=asc" + SortOrder.NEWEST -> "[createdAt]=desc" + SortOrder.POPULARITY -> "[followedCount]=desc" + else -> "[followedCount]=desc" + }, + ) + } + val json = webClient.httpGet(url).parseJson().getJSONArray("data") + return json.mapJSON { jo -> + val id = jo.getString("id") + val attrs = jo.getJSONObject("attributes") + val relations = jo.getJSONArray("relationships").associateByKey("type") + val cover = relations["cover_art"] + ?.getJSONObject("attributes") + ?.getString("fileName") + ?.let { + "https://uploads.$domain/covers/$id/$it" + } + Manga( + id = generateUid(id), + title = requireNotNull(attrs.getJSONObject("title").selectByLocale()) { + "Title should not be null" + }, + altTitle = attrs.optJSONObject("altTitles")?.selectByLocale(), + url = id, + publicUrl = "https://$domain/title/$id", + rating = RATING_UNKNOWN, + isNsfw = attrs.getStringOrNull("contentRating") == "erotica", + coverUrl = cover?.plus(".256.jpg").orEmpty(), + largeCoverUrl = cover, + description = attrs.optJSONObject("description")?.selectByLocale(), + tags = attrs.getJSONArray("tags").mapJSONToSet { tag -> + MangaTag( + title = tag.getJSONObject("attributes") + .getJSONObject("name") + .firstStringValue() + .toTitleCase(), + key = tag.getString("id"), + source = source, + ) + }, + state = when (jo.getStringOrNull("status")) { + "ongoing" -> MangaState.ONGOING + "completed" -> MangaState.FINISHED + else -> null + }, + author = (relations["author"] ?: relations["artist"]) + ?.getJSONObject("attributes") + ?.getStringOrNull("name"), + source = source, + ) + } + } - override suspend fun getDetails(manga: Manga): Manga = coroutineScope { - val domain = getDomain() - val mangaId = manga.url.removePrefix("/") - val attrsDeferred = async { - context.httpGet( - "https://api.$domain/manga/${mangaId}?includes[]=artist&includes[]=author&includes[]=cover_art", - ).parseJson().getJSONObject("data").getJSONObject("attributes") - } - val feedDeferred = async { loadChapters(mangaId) } - val mangaAttrs = attrsDeferred.await() - val feed = feedDeferred.await() - // 2022-01-02T00:27:11+00:00 - val dateFormat = SimpleDateFormat( - "yyyy-MM-dd'T'HH:mm:ss'+00:00'", - Locale.ROOT, - ) - manga.copy( - description = mangaAttrs.optJSONObject("description")?.selectByLocale() - ?: manga.description, - chapters = feed.mapChapters { _, jo -> - val id = jo.getString("id") - val attrs = jo.getJSONObject("attributes") - if (!attrs.isNull("externalUrl")) { - return@mapChapters null - } - val locale = attrs.getStringOrNull("translatedLanguage")?.let { Locale.forLanguageTag(it) } - val relations = jo.getJSONArray("relationships").associateByKey("type") - val number = attrs.getIntOrDefault("chapter", 0) - MangaChapter( - id = generateUid(id), - name = attrs.getStringOrNull("title")?.takeUnless(String::isEmpty) - ?: "Chapter #$number", - number = number, - url = id, - scanlator = relations["scanlation_group"]?.getStringOrNull("name"), - uploadDate = dateFormat.tryParse(attrs.getString("publishAt")), - branch = locale?.getDisplayName(locale)?.toTitleCase(locale), - source = source, - ) - }, - ) - } + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { + val domain = domain + val mangaId = manga.url.removePrefix("/") + val attrsDeferred = async { + webClient.httpGet( + "https://api.$domain/manga/${mangaId}?includes[]=artist&includes[]=author&includes[]=cover_art", + ).parseJson().getJSONObject("data").getJSONObject("attributes") + } + val feedDeferred = async { loadChapters(mangaId) } + val mangaAttrs = attrsDeferred.await() + val feed = feedDeferred.await() + // 2022-01-02T00:27:11+00:00 + val dateFormat = SimpleDateFormat( + "yyyy-MM-dd'T'HH:mm:ss'+00:00'", + Locale.ROOT, + ) + manga.copy( + description = mangaAttrs.optJSONObject("description")?.selectByLocale() + ?: manga.description, + chapters = feed.mapChapters { _, jo -> + val id = jo.getString("id") + val attrs = jo.getJSONObject("attributes") + if (!attrs.isNull("externalUrl")) { + return@mapChapters null + } + val locale = attrs.getStringOrNull("translatedLanguage")?.let { Locale.forLanguageTag(it) } + val relations = jo.getJSONArray("relationships").associateByKey("type") + val number = attrs.getIntOrDefault("chapter", 0) + MangaChapter( + id = generateUid(id), + name = attrs.getStringOrNull("title")?.takeUnless(String::isEmpty) + ?: "Chapter #$number", + number = number, + url = id, + scanlator = relations["scanlation_group"]?.getStringOrNull("name"), + uploadDate = dateFormat.tryParse(attrs.getString("publishAt")), + branch = locale?.getDisplayName(locale)?.toTitleCase(locale), + source = source, + ) + }, + ) + } - override suspend fun getPages(chapter: MangaChapter): List { - val domain = getDomain() - val chapterJson = context.httpGet("https://api.$domain/at-home/server/${chapter.url}?forcePort443=false") - .parseJson() - .getJSONObject("chapter") - val pages = chapterJson.getJSONArray("data") - val prefix = "https://uploads.$domain/data/${chapterJson.getString("hash")}/" - val referer = "https://$domain/" - return List(pages.length()) { i -> - val url = prefix + pages.getString(i) - MangaPage( - id = generateUid(url), - url = url, - referer = referer, - preview = null, // TODO prefix + dataSaver.getString(i), - source = source, - ) - } - } + override suspend fun getPages(chapter: MangaChapter): List { + val domain = domain + val chapterJson = webClient.httpGet("https://api.$domain/at-home/server/${chapter.url}?forcePort443=false") + .parseJson() + .getJSONObject("chapter") + val pages = chapterJson.getJSONArray("data") + val prefix = "https://uploads.$domain/data/${chapterJson.getString("hash")}/" + val referer = "https://$domain/" + return List(pages.length()) { i -> + val url = prefix + pages.getString(i) + MangaPage( + id = generateUid(url), + url = url, + referer = referer, + preview = null, // TODO prefix + dataSaver.getString(i), + source = source, + ) + } + } - override suspend fun getTags(): Set { - val tags = context.httpGet("https://api.${getDomain()}/manga/tag").parseJson() - .getJSONArray("data") - return tags.mapJSONToSet { jo -> - MangaTag( - title = jo.getJSONObject("attributes").getJSONObject("name").firstStringValue().toTitleCase(), - key = jo.getString("id"), - source = source, - ) - } - } + override suspend fun getTags(): Set { + val tags = webClient.httpGet("https://api.${domain}/manga/tag").parseJson() + .getJSONArray("data") + return tags.mapJSONToSet { jo -> + MangaTag( + title = jo.getJSONObject("attributes").getJSONObject("name").firstStringValue().toTitleCase(), + key = jo.getString("id"), + source = source, + ) + } + } - private fun JSONObject.firstStringValue() = values().next() as String + private fun JSONObject.firstStringValue() = values().next() as String - private fun JSONObject.selectByLocale(): String? { - val preferredLocales = context.getPreferredLocales() - for (locale in preferredLocales) { - getStringOrNull(locale.language)?.let { return it } - getStringOrNull(locale.toLanguageTag())?.let { return it } - } - return getStringOrNull(LOCALE_FALLBACK) ?: values().nextOrNull() as? String - } + private fun JSONObject.selectByLocale(): String? { + val preferredLocales = context.getPreferredLocales() + for (locale in preferredLocales) { + getStringOrNull(locale.language)?.let { return it } + getStringOrNull(locale.toLanguageTag())?.let { return it } + } + return getStringOrNull(LOCALE_FALLBACK) ?: values().nextOrNull() as? String + } - private suspend fun loadChapters(mangaId: String): List { - val firstPage = loadChapters(mangaId, offset = 0, limit = CHAPTERS_FIRST_PAGE_SIZE) - if (firstPage.size >= firstPage.total) { - return firstPage.data - } - val tail = coroutineScope { - val leftCount = firstPage.total.coerceAtMost(CHAPTERS_MAX_COUNT) - firstPage.size - val pages = (leftCount / CHAPTERS_MAX_PAGE_SIZE.toFloat()).toIntUp() - val dispatcher = Dispatchers.Default.limitedParallelism(CHAPTERS_PARALLELISM) - List(pages) { page -> - val offset = page * CHAPTERS_MAX_PAGE_SIZE + firstPage.size - async(dispatcher) { - loadChapters(mangaId, offset, CHAPTERS_MAX_PAGE_SIZE) - } - }.awaitAll() - } - val result = ArrayList(firstPage.total) - result += firstPage.data - tail.flatMapTo(result) { it.data } - return result - } + private suspend fun loadChapters(mangaId: String): List { + val firstPage = loadChapters(mangaId, offset = 0, limit = CHAPTERS_FIRST_PAGE_SIZE) + if (firstPage.size >= firstPage.total) { + return firstPage.data + } + val tail = coroutineScope { + val leftCount = firstPage.total.coerceAtMost(CHAPTERS_MAX_COUNT) - firstPage.size + val pages = (leftCount / CHAPTERS_MAX_PAGE_SIZE.toFloat()).toIntUp() + val dispatcher = Dispatchers.Default.limitedParallelism(CHAPTERS_PARALLELISM) + List(pages) { page -> + val offset = page * CHAPTERS_MAX_PAGE_SIZE + firstPage.size + async(dispatcher) { + loadChapters(mangaId, offset, CHAPTERS_MAX_PAGE_SIZE) + } + }.awaitAll() + } + val result = ArrayList(firstPage.total) + result += firstPage.data + tail.flatMapTo(result) { it.data } + return result + } - private suspend fun loadChapters(mangaId: String, offset: Int, limit: Int): Chapters { - val limitedLimit = when { - offset >= CHAPTERS_MAX_COUNT -> return Chapters(emptyList(), CHAPTERS_MAX_COUNT) - offset + limit > CHAPTERS_MAX_COUNT -> CHAPTERS_MAX_COUNT - offset - else -> limit - } - val url = buildString { - append("https://api.") - append(getDomain()) - append("/manga/") - append(mangaId) - append("/feed") - append("?limit=") - append(limitedLimit) - append("&includes[]=scanlation_group&order[volume]=asc&order[chapter]=asc&offset=") - append(offset) - append('&') - append(CONTENT_RATING) - } - val json = context.httpGet(url).parseJson() - if (json.getString("result") == "ok") { - return Chapters( - data = json.optJSONArray("data")?.toJSONList().orEmpty(), - total = json.getInt("total"), - ) - } else { - val error = json.optJSONArray("errors").mapJSON { jo -> - jo.getString("detail") - }.joinToString("\n") - throw ParseException(error, url) - } - } + private suspend fun loadChapters(mangaId: String, offset: Int, limit: Int): Chapters { + val limitedLimit = when { + offset >= CHAPTERS_MAX_COUNT -> return Chapters(emptyList(), CHAPTERS_MAX_COUNT) + offset + limit > CHAPTERS_MAX_COUNT -> CHAPTERS_MAX_COUNT - offset + else -> limit + } + val url = buildString { + append("https://api.") + append(domain) + append("/manga/") + append(mangaId) + append("/feed") + append("?limit=") + append(limitedLimit) + append("&includes[]=scanlation_group&order[volume]=asc&order[chapter]=asc&offset=") + append(offset) + append('&') + append(CONTENT_RATING) + } + val json = webClient.httpGet(url).parseJson() + if (json.getString("result") == "ok") { + return Chapters( + data = json.optJSONArray("data")?.toJSONList().orEmpty(), + total = json.getInt("total"), + ) + } else { + val error = json.optJSONArray("errors").mapJSON { jo -> + jo.getString("detail") + }.joinToString("\n") + throw ParseException(error, url) + } + } - private class Chapters( - val data: List, - val total: Int, - ) { + private class Chapters( + val data: List, + val total: Int, + ) { - val size: Int - get() = data.size - } -} \ No newline at end of file + val size: Int + get() = data.size + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaInUaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaInUaParser.kt index e041a086..68726425 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaInUaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaInUaParser.kt @@ -12,141 +12,142 @@ import java.util.* private const val DEF_BRANCH_NAME = "Основний переклад" @MangaSourceParser("MANGAINUA", "MANGA/in/UA", "uk") -class MangaInUaParser(override val context: MangaLoaderContext) : PagedMangaParser( - source = MangaSource.MANGAINUA, - pageSize = 24, - searchPageSize = 10, +class MangaInUaParser(context: MangaLoaderContext) : PagedMangaParser( + context = context, + source = MangaSource.MANGAINUA, + pageSize = 24, + searchPageSize = 10, ) { - override val sortOrders: Set - get() = Collections.singleton(SortOrder.UPDATED) + override val sortOrders: Set + get() = Collections.singleton(SortOrder.UPDATED) - override val configKeyDomain: ConfigKey.Domain = ConfigKey.Domain("manga.in.ua", null) + override val configKeyDomain: ConfigKey.Domain = ConfigKey.Domain("manga.in.ua", null) - override suspend fun getListPage( - page: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - val url = when { - !query.isNullOrEmpty() -> ( - "/index.php?do=search" + - "&subaction=search" + - "&search_start=$page" + - "&full_search=1" + - "&story=$query" + - "&titleonly=3" - ).toAbsoluteUrl(getDomain()) + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + val url = when { + !query.isNullOrEmpty() -> ( + "/index.php?do=search" + + "&subaction=search" + + "&search_start=$page" + + "&full_search=1" + + "&story=$query" + + "&titleonly=3" + ).toAbsoluteUrl(domain) - tags.isNullOrEmpty() -> "/mangas/page/$page".toAbsoluteUrl(getDomain()) - tags.size == 1 -> "${tags.first().key}/page/$page" - tags.size > 1 -> throw IllegalArgumentException("This source supports only 1 genre") - else -> "/mangas/page/$page".toAbsoluteUrl(getDomain()) - } - val doc = context.httpGet(url).parseHtml() - val container = doc.body().requireElementById("site-content") - val items = container.select("div.col-6") - return items.mapNotNull { item -> - val href = item.selectFirst("a")?.attrAsRelativeUrl("href") ?: return@mapNotNull null - Manga( - id = generateUid(href), - title = item.selectFirst("h3.card__title")?.text() ?: return@mapNotNull null, - coverUrl = item.selectFirst("header.card__cover")?.selectFirst("img")?.run { - attrAsAbsoluteUrlOrNull("data-src") ?: attrAsAbsoluteUrlOrNull("src") - }.orEmpty(), - altTitle = null, - author = null, - rating = item.selectFirst("div.card__short-rate--num") - ?.text() - ?.toFloatOrNull() - ?.div(10F) ?: RATING_UNKNOWN, - url = href, - isNsfw = item.selectFirst("ul.card__list")?.select("li")?.lastOrNull()?.text() == "18+", - tags = runCatching { - item.selectFirst("div.card__category")?.select("a")?.mapToSet { - MangaTag( - title = it.ownText(), - key = it.attr("href").removeSuffix("/"), - source = source, - ) - } - }.getOrNull().orEmpty(), - state = null, - publicUrl = href.toAbsoluteUrl(container.host ?: getDomain()), - source = source, - ) - } - } + tags.isNullOrEmpty() -> "/mangas/page/$page".toAbsoluteUrl(domain) + tags.size == 1 -> "${tags.first().key}/page/$page" + tags.size > 1 -> throw IllegalArgumentException("This source supports only 1 genre") + else -> "/mangas/page/$page".toAbsoluteUrl(domain) + } + val doc = webClient.httpGet(url).parseHtml() + val container = doc.body().requireElementById("site-content") + val items = container.select("div.col-6") + return items.mapNotNull { item -> + val href = item.selectFirst("a")?.attrAsRelativeUrl("href") ?: return@mapNotNull null + Manga( + id = generateUid(href), + title = item.selectFirst("h3.card__title")?.text() ?: return@mapNotNull null, + coverUrl = item.selectFirst("header.card__cover")?.selectFirst("img")?.run { + attrAsAbsoluteUrlOrNull("data-src") ?: attrAsAbsoluteUrlOrNull("src") + }.orEmpty(), + altTitle = null, + author = null, + rating = item.selectFirst("div.card__short-rate--num") + ?.text() + ?.toFloatOrNull() + ?.div(10F) ?: RATING_UNKNOWN, + url = href, + isNsfw = item.selectFirst("ul.card__list")?.select("li")?.lastOrNull()?.text() == "18+", + tags = runCatching { + item.selectFirst("div.card__category")?.select("a")?.mapToSet { + MangaTag( + title = it.ownText(), + key = it.attr("href").removeSuffix("/"), + source = source, + ) + } + }.getOrNull().orEmpty(), + state = null, + publicUrl = href.toAbsoluteUrl(container.host ?: domain), + source = source, + ) + } + } - override suspend fun getDetails(manga: Manga): Manga { - val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml() - val root = doc.body().requireElementById("site-content") - val dateFormat = SimpleDateFormat("dd.MM.yyyy", Locale.US) - val chapterNodes = root.selectFirstOrThrow(".linkstocomics").select(".ltcitems") - var prevChapterName: String? = null - var i = 0 - return manga.copy( - description = root.selectFirst("div.item__full-description")?.text(), - largeCoverUrl = root.selectFirst("div.item__full-sidebar--poster")?.selectFirst("img") - ?.attrAsAbsoluteUrlOrNull("src"), - chapters = chapterNodes.mapChapters { _, item -> - val href = item?.selectFirst("a")?.attrAsRelativeUrlOrNull("href") - ?: return@mapChapters null - val isAlternative = item.styleValueOrNull("background") != null - val name = item.selectFirst("a")?.text().orEmpty() - if (!isAlternative) i++ - MangaChapter( - id = generateUid(href), - name = if (isAlternative) { - prevChapterName ?: return@mapChapters null - } else { - prevChapterName = name - name - }, - number = i, - url = href, - scanlator = null, - branch = if (isAlternative) { - name.substringAfterLast(':').trim() - } else { - DEF_BRANCH_NAME - }, - uploadDate = dateFormat.tryParse(item.selectFirst("div.ltcright")?.text()), - source = source, - ) - }, - ) - } + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val root = doc.body().requireElementById("site-content") + val dateFormat = SimpleDateFormat("dd.MM.yyyy", Locale.US) + val chapterNodes = root.selectFirstOrThrow(".linkstocomics").select(".ltcitems") + var prevChapterName: String? = null + var i = 0 + return manga.copy( + description = root.selectFirst("div.item__full-description")?.text(), + largeCoverUrl = root.selectFirst("div.item__full-sidebar--poster")?.selectFirst("img") + ?.attrAsAbsoluteUrlOrNull("src"), + chapters = chapterNodes.mapChapters { _, item -> + val href = item?.selectFirst("a")?.attrAsRelativeUrlOrNull("href") + ?: return@mapChapters null + val isAlternative = item.styleValueOrNull("background") != null + val name = item.selectFirst("a")?.text().orEmpty() + if (!isAlternative) i++ + MangaChapter( + id = generateUid(href), + name = if (isAlternative) { + prevChapterName ?: return@mapChapters null + } else { + prevChapterName = name + name + }, + number = i, + url = href, + scanlator = null, + branch = if (isAlternative) { + name.substringAfterLast(':').trim() + } else { + DEF_BRANCH_NAME + }, + uploadDate = dateFormat.tryParse(item.selectFirst("div.ltcright")?.text()), + source = source, + ) + }, + ) + } - override suspend fun getPages(chapter: MangaChapter): List { - val fullUrl = chapter.url.toAbsoluteUrl(getDomain()) - val doc = context.httpGet(fullUrl).parseHtml() - val root = doc.body().requireElementById("comics").selectFirstOrThrow("ul.xfieldimagegallery") - return root.select("li").map { ul -> - val img = ul.selectFirstOrThrow("img") - val url = img.attrAsAbsoluteUrl("data-src") - MangaPage( - id = generateUid(url), - url = url, - preview = null, - referer = fullUrl, - source = source, - ) - } - } + override suspend fun getPages(chapter: MangaChapter): List { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + val root = doc.body().requireElementById("comics").selectFirstOrThrow("ul.xfieldimagegallery") + return root.select("li").map { ul -> + val img = ul.selectFirstOrThrow("img") + val url = img.attrAsAbsoluteUrl("data-src") + MangaPage( + id = generateUid(url), + url = url, + preview = null, + referer = fullUrl, + source = source, + ) + } + } - override suspend fun getTags(): Set { - val domain = getDomain() - val doc = context.httpGet("https://$domain/mangas").parseHtml() - val root = doc.body().requireElementById("menu_1").selectFirstOrThrow("div.menu__wrapper") - return root.select("li").mapNotNullToSet { li -> - val a = li.selectFirst("a") ?: return@mapNotNullToSet null - MangaTag( - title = a.ownText(), - key = a.attr("href").removeSuffix("/"), - source = source, - ) - } - } -} \ No newline at end of file + override suspend fun getTags(): Set { + val domain = domain + val doc = webClient.httpGet("https://$domain/mangas").parseHtml() + val root = doc.body().requireElementById("menu_1").selectFirstOrThrow("div.menu__wrapper") + return root.select("li").mapNotNullToSet { li -> + val a = li.selectFirst("a") ?: return@mapNotNullToSet null + MangaTag( + title = a.ownText(), + key = a.attr("href").removeSuffix("/"), + source = source, + ) + } + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaOwlParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaOwlParser.kt deleted file mode 100644 index 06dbf9fb..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaOwlParser.kt +++ /dev/null @@ -1,187 +0,0 @@ -package org.koitharu.kotatsu.parsers.site - -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaParser -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.util.* -import java.text.SimpleDateFormat -import java.util.* - -@Deprecated("Permanently shutdown") -@MangaSourceParser("MANGAOWL", "MangaOwl", "en") -internal class MangaOwlParser(override val context: MangaLoaderContext) : MangaParser(MangaSource.MANGAOWL) { - - override val configKeyDomain = ConfigKey.Domain("mangaowls.com", null) - - override val sortOrders: Set = EnumSet.of( - SortOrder.POPULARITY, - SortOrder.NEWEST, - SortOrder.UPDATED, - ) - - private val regexNsfw = Regex("(yaoi)|(yuri)|(smut)|(mature)|(adult)", RegexOption.IGNORE_CASE) - - override suspend fun getList( - offset: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - val page = (offset / 36f).toIntUp().inc() - val link = buildString { - append("https://") - append(getDomain()) - when { - !query.isNullOrEmpty() -> { - append("/search/$page?search=") - append(query.urlEncoded()) - } - - !tags.isNullOrEmpty() -> { - for (tag in tags) { - append(tag.key) - } - append("/$page?type=${getAlternativeSortKey(sortOrder)}") - } - - else -> { - append("/${getSortKey(sortOrder)}/$page") - } - } - } - val doc = context.httpGet(link).parseHtml() - val slides = doc.body().selectOrThrow("ul.slides") - val items = slides.select("div.col-md-2") - return items.mapNotNull { item -> - val href = item.selectFirst("h6 a")?.attrAsRelativeUrlOrNull("href") ?: return@mapNotNull null - Manga( - id = generateUid(href), - title = item.selectFirst("h6 a")?.text() ?: return@mapNotNull null, - coverUrl = item.select("div.img-responsive").attr("abs:data-background-image"), - altTitle = null, - author = null, - rating = runCatching { - item.selectFirst("div.block-stars") - ?.text() - ?.toFloatOrNull() - ?.div(10f) - }.getOrNull() ?: RATING_UNKNOWN, - url = href, - isNsfw = false, - tags = emptySet(), - state = null, - publicUrl = href.toAbsoluteUrl(getDomain()), - source = source, - ) - } - } - - override suspend fun getDetails(manga: Manga): Manga { - val doc = context.httpGet(manga.publicUrl).parseHtml() - val info = doc.body().selectFirstOrThrow("div.single_detail") - val table = doc.body().selectFirstOrThrow("div.single-grid-right") - val dateFormat = SimpleDateFormat("MM/dd/yyyy", Locale.US) - val trRegex = "window\\['tr'] = '([^']*)';".toRegex(RegexOption.IGNORE_CASE) - val trElement = doc.getElementsByTag("script").find { trRegex.find(it.data()) != null } - ?: doc.parseFailed("Oops, tr not found") - val tr = trRegex.find(trElement.data())!!.groups[1]!!.value - val s = context.encodeBase64(getDomain().toByteArray()) - var isNsfw = manga.isNsfw - val parsedTags = info.select("div.col-xs-12.col-md-8.single-right-grid-right > p > a[href*=genres]") - .mapNotNullToSet { - val a = it.selectFirst("a") ?: return@mapNotNullToSet null - val name = a.text() - if (!isNsfw && isNsfwGenre(name)) { - isNsfw = true - } - MangaTag( - title = name.toTitleCase(), - key = a.attr("href"), - source = source, - ) - } - return manga.copy( - description = info.selectFirst(".description")?.html(), - largeCoverUrl = info.select("img").first()?.let { img -> - if (img.hasAttr("data-src")) img.attr("abs:data-src") else img.attr("abs:src") - }, - isNsfw = isNsfw, - author = info.selectFirst("p.fexi_header_para a.author_link")?.text(), - state = parseStatus(info.select("p.fexi_header_para:contains(status)").first()?.ownText()), - tags = manga.tags + parsedTags, - chapters = table.select("div.table.table-chapter-list").select("li.list-group-item.chapter_list") - .asReversed().mapChapters { i, li -> - val a = li.select("a") - val href = a.attr("data-href").ifEmpty { - li.parseFailed("Link is missing") - } - MangaChapter( - id = generateUid(href), - name = a.select("label").text(), - number = i + 1, - url = "$href?tr=$tr&s=$s", - scanlator = null, - branch = null, - uploadDate = dateFormat.tryParse(li.selectFirst("small:last-of-type")?.text()), - source = MangaSource.MANGAOWL, - ) - }, - ) - } - - override suspend fun getPages(chapter: MangaChapter): List { - val fullUrl = chapter.url.toAbsoluteUrl(getDomain()) - val doc = context.httpGet(fullUrl).parseHtml() - val root = doc.body().selectOrThrow("div.item img.owl-lazy") - return root.map { div -> - val url = div?.attrAsRelativeUrlOrNull("data-src") ?: doc.parseFailed("Page image not found") - MangaPage( - id = generateUid(url), - url = url, - preview = null, - referer = url, - source = MangaSource.MANGAOWL, - ) - } - } - - private fun parseStatus(status: String?) = when { - status == null -> null - status.contains("Ongoing") -> MangaState.ONGOING - status.contains("Completed") -> MangaState.FINISHED - else -> null - } - - override suspend fun getTags(): Set { - val doc = context.httpGet("https://${getDomain()}/").parseHtml() - val root = doc.body().select("ul.dropdown-menu.multi-column.columns-3").select("li") - return root.mapToSet { p -> - val a = p.selectFirstOrThrow("a") - MangaTag( - title = a.text().toTitleCase(), - key = a.attr("href"), - source = source, - ) - } - } - - private fun getSortKey(sortOrder: SortOrder) = - when (sortOrder) { - SortOrder.POPULARITY -> "popular" - SortOrder.NEWEST -> "new_release" - SortOrder.UPDATED -> "lastest" - else -> "lastest" - } - - private fun getAlternativeSortKey(sortOrder: SortOrder) = - when (sortOrder) { - SortOrder.POPULARITY -> "0" - SortOrder.NEWEST -> "2" - SortOrder.UPDATED -> "3" - else -> "3" - } - - private fun isNsfwGenre(name: String): Boolean = regexNsfw.containsMatchIn(name) -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaTownParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaTownParser.kt index 95623121..dbc5ee62 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaTownParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaTownParser.kt @@ -11,207 +11,207 @@ import java.text.SimpleDateFormat import java.util.* @MangaSourceParser("MANGATOWN", "MangaTown", "en") -internal class MangaTownParser(override val context: MangaLoaderContext) : MangaParser(MangaSource.MANGATOWN) { +internal class MangaTownParser(context: MangaLoaderContext) : MangaParser(context, MangaSource.MANGATOWN) { - override val configKeyDomain = ConfigKey.Domain("www.mangatown.com", null) + override val configKeyDomain = ConfigKey.Domain("www.mangatown.com", null) - override val sortOrders: Set = EnumSet.of( - SortOrder.ALPHABETICAL, - SortOrder.RATING, - SortOrder.POPULARITY, - SortOrder.UPDATED, - ) + override val sortOrders: Set = EnumSet.of( + SortOrder.ALPHABETICAL, + SortOrder.RATING, + SortOrder.POPULARITY, + SortOrder.UPDATED, + ) - private val regexTag = Regex("[^\\-]+-[^\\-]+-[^\\-]+-[^\\-]+-[^\\-]+-[^\\-]+") + private val regexTag = Regex("[^\\-]+-[^\\-]+-[^\\-]+-[^\\-]+-[^\\-]+-[^\\-]+") - override suspend fun getList( - offset: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - val sortKey = when (sortOrder) { - SortOrder.ALPHABETICAL -> "?name.az" - SortOrder.RATING -> "?rating.za" - SortOrder.UPDATED -> "?last_chapter_time.za" - else -> "" - } - val page = (offset / 30) + 1 - val url = when { - !query.isNullOrEmpty() -> { - if (offset != 0) { - return emptyList() - } - "/search?name=${query.urlEncoded()}".toAbsoluteUrl(getDomain()) - } + override suspend fun getList( + offset: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + val sortKey = when (sortOrder) { + SortOrder.ALPHABETICAL -> "?name.az" + SortOrder.RATING -> "?rating.za" + SortOrder.UPDATED -> "?last_chapter_time.za" + else -> "" + } + val page = (offset / 30) + 1 + val url = when { + !query.isNullOrEmpty() -> { + if (offset != 0) { + return emptyList() + } + "/search?name=${query.urlEncoded()}".toAbsoluteUrl(domain) + } - tags.isNullOrEmpty() -> "/directory/$page.htm$sortKey".toAbsoluteUrl(getDomain()) - tags.size == 1 -> "/directory/${tags.first().key}/$page.htm$sortKey".toAbsoluteUrl(getDomain()) - else -> tags.joinToString( - prefix = "/search?page=$page".toAbsoluteUrl(getDomain()), - ) { tag -> - "&genres[${tag.key}]=1" - } - } - val doc = context.httpGet(url).parseHtml() - val root = doc.body().selectFirstOrThrow("ul.manga_pic_list") - return root.select("li").mapNotNull { li -> - val a = li.selectFirst("a.manga_cover") - val href = a?.attrAsRelativeUrlOrNull("href") - ?: return@mapNotNull null - val views = li.select("p.view") - val status = views.firstNotNullOfOrNull { it.ownText().takeIf { x -> x.startsWith("Status:") } } - ?.substringAfter(':')?.trim()?.lowercase(Locale.ROOT) - Manga( - id = generateUid(href), - title = a.attr("title"), - coverUrl = a.selectFirst("img")?.absUrl("src").orEmpty(), - source = MangaSource.MANGATOWN, - altTitle = null, - rating = li.selectFirst("p.score")?.selectFirst("b") - ?.ownText()?.toFloatOrNull()?.div(5f) ?: RATING_UNKNOWN, - author = views.firstNotNullOfOrNull { it.text().takeIf { x -> x.startsWith("Author:") } } - ?.substringAfter(':') - ?.trim(), - state = when (status) { - "ongoing" -> MangaState.ONGOING - "completed" -> MangaState.FINISHED - else -> null - }, - tags = li.selectFirst("p.keyWord")?.select("a")?.mapNotNullToSet tags@{ x -> - MangaTag( - title = x.attr("title").toTitleCase(), - key = x.attr("href").parseTagKey() ?: return@tags null, - source = MangaSource.MANGATOWN, - ) - }.orEmpty(), - url = href, - isNsfw = false, - publicUrl = href.toAbsoluteUrl(a.host ?: getDomain()), - ) - } - } + tags.isNullOrEmpty() -> "/directory/$page.htm$sortKey".toAbsoluteUrl(domain) + tags.size == 1 -> "/directory/${tags.first().key}/$page.htm$sortKey".toAbsoluteUrl(domain) + else -> tags.joinToString( + prefix = "/search?page=$page".toAbsoluteUrl(domain), + ) { tag -> + "&genres[${tag.key}]=1" + } + } + val doc = webClient.httpGet(url).parseHtml() + val root = doc.body().selectFirstOrThrow("ul.manga_pic_list") + return root.select("li").mapNotNull { li -> + val a = li.selectFirst("a.manga_cover") + val href = a?.attrAsRelativeUrlOrNull("href") + ?: return@mapNotNull null + val views = li.select("p.view") + val status = views.firstNotNullOfOrNull { it.ownText().takeIf { x -> x.startsWith("Status:") } } + ?.substringAfter(':')?.trim()?.lowercase(Locale.ROOT) + Manga( + id = generateUid(href), + title = a.attr("title"), + coverUrl = a.selectFirst("img")?.absUrl("src").orEmpty(), + source = MangaSource.MANGATOWN, + altTitle = null, + rating = li.selectFirst("p.score")?.selectFirst("b") + ?.ownText()?.toFloatOrNull()?.div(5f) ?: RATING_UNKNOWN, + author = views.firstNotNullOfOrNull { it.text().takeIf { x -> x.startsWith("Author:") } } + ?.substringAfter(':') + ?.trim(), + state = when (status) { + "ongoing" -> MangaState.ONGOING + "completed" -> MangaState.FINISHED + else -> null + }, + tags = li.selectFirst("p.keyWord")?.select("a")?.mapNotNullToSet tags@{ x -> + MangaTag( + title = x.attr("title").toTitleCase(), + key = x.attr("href").parseTagKey() ?: return@tags null, + source = MangaSource.MANGATOWN, + ) + }.orEmpty(), + url = href, + isNsfw = false, + publicUrl = href.toAbsoluteUrl(a.host ?: domain), + ) + } + } - override suspend fun getDetails(manga: Manga): Manga { - val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml() - val root = doc.body().selectFirstOrThrow("section.main") - .selectFirstOrThrow("div.article_content") - val info = root.selectFirst("div.detail_info")?.selectFirst("ul") - val chaptersList = root.selectFirst("div.chapter_content") - ?.selectFirst("ul.chapter_list")?.select("li")?.asReversed() - val dateFormat = SimpleDateFormat("MMM dd,yyyy", Locale.US) - return manga.copy( - tags = manga.tags + info?.select("li")?.find { x -> - x.selectFirst("b")?.ownText() == "Genre(s):" - }?.select("a")?.mapNotNull { a -> - MangaTag( - title = a.attr("title").toTitleCase(), - key = a.attr("href").parseTagKey() ?: return@mapNotNull null, - source = MangaSource.MANGATOWN, - ) - }.orEmpty(), - description = info?.getElementById("show")?.ownText(), - chapters = chaptersList?.mapChapters { i, li -> - val href = li.selectFirst("a")?.attrAsRelativeUrlOrNull("href") - ?: return@mapChapters null - val name = li.select("span") - .filter { x -> x.className().isEmpty() } - .joinToString(" - ") { it.text() }.trim() - MangaChapter( - id = generateUid(href), - url = href, - source = MangaSource.MANGATOWN, - number = i + 1, - uploadDate = parseChapterDate( - dateFormat, - li.selectFirst("span.time")?.text(), - ), - name = name.ifEmpty { "${manga.title} - ${i + 1}" }, - scanlator = null, - branch = null, - ) - } ?: bypassLicensedChapters(manga), - ) - } + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val root = doc.body().selectFirstOrThrow("section.main") + .selectFirstOrThrow("div.article_content") + val info = root.selectFirst("div.detail_info")?.selectFirst("ul") + val chaptersList = root.selectFirst("div.chapter_content") + ?.selectFirst("ul.chapter_list")?.select("li")?.asReversed() + val dateFormat = SimpleDateFormat("MMM dd,yyyy", Locale.US) + return manga.copy( + tags = manga.tags + info?.select("li")?.find { x -> + x.selectFirst("b")?.ownText() == "Genre(s):" + }?.select("a")?.mapNotNull { a -> + MangaTag( + title = a.attr("title").toTitleCase(), + key = a.attr("href").parseTagKey() ?: return@mapNotNull null, + source = MangaSource.MANGATOWN, + ) + }.orEmpty(), + description = info?.getElementById("show")?.ownText(), + chapters = chaptersList?.mapChapters { i, li -> + val href = li.selectFirst("a")?.attrAsRelativeUrlOrNull("href") + ?: return@mapChapters null + val name = li.select("span") + .filter { x -> x.className().isEmpty() } + .joinToString(" - ") { it.text() }.trim() + MangaChapter( + id = generateUid(href), + url = href, + source = MangaSource.MANGATOWN, + number = i + 1, + uploadDate = parseChapterDate( + dateFormat, + li.selectFirst("span.time")?.text(), + ), + name = name.ifEmpty { "${manga.title} - ${i + 1}" }, + scanlator = null, + branch = null, + ) + } ?: bypassLicensedChapters(manga), + ) + } - override suspend fun getPages(chapter: MangaChapter): List { - val fullUrl = chapter.url.toAbsoluteUrl(getDomain()) - val doc = context.httpGet(fullUrl).parseHtml() - val root = doc.body().selectFirstOrThrow("div.page_select") - return root.selectFirstOrThrow("select").selectOrThrow("option").mapNotNull { - val href = it.attrAsRelativeUrlOrNull("value") - if (href == null || href.endsWith("featured.html")) { - return@mapNotNull null - } - MangaPage( - id = generateUid(href), - url = href, - preview = null, - referer = fullUrl, - source = MangaSource.MANGATOWN, - ) - } - } + override suspend fun getPages(chapter: MangaChapter): List { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + val root = doc.body().selectFirstOrThrow("div.page_select") + return root.selectFirstOrThrow("select").selectOrThrow("option").mapNotNull { + val href = it.attrAsRelativeUrlOrNull("value") + if (href == null || href.endsWith("featured.html")) { + return@mapNotNull null + } + MangaPage( + id = generateUid(href), + url = href, + preview = null, + referer = fullUrl, + source = MangaSource.MANGATOWN, + ) + } + } - override suspend fun getPageUrl(page: MangaPage): String { - val doc = context.httpGet(page.url.toAbsoluteUrl(getDomain())).parseHtml() - return doc.requireElementById("image").attrAsAbsoluteUrl("src") - } + override suspend fun getPageUrl(page: MangaPage): String { + val doc = webClient.httpGet(page.url.toAbsoluteUrl(domain)).parseHtml() + return doc.requireElementById("image").attrAsAbsoluteUrl("src") + } - override suspend fun getTags(): Set { - val doc = context.httpGet("/directory/".toAbsoluteUrl(getDomain())).parseHtml() - val root = doc.body().selectFirst("aside.right") - ?.getElementsContainingOwnText("Genres") - ?.first() - ?.nextElementSibling() ?: doc.parseFailed("Root not found") - return root.select("li").mapNotNullToSet { li -> - val a = li.selectFirst("a") ?: return@mapNotNullToSet null - val key = a.attr("href").parseTagKey() - if (key.isNullOrEmpty()) { - return@mapNotNullToSet null - } - MangaTag( - source = MangaSource.MANGATOWN, - key = key, - title = a.text().toTitleCase(), - ) - } - } + override suspend fun getTags(): Set { + val doc = webClient.httpGet("/directory/".toAbsoluteUrl(domain)).parseHtml() + val root = doc.body().selectFirst("aside.right") + ?.getElementsContainingOwnText("Genres") + ?.first() + ?.nextElementSibling() ?: doc.parseFailed("Root not found") + return root.select("li").mapNotNullToSet { li -> + val a = li.selectFirst("a") ?: return@mapNotNullToSet null + val key = a.attr("href").parseTagKey() + if (key.isNullOrEmpty()) { + return@mapNotNullToSet null + } + MangaTag( + source = MangaSource.MANGATOWN, + key = key, + title = a.text().toTitleCase(), + ) + } + } - private fun parseChapterDate(dateFormat: DateFormat, date: String?): Long { - return when { - date.isNullOrEmpty() -> 0L - date.contains("Today") -> Calendar.getInstance().timeInMillis - date.contains("Yesterday") -> Calendar.getInstance().apply { add(Calendar.DAY_OF_MONTH, -1) }.timeInMillis - else -> dateFormat.tryParse(date) - } - } + private fun parseChapterDate(dateFormat: DateFormat, date: String?): Long { + return when { + date.isNullOrEmpty() -> 0L + date.contains("Today") -> Calendar.getInstance().timeInMillis + date.contains("Yesterday") -> Calendar.getInstance().apply { add(Calendar.DAY_OF_MONTH, -1) }.timeInMillis + else -> dateFormat.tryParse(date) + } + } - private suspend fun bypassLicensedChapters(manga: Manga): List { - val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain("m"))).parseHtml() - val list = doc.body().selectFirst("ul.detail-ch-list") ?: return emptyList() - val dateFormat = SimpleDateFormat("MMM dd,yyyy", Locale.US) - return list.select("li").asReversed().mapIndexedNotNull { i, li -> - val a = li.selectFirst("a") ?: return@mapIndexedNotNull null - val href = a.attrAsRelativeUrl("href") - val name = a.selectFirst("span.vol")?.text().orEmpty().ifEmpty { - a.ownText() - } - MangaChapter( - id = generateUid(href), - url = href, - source = MangaSource.MANGATOWN, - number = i + 1, - uploadDate = parseChapterDate( - dateFormat, - li.selectFirst("span.time")?.text(), - ), - name = name.ifEmpty { "${manga.title} - ${i + 1}" }, - scanlator = null, - branch = null, - ) - } - } + private suspend fun bypassLicensedChapters(manga: Manga): List { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(getDomain("m"))).parseHtml() + val list = doc.body().selectFirst("ul.detail-ch-list") ?: return emptyList() + val dateFormat = SimpleDateFormat("MMM dd,yyyy", Locale.US) + return list.select("li").asReversed().mapIndexedNotNull { i, li -> + val a = li.selectFirst("a") ?: return@mapIndexedNotNull null + val href = a.attrAsRelativeUrl("href") + val name = a.selectFirst("span.vol")?.text().orEmpty().ifEmpty { + a.ownText() + } + MangaChapter( + id = generateUid(href), + url = href, + source = MangaSource.MANGATOWN, + number = i + 1, + uploadDate = parseChapterDate( + dateFormat, + li.selectFirst("span.time")?.text(), + ), + name = name.ifEmpty { "${manga.title} - ${i + 1}" }, + scanlator = null, + branch = null, + ) + } + } - private fun String.parseTagKey() = split('/').findLast { regexTag matches it } -} \ No newline at end of file + private fun String.parseTagKey() = split('/').findLast { regexTag matches it } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/Manhwa18Parser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/Manhwa18Parser.kt index 0f8b82b1..97a40b94 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/Manhwa18Parser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/Manhwa18Parser.kt @@ -9,173 +9,177 @@ import org.koitharu.kotatsu.parsers.util.* import java.util.* @MangaSourceParser("MANHWA18", "Manhwa18", "en") -class Manhwa18Parser(override val context: MangaLoaderContext) : PagedMangaParser(MangaSource.MANHWA18, pageSize = 20, searchPageSize = 20) { +class Manhwa18Parser(context: MangaLoaderContext) : + PagedMangaParser(context, MangaSource.MANHWA18, pageSize = 20, searchPageSize = 20) { - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("manhwa18.net", null) + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("manhwa18.net", null) - override val sortOrders: Set - get() = EnumSet.of(SortOrder.UPDATED, SortOrder.POPULARITY, SortOrder.ALPHABETICAL) + override val sortOrders: Set + get() = EnumSet.of(SortOrder.UPDATED, SortOrder.POPULARITY, SortOrder.ALPHABETICAL) - override suspend fun getFavicons(): Favicons { - return Favicons(listOf( - Favicon("https://${getDomain()}/uploads/logos/logo-mini.png", 92, null) - ), getDomain()) - } + override suspend fun getFavicons(): Favicons { + return Favicons( + listOf( + Favicon("https://${domain}/uploads/logos/logo-mini.png", 92, null), + ), + domain, + ) + } - override suspend fun getDetails(manga: Manga): Manga { - val docs = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml() - val cardInfoElement = docs.selectFirst(".card .manga-info") - val author = cardInfoElement?.selectFirst("b:contains(Author(s))")?.parent() - ?.select("a.btn") - ?.joinToString(", ") { it.text() } - val tags = cardInfoElement?.selectFirst("b:contains(Genre(s))")?.parent() - ?.select("a.btn") - ?.mapToSet { MangaTag(it.text(), it.text().lowercase(), MangaSource.MANHWA18) } - val state = cardInfoElement?.selectFirst("b:contains(Status)")?.parent() - ?.selectFirst("a.btn") - ?.let { - when (it.text()) { - "On going" -> MangaState.ONGOING - "Completed" -> MangaState.FINISHED - else -> null - } - } + override suspend fun getDetails(manga: Manga): Manga { + val docs = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val cardInfoElement = docs.selectFirst(".card .manga-info") + val author = cardInfoElement?.selectFirst("b:contains(Author(s))")?.parent() + ?.select("a.btn") + ?.joinToString(", ") { it.text() } + val tags = cardInfoElement?.selectFirst("b:contains(Genre(s))")?.parent() + ?.select("a.btn") + ?.mapToSet { MangaTag(it.text(), it.text().lowercase(), MangaSource.MANHWA18) } + val state = cardInfoElement?.selectFirst("b:contains(Status)")?.parent() + ?.selectFirst("a.btn") + ?.let { + when (it.text()) { + "On going" -> MangaState.ONGOING + "Completed" -> MangaState.FINISHED + else -> null + } + } - return manga.copy( - altTitle = cardInfoElement?.selectFirst("b:contains(Other names)")?.parent()?.ownText()?.removePrefix(": "), - author = author, - description = docs.selectFirst(".series-summary .summary-content")?.html(), - tags = tags.orEmpty(), - state = state, - chapters = docs.select(".card-body > .list-chapters > a").asReversed().mapChapters { index, element -> - // attrAsRelativeUrl only return page url without the '/' - val chapterUrl = element.attrAsAbsoluteUrlOrNull("href")?.toRelativeUrl(getDomain()) - ?: return@mapChapters null - val uploadDate = parseUploadDate(element.selectFirst(".chapter-time")?.text()) - MangaChapter( - id = generateUid(chapterUrl), - name = element.selectFirst(".chapter-name")?.text().orEmpty(), - number = index + 1, - url = chapterUrl, - scanlator = null, - uploadDate = uploadDate, - branch = null, - source = MangaSource.MANHWA18, - ) - } - ) - } + return manga.copy( + altTitle = cardInfoElement?.selectFirst("b:contains(Other names)")?.parent()?.ownText()?.removePrefix(": "), + author = author, + description = docs.selectFirst(".series-summary .summary-content")?.html(), + tags = tags.orEmpty(), + state = state, + chapters = docs.select(".card-body > .list-chapters > a").asReversed().mapChapters { index, element -> + // attrAsRelativeUrl only return page url without the '/' + val chapterUrl = element.attrAsAbsoluteUrlOrNull("href")?.toRelativeUrl(domain) + ?: return@mapChapters null + val uploadDate = parseUploadDate(element.selectFirst(".chapter-time")?.text()) + MangaChapter( + id = generateUid(chapterUrl), + name = element.selectFirst(".chapter-name")?.text().orEmpty(), + number = index + 1, + url = chapterUrl, + scanlator = null, + uploadDate = uploadDate, + branch = null, + source = MangaSource.MANHWA18, + ) + }, + ) + } - // 7 minutes ago - // 5 hours ago - // 2 days ago - // 2 weeks ago - // 4 years ago - private fun parseUploadDate(timeStr: String?): Long { - timeStr ?: return 0 + // 7 minutes ago + // 5 hours ago + // 2 days ago + // 2 weeks ago + // 4 years ago + private fun parseUploadDate(timeStr: String?): Long { + timeStr ?: return 0 - val timeWords = timeStr.split(' ') - if (timeWords.size != 3) return 0 - val timeWord = timeWords[1] - val timeAmount = timeWords[0].toIntOrNull() ?: return 0 - val timeUnit = when (timeWord) { - "minute", "minutes" -> Calendar.MINUTE - "hour", "hours" -> Calendar.HOUR - "day", "days" -> Calendar.DAY_OF_YEAR - "week", "weeks" -> Calendar.WEEK_OF_YEAR - "month", "months" -> Calendar.MONTH - "year", "years" -> Calendar.YEAR - else -> return 0 - } - val cal = Calendar.getInstance() - cal.add(timeUnit, -timeAmount) - return cal.time.time - } + val timeWords = timeStr.split(' ') + if (timeWords.size != 3) return 0 + val timeWord = timeWords[1] + val timeAmount = timeWords[0].toIntOrNull() ?: return 0 + val timeUnit = when (timeWord) { + "minute", "minutes" -> Calendar.MINUTE + "hour", "hours" -> Calendar.HOUR + "day", "days" -> Calendar.DAY_OF_YEAR + "week", "weeks" -> Calendar.WEEK_OF_YEAR + "month", "months" -> Calendar.MONTH + "year", "years" -> Calendar.YEAR + else -> return 0 + } + val cal = Calendar.getInstance() + cal.add(timeUnit, -timeAmount) + return cal.time.time + } - override suspend fun getListPage( - page: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - val sortQuery = when(sortOrder) { - SortOrder.ALPHABETICAL -> "name" - SortOrder.POPULARITY -> "views" - SortOrder.UPDATED -> "last_update" - else -> "" - } + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + val sortQuery = when (sortOrder) { + SortOrder.ALPHABETICAL -> "name" + SortOrder.POPULARITY -> "views" + SortOrder.UPDATED -> "last_update" + else -> "" + } - val sortType = if (sortOrder == SortOrder.ALPHABETICAL) "ASC" else "DESC" - val tagQuery = tags?.joinToString(",") { it.key }.orEmpty() - val url = buildString { - append("https://") - append(getDomain()) - append("/manga-list.html?listType=pagination&page=") - append(page) - append("&artist=&author=&group=&m_status=&name=") - append(query?.urlEncoded().orEmpty()) - append("&genre=$tagQuery") - append("&ungenre=") - append("&sort=") - append(sortQuery) - append("&sort_type=") - append(sortType) - } + val sortType = if (sortOrder == SortOrder.ALPHABETICAL) "ASC" else "DESC" + val tagQuery = tags?.joinToString(",") { it.key }.orEmpty() + val url = buildString { + append("https://") + append(domain) + append("/manga-list.html?listType=pagination&page=") + append(page) + append("&artist=&author=&group=&m_status=&name=") + append(query?.urlEncoded().orEmpty()) + append("&genre=$tagQuery") + append("&ungenre=") + append("&sort=") + append(sortQuery) + append("&sort_type=") + append(sortType) + } - val docs = context.httpGet(url).parseHtml() - val actualPage = docs.selectFirst("ul.pagination a.active")?.text()?.toIntOrNull() - if (actualPage != page) { - return emptyList() - } + val docs = webClient.httpGet(url).parseHtml() + val actualPage = docs.selectFirst("ul.pagination a.active")?.text()?.toIntOrNull() + if (actualPage != page) { + return emptyList() + } - return docs.select(".card-body .thumb-item-flow") - .map { - val titleElement = it.selectFirstOrThrow(".thumb_attr.series-title > a") - val absUrl = titleElement.attrAsAbsoluteUrl("href") - Manga( - id = generateUid(absUrl.toRelativeUrl(getDomain())), - title = titleElement.text(), - altTitle = null, - url = absUrl.toRelativeUrl(getDomain()), - publicUrl = absUrl, - rating = RATING_UNKNOWN, - isNsfw = true, - coverUrl = it.selectFirst("div.img-in-ratio")?.attrAsAbsoluteUrl("data-bg").orEmpty(), - tags = emptySet(), - state = null, - author = null, - largeCoverUrl = null, - description = null, - source = MangaSource.MANHWA18, - ) - } - } + return docs.select(".card-body .thumb-item-flow") + .map { + val titleElement = it.selectFirstOrThrow(".thumb_attr.series-title > a") + val absUrl = titleElement.attrAsAbsoluteUrl("href") + Manga( + id = generateUid(absUrl.toRelativeUrl(domain)), + title = titleElement.text(), + altTitle = null, + url = absUrl.toRelativeUrl(domain), + publicUrl = absUrl, + rating = RATING_UNKNOWN, + isNsfw = true, + coverUrl = it.selectFirst("div.img-in-ratio")?.attrAsAbsoluteUrl("data-bg").orEmpty(), + tags = emptySet(), + state = null, + author = null, + largeCoverUrl = null, + description = null, + source = MangaSource.MANHWA18, + ) + } + } - override suspend fun getPages(chapter: MangaChapter): List { - val chapterUrl = chapter.url.toAbsoluteUrl(getDomain()) - return context.httpGet(chapterUrl).parseHtml() - .select("div.chapter-content > img").map { - val url = it.attrAsRelativeUrlOrNull("src").orEmpty() - MangaPage( - id = generateUid(url), - url = url, - referer = chapterUrl, - preview = null, - source = MangaSource.MANHWA18, - ) - } - } + override suspend fun getPages(chapter: MangaChapter): List { + val chapterUrl = chapter.url.toAbsoluteUrl(domain) + return webClient.httpGet(chapterUrl).parseHtml() + .select("div.chapter-content > img").map { + val url = it.attrAsRelativeUrlOrNull("src").orEmpty() + MangaPage( + id = generateUid(url), + url = url, + referer = chapterUrl, + preview = null, + source = MangaSource.MANHWA18, + ) + } + } - override suspend fun getTags(): Set { - return context.httpGet("https://${getDomain()}/").parseHtml().selectFirstOrThrow(".genres-menu") - .select("a.genres-item").orEmpty() - .mapToSet { - MangaTag( - title = it.text(), - key = it.text().lowercase(), - source = MangaSource.MANHWA18, - ) - } - } -} \ No newline at end of file + override suspend fun getTags(): Set { + return webClient.httpGet("https://${domain}/").parseHtml().selectFirstOrThrow(".genres-menu") + .select("a.genres-item").orEmpty() + .mapToSet { + MangaTag( + title = it.text(), + key = it.text().lowercase(), + source = MangaSource.MANHWA18, + ) + } + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NHentaiParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NHentaiParser.kt index 42170f49..5ca2f515 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NHentaiParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NHentaiParser.kt @@ -15,174 +15,174 @@ import java.text.SimpleDateFormat import java.util.* @MangaSourceParser("NHENTAI", "N-Hentai") -class NHentaiParser(override val context: MangaLoaderContext) : PagedMangaParser(MangaSource.NHENTAI, pageSize = 25) { +class NHentaiParser(context: MangaLoaderContext) : PagedMangaParser(context, MangaSource.NHENTAI, pageSize = 25) { - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("nhentai.net", null) + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("nhentai.net", null) - override val sortOrders: Set - get() = EnumSet.of(SortOrder.NEWEST, SortOrder.POPULARITY) + override val sortOrders: Set + get() = EnumSet.of(SortOrder.NEWEST, SortOrder.POPULARITY) - override suspend fun getListPage( - page: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - if (query.isNullOrEmpty() && tags != null && tags.size > 1) { - return getListPage(page, buildQuery(tags), emptySet(), sortOrder) - } - val domain = getDomain() - val url = buildString { - append("https://") - append(domain) - if (!query.isNullOrEmpty()) { - append("/search/?q=") - append(query.urlEncoded()) - append("&page=") - append(page) - if (sortOrder == SortOrder.POPULARITY) { - append("&sort=popular") - } - } else { - append('/') - if (!tags.isNullOrEmpty()) { - val tag = tags.single() - append("tag/") - append(tag.key) - append('/') - if (sortOrder == SortOrder.POPULARITY) { - append("popular") - } - append("?page=") - append(page) - } else { - if (sortOrder == SortOrder.POPULARITY) { - append("?sort=popular&page=") - } else { - append("?page=") - } - append(page) - } - } - } - val root = context.httpGet(url).parseHtml().body().requireElementById("content") - .selectLastOrThrow("div.index-container") - val regexBrackets = Regex("\\[[^]]+]|\\([^)]+\\)") - val regexSpaces = Regex("\\s+") - return root.select(".gallery").map { div -> - val a = div.selectFirstOrThrow("a.cover") - val href = a.attrAsRelativeUrl("href") - val img = div.selectFirstOrThrow("img") - val title = div.selectFirstOrThrow(".caption").text() - Manga( - id = generateUid(href), - title = title.replace(regexBrackets, "") - .replace(regexSpaces, " ") - .trim(), - altTitle = null, - url = href, - publicUrl = href.toAbsoluteUrl(domain), - rating = RATING_UNKNOWN, - isNsfw = true, - coverUrl = img.attrAsAbsoluteUrlOrNull("data-src") - ?: img.attrAsAbsoluteUrl("src"), - tags = setOf(), - state = null, - author = null, - largeCoverUrl = null, - description = null, - chapters = listOf(), - source = source, - ) - } - } + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + if (query.isNullOrEmpty() && tags != null && tags.size > 1) { + return getListPage(page, buildQuery(tags), emptySet(), sortOrder) + } + val domain = domain + val url = buildString { + append("https://") + append(domain) + if (!query.isNullOrEmpty()) { + append("/search/?q=") + append(query.urlEncoded()) + append("&page=") + append(page) + if (sortOrder == SortOrder.POPULARITY) { + append("&sort=popular") + } + } else { + append('/') + if (!tags.isNullOrEmpty()) { + val tag = tags.single() + append("tag/") + append(tag.key) + append('/') + if (sortOrder == SortOrder.POPULARITY) { + append("popular") + } + append("?page=") + append(page) + } else { + if (sortOrder == SortOrder.POPULARITY) { + append("?sort=popular&page=") + } else { + append("?page=") + } + append(page) + } + } + } + val root = webClient.httpGet(url).parseHtml().body().requireElementById("content") + .selectLastOrThrow("div.index-container") + val regexBrackets = Regex("\\[[^]]+]|\\([^)]+\\)") + val regexSpaces = Regex("\\s+") + return root.select(".gallery").map { div -> + val a = div.selectFirstOrThrow("a.cover") + val href = a.attrAsRelativeUrl("href") + val img = div.selectFirstOrThrow("img") + val title = div.selectFirstOrThrow(".caption").text() + Manga( + id = generateUid(href), + title = title.replace(regexBrackets, "") + .replace(regexSpaces, " ") + .trim(), + altTitle = null, + url = href, + publicUrl = href.toAbsoluteUrl(domain), + rating = RATING_UNKNOWN, + isNsfw = true, + coverUrl = img.attrAsAbsoluteUrlOrNull("data-src") + ?: img.attrAsAbsoluteUrl("src"), + tags = setOf(), + state = null, + author = null, + largeCoverUrl = null, + description = null, + chapters = listOf(), + source = source, + ) + } + } - override suspend fun getDetails(manga: Manga): Manga { - val root = context.httpGet( - url = manga.url.toAbsoluteUrl(getDomain()), - ).parseHtml().body().requireElementById("bigcontainer") - val img = root.requireElementById("cover").selectFirstOrThrow("img") - val tagContainers = root.requireElementById("tags").select(".tag-container") - val dateFormat = SimpleDateFormat( - "yyyy-MM-dd'T'HH:mm:ss.SSSSSS'+00:00'", - Locale.ROOT, - ) - return manga.copy( - tags = tagContainers.find { x -> x.ownText() == "Tags:" }?.parseTags() ?: manga.tags, - author = tagContainers.find { x -> x.ownText() == "Artists:" } - ?.selectFirst("span.name")?.text()?.toCamelCase(), - largeCoverUrl = img.attrAsAbsoluteUrlOrNull("data-src") - ?: img.attrAsAbsoluteUrl("src"), - description = null, - chapters = listOf( - MangaChapter( - id = manga.id, - name = manga.title, - number = 1, - url = manga.url, - scanlator = null, - uploadDate = dateFormat.tryParse( - tagContainers.find { x -> x.ownText() == "Uploaded:" } - ?.selectFirst("time") - ?.attr("datetime"), - ), - branch = null, - source = source, - ), - ), - ) - } + override suspend fun getDetails(manga: Manga): Manga { + val root = webClient.httpGet( + url = manga.url.toAbsoluteUrl(domain), + ).parseHtml().body().requireElementById("bigcontainer") + val img = root.requireElementById("cover").selectFirstOrThrow("img") + val tagContainers = root.requireElementById("tags").select(".tag-container") + val dateFormat = SimpleDateFormat( + "yyyy-MM-dd'T'HH:mm:ss.SSSSSS'+00:00'", + Locale.ROOT, + ) + return manga.copy( + tags = tagContainers.find { x -> x.ownText() == "Tags:" }?.parseTags() ?: manga.tags, + author = tagContainers.find { x -> x.ownText() == "Artists:" } + ?.selectFirst("span.name")?.text()?.toCamelCase(), + largeCoverUrl = img.attrAsAbsoluteUrlOrNull("data-src") + ?: img.attrAsAbsoluteUrl("src"), + description = null, + chapters = listOf( + MangaChapter( + id = manga.id, + name = manga.title, + number = 1, + url = manga.url, + scanlator = null, + uploadDate = dateFormat.tryParse( + tagContainers.find { x -> x.ownText() == "Uploaded:" } + ?.selectFirst("time") + ?.attr("datetime"), + ), + branch = null, + source = source, + ), + ), + ) + } - override suspend fun getPages(chapter: MangaChapter): List { - val url = chapter.url.toAbsoluteUrl(getDomain()) - val root = context.httpGet(url).parseHtml().requireElementById("thumbnail-container") - return root.select(".thumb-container").map { div -> - val a = div.selectFirstOrThrow("a") - val img = div.selectFirstOrThrow("img") - val href = a.attrAsRelativeUrl("href") - MangaPage( - id = generateUid(href), - url = href, - referer = url, - preview = img.attrAsAbsoluteUrlOrNull("data-src") - ?: img.attrAsAbsoluteUrl("src"), - source = source, - ) - } - } + override suspend fun getPages(chapter: MangaChapter): List { + val url = chapter.url.toAbsoluteUrl(domain) + val root = webClient.httpGet(url).parseHtml().requireElementById("thumbnail-container") + return root.select(".thumb-container").map { div -> + val a = div.selectFirstOrThrow("a") + val img = div.selectFirstOrThrow("img") + val href = a.attrAsRelativeUrl("href") + MangaPage( + id = generateUid(href), + url = href, + referer = url, + preview = img.attrAsAbsoluteUrlOrNull("data-src") + ?: img.attrAsAbsoluteUrl("src"), + source = source, + ) + } + } - override suspend fun getPageUrl(page: MangaPage): String { - val root = context.httpGet(page.url.toAbsoluteUrl(getDomain())).parseHtml().body() - .requireElementById("image-container") - return root.selectFirstOrThrow("img").attrAsAbsoluteUrl("src") - } + override suspend fun getPageUrl(page: MangaPage): String { + val root = webClient.httpGet(page.url.toAbsoluteUrl(domain)).parseHtml().body() + .requireElementById("image-container") + return root.selectFirstOrThrow("img").attrAsAbsoluteUrl("src") + } - override suspend fun getTags(): Set { - return coroutineScope { - // parse first 3 pages of tags - (1..3).map { page -> - async { getTags(page) } - } - }.awaitAll().flattenTo(ArraySet(360)) - } + override suspend fun getTags(): Set { + return coroutineScope { + // parse first 3 pages of tags + (1..3).map { page -> + async { getTags(page) } + } + }.awaitAll().flattenTo(ArraySet(360)) + } - private suspend fun getTags(page: Int): Set { - val root = context.httpGet("https://${getDomain()}/tags/popular?page=$page").parseHtml().body() - .getElementById("tag-container") - return root?.parseTags().orEmpty() - } + private suspend fun getTags(page: Int): Set { + val root = webClient.httpGet("https://${domain}/tags/popular?page=$page").parseHtml().body() + .getElementById("tag-container") + return root?.parseTags().orEmpty() + } - private fun Element.parseTags() = select("a.tag").mapToSet { a -> - val href = a.attr("href").removeSuffix('/') - MangaTag( - title = a.selectFirstOrThrow(".name").text().toTitleCase(), - key = href.substringAfterLast('/'), - source = source, - ) - } + private fun Element.parseTags() = select("a.tag").mapToSet { a -> + val href = a.attr("href").removeSuffix('/') + MangaTag( + title = a.selectFirstOrThrow(".name").text().toTitleCase(), + key = href.substringAfterLast('/'), + source = source, + ) + } - private fun buildQuery(tags: Collection) = tags.joinToString(separator = " ") { tag -> - "tag:\"${tag.key}\"" - } -} \ No newline at end of file + private fun buildQuery(tags: Collection) = tags.joinToString(separator = " ") { tag -> + "tag:\"${tag.key}\"" + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NetTruyenParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NetTruyenParser.kt index afe8664f..aaf4c956 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NetTruyenParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NetTruyenParser.kt @@ -15,210 +15,210 @@ import java.text.SimpleDateFormat import java.util.* @MangaSourceParser("NETTRUYEN", "NetTruyen", "vi") -class NetTruyenParser(override val context: MangaLoaderContext) : - PagedMangaParser(MangaSource.NETTRUYEN, pageSize = 36) { - - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("nettruyenin.com", null) - - override val sortOrders: Set - get() = EnumSet.of(SortOrder.UPDATED, SortOrder.POPULARITY, SortOrder.NEWEST, SortOrder.RATING) - - private val mutex = Mutex() - private val dateFormat = SimpleDateFormat("dd/MM/yy", Locale.US) - private var tagCache: ArrayMap? = null - - override suspend fun getDetails(manga: Manga): Manga { - val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml() - val rating = doc.selectFirst("span[itemprop=ratingValue]") - ?.ownText() - ?.toFloatOrNull() ?: 0f - - val chapterElements = doc.getElementById("nt_listchapter")?.select("ul > li") ?: doc.parseFailed() - val chapters = chapterElements.asReversed().mapChapters { index, element -> - val a = element.selectFirst("div.chapter > a") ?: return@mapChapters null - val relativeUrl = a.attrAsRelativeUrlOrNull("href") ?: return@mapChapters null - val timeText = element.selectFirst("div.col-xs-4.text-center.no-wrap.small")?.text() - - MangaChapter( - id = generateUid(relativeUrl), - name = a.text(), - number = index + 1, - url = relativeUrl, - scanlator = null, - uploadDate = parseChapterTime(timeText), - branch = null, - source = source, - ) - } - - return manga.copy( - rating = rating / 5, - chapters = chapters, - description = doc.selectFirst("div.detail-content > p")?.html(), - isNsfw = doc.selectFirst("div.alert.alert-danger > strong:contains(Cảnh báo độ tuổi)") != null, - ) - } - - // 20 giây trước - // 52 phút trước - // 6 giờ trước - // 2 ngày trước - // 19:09 30/07 - // 23/12/21 - private fun parseChapterTime(timeText: String?): Long { - if (timeText.isNullOrEmpty()) { - return 0L - } - - val timeWords = arrayOf("giây", "phút", "giờ", "ngày") - val calendar = Calendar.getInstance() - val timeArr = timeText.split(' ') - if (WordSet(*timeWords).anyWordIn(timeText)) { - val timeSuffix = timeArr.getOrNull(1) - val timeDiff = timeArr.getOrNull(0)?.toIntOrNull() ?: return 0L - when (timeSuffix) { - timeWords[0] -> calendar.add(Calendar.SECOND, -timeDiff) - timeWords[1] -> calendar.add(Calendar.MINUTE, -timeDiff) - timeWords[2] -> calendar.add(Calendar.HOUR, -timeDiff) - timeWords[3] -> calendar.add(Calendar.DATE, -timeDiff) - else -> return 0L - } - } else { - val relativeDate = timeArr.lastOrNull() ?: return 0L - val dateString = when (relativeDate.split('/').size) { - 2 -> { - val currentYear = calendar.get(Calendar.YEAR).toString().takeLast(2) - "$relativeDate/$currentYear" - } - - 3 -> relativeDate - else -> return 0L - } - - calendar.timeInMillis = dateFormat.tryParse(dateString) - } - - - return calendar.time.time - } - - override suspend fun getListPage( - page: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - val isSearching = !query.isNullOrEmpty() - val url = buildString { - append("https://") - append(getDomain()) - if (isSearching) { - append("/tim-truyen?keyword=") - append(query!!.urlEncoded()) - append("&page=") - append(page) - } else { - val tagQuery = tags.orEmpty().joinToString(",") { it.key } - append("/tim-truyen-nang-cao?genres=$tagQuery") - append("¬genres=&gender=-1&status=-1&minchapter=1&sort=${getSortOrderKey(sortOrder)}") - append("&page=$page") - } - } - - val response = if (isSearching) { - val result = runCatchingCancellable { context.httpGet(url) } - val exception = result.exceptionOrNull() - if (exception is NotFoundException) { - return emptyList() - } - - result.getOrThrow() - } else { - context.httpGet(url) - } - - val itemsElements = response.parseHtml() - .select("div.ModuleContent > div.items") - .select("div.item") - return itemsElements.mapNotNull { item -> - val tooltipElement = item.selectFirst("div.box_tootip") ?: return@mapNotNull null - val absUrl = item.selectFirst("div.image > a")?.attrAsAbsoluteUrlOrNull("href") ?: return@mapNotNull null - val slug = absUrl.substringAfterLast('/') - val mangaState = when (tooltipElement.selectFirst("div.message_main > p:contains(Tình trạng)")?.ownText()) { - "Đang tiến hành" -> MangaState.ONGOING - "Hoàn thành" -> MangaState.FINISHED - else -> null - } - - val tagMap = getOrCreateTagMap() - val tagsElement = tooltipElement.selectFirst("div.message_main > p:contains(Thể loại)")?.ownText().orEmpty() - val mangaTags = tagsElement.split(',').mapNotNullToSet { tagMap[it.trim()] } - Manga( - id = generateUid(slug), - title = tooltipElement.selectFirst("div.title")?.text().orEmpty(), - altTitle = null, - url = absUrl.toRelativeUrl(getDomain()), - publicUrl = absUrl, - rating = RATING_UNKNOWN, - isNsfw = false, - coverUrl = item.selectFirst("div.image a img")?.absUrl("data-original").orEmpty(), - largeCoverUrl = null, - tags = mangaTags, - state = mangaState, - author = tooltipElement.selectFirst("div.message_main > p:contains(Tác giả)")?.ownText(), - description = tooltipElement.selectFirst("div.box_text")?.text(), - chapters = null, - source = source, - ) - } - } - - - override suspend fun getPages(chapter: MangaChapter): List { - val pageElements = context.httpGet(chapter.url.toAbsoluteUrl(getDomain())).parseHtml() - .select("div.reading-detail.box_doc > div img") - return pageElements.map { element -> - val url = element.attrAsAbsoluteUrl("data-original") - MangaPage( - id = generateUid(url), - url = url, - referer = getDomain(), - preview = null, - source = source, - ) - } - } - - override suspend fun getTags(): Set { - val map = getOrCreateTagMap() - val tagSet = ArraySet(map.size) - for (entry in map) { - tagSet.add(entry.value) - } - - return tagSet - } - - private suspend fun getOrCreateTagMap(): ArrayMap = mutex.withLock { - tagCache?.let { return@withLock it } - val doc = context.httpGet("/tim-truyen-nang-cao".toAbsoluteUrl(getDomain())).parseHtml() - val tagItems = doc.select("div.genre-item") - val result = ArrayMap(tagItems.size) - for (item in tagItems) { - val title = item.text().trim() - val key = item.select("span[data-id]").attr("data-id") - result[title] = MangaTag(title = title, key = key, source = source) - } - tagCache = result - result - } - - private fun getSortOrderKey(sortOrder: SortOrder) = when (sortOrder) { - SortOrder.UPDATED -> 0 - SortOrder.POPULARITY -> 10 - SortOrder.NEWEST -> 15 - SortOrder.RATING -> 20 - else -> throw IllegalArgumentException("Sort order ${sortOrder.name} not supported") - } +class NetTruyenParser(context: MangaLoaderContext) : + PagedMangaParser(context, MangaSource.NETTRUYEN, pageSize = 36) { + + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("nettruyenin.com", null) + + override val sortOrders: Set + get() = EnumSet.of(SortOrder.UPDATED, SortOrder.POPULARITY, SortOrder.NEWEST, SortOrder.RATING) + + private val mutex = Mutex() + private val dateFormat = SimpleDateFormat("dd/MM/yy", Locale.US) + private var tagCache: ArrayMap? = null + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val rating = doc.selectFirst("span[itemprop=ratingValue]") + ?.ownText() + ?.toFloatOrNull() ?: 0f + + val chapterElements = doc.getElementById("nt_listchapter")?.select("ul > li") ?: doc.parseFailed() + val chapters = chapterElements.asReversed().mapChapters { index, element -> + val a = element.selectFirst("div.chapter > a") ?: return@mapChapters null + val relativeUrl = a.attrAsRelativeUrlOrNull("href") ?: return@mapChapters null + val timeText = element.selectFirst("div.col-xs-4.text-center.no-wrap.small")?.text() + + MangaChapter( + id = generateUid(relativeUrl), + name = a.text(), + number = index + 1, + url = relativeUrl, + scanlator = null, + uploadDate = parseChapterTime(timeText), + branch = null, + source = source, + ) + } + + return manga.copy( + rating = rating / 5, + chapters = chapters, + description = doc.selectFirst("div.detail-content > p")?.html(), + isNsfw = doc.selectFirst("div.alert.alert-danger > strong:contains(Cảnh báo độ tuổi)") != null, + ) + } + + // 20 giây trước + // 52 phút trước + // 6 giờ trước + // 2 ngày trước + // 19:09 30/07 + // 23/12/21 + private fun parseChapterTime(timeText: String?): Long { + if (timeText.isNullOrEmpty()) { + return 0L + } + + val timeWords = arrayOf("giây", "phút", "giờ", "ngày") + val calendar = Calendar.getInstance() + val timeArr = timeText.split(' ') + if (WordSet(*timeWords).anyWordIn(timeText)) { + val timeSuffix = timeArr.getOrNull(1) + val timeDiff = timeArr.getOrNull(0)?.toIntOrNull() ?: return 0L + when (timeSuffix) { + timeWords[0] -> calendar.add(Calendar.SECOND, -timeDiff) + timeWords[1] -> calendar.add(Calendar.MINUTE, -timeDiff) + timeWords[2] -> calendar.add(Calendar.HOUR, -timeDiff) + timeWords[3] -> calendar.add(Calendar.DATE, -timeDiff) + else -> return 0L + } + } else { + val relativeDate = timeArr.lastOrNull() ?: return 0L + val dateString = when (relativeDate.split('/').size) { + 2 -> { + val currentYear = calendar.get(Calendar.YEAR).toString().takeLast(2) + "$relativeDate/$currentYear" + } + + 3 -> relativeDate + else -> return 0L + } + + calendar.timeInMillis = dateFormat.tryParse(dateString) + } + + + return calendar.time.time + } + + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + val isSearching = !query.isNullOrEmpty() + val url = buildString { + append("https://") + append(domain) + if (isSearching) { + append("/tim-truyen?keyword=") + append(query!!.urlEncoded()) + append("&page=") + append(page) + } else { + val tagQuery = tags.orEmpty().joinToString(",") { it.key } + append("/tim-truyen-nang-cao?genres=$tagQuery") + append("¬genres=&gender=-1&status=-1&minchapter=1&sort=${getSortOrderKey(sortOrder)}") + append("&page=$page") + } + } + + val response = if (isSearching) { + val result = runCatchingCancellable { webClient.httpGet(url) } + val exception = result.exceptionOrNull() + if (exception is NotFoundException) { + return emptyList() + } + + result.getOrThrow() + } else { + webClient.httpGet(url) + } + + val itemsElements = response.parseHtml() + .select("div.ModuleContent > div.items") + .select("div.item") + return itemsElements.mapNotNull { item -> + val tooltipElement = item.selectFirst("div.box_tootip") ?: return@mapNotNull null + val absUrl = item.selectFirst("div.image > a")?.attrAsAbsoluteUrlOrNull("href") ?: return@mapNotNull null + val slug = absUrl.substringAfterLast('/') + val mangaState = when (tooltipElement.selectFirst("div.message_main > p:contains(Tình trạng)")?.ownText()) { + "Đang tiến hành" -> MangaState.ONGOING + "Hoàn thành" -> MangaState.FINISHED + else -> null + } + + val tagMap = getOrCreateTagMap() + val tagsElement = tooltipElement.selectFirst("div.message_main > p:contains(Thể loại)")?.ownText().orEmpty() + val mangaTags = tagsElement.split(',').mapNotNullToSet { tagMap[it.trim()] } + Manga( + id = generateUid(slug), + title = tooltipElement.selectFirst("div.title")?.text().orEmpty(), + altTitle = null, + url = absUrl.toRelativeUrl(domain), + publicUrl = absUrl, + rating = RATING_UNKNOWN, + isNsfw = false, + coverUrl = item.selectFirst("div.image a img")?.absUrl("data-original").orEmpty(), + largeCoverUrl = null, + tags = mangaTags, + state = mangaState, + author = tooltipElement.selectFirst("div.message_main > p:contains(Tác giả)")?.ownText(), + description = tooltipElement.selectFirst("div.box_text")?.text(), + chapters = null, + source = source, + ) + } + } + + + override suspend fun getPages(chapter: MangaChapter): List { + val pageElements = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + .select("div.reading-detail.box_doc > div img") + return pageElements.map { element -> + val url = element.attrAsAbsoluteUrl("data-original") + MangaPage( + id = generateUid(url), + url = url, + referer = domain, + preview = null, + source = source, + ) + } + } + + override suspend fun getTags(): Set { + val map = getOrCreateTagMap() + val tagSet = ArraySet(map.size) + for (entry in map) { + tagSet.add(entry.value) + } + + return tagSet + } + + private suspend fun getOrCreateTagMap(): ArrayMap = mutex.withLock { + tagCache?.let { return@withLock it } + val doc = webClient.httpGet("/tim-truyen-nang-cao".toAbsoluteUrl(domain)).parseHtml() + val tagItems = doc.select("div.genre-item") + val result = ArrayMap(tagItems.size) + for (item in tagItems) { + val title = item.text().trim() + val key = item.select("span[data-id]").attr("data-id") + result[title] = MangaTag(title = title, key = key, source = source) + } + tagCache = result + result + } + + private fun getSortOrderKey(sortOrder: SortOrder) = when (sortOrder) { + SortOrder.UPDATED -> 0 + SortOrder.POPULARITY -> 10 + SortOrder.NEWEST -> 15 + SortOrder.RATING -> 20 + else -> throw IllegalArgumentException("Sort order ${sortOrder.name} not supported") + } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NicovideoSeigaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NicovideoSeigaParser.kt index de8953e4..2434199b 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NicovideoSeigaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NicovideoSeigaParser.kt @@ -11,175 +11,175 @@ private const val STATUS_ONGOING = "連載" private const val STATUS_FINISHED = "完結" @MangaSourceParser("NICOVIDEO_SEIGA", "Nicovideo Seiga", "ja") -class NicovideoSeigaParser(override val context: MangaLoaderContext) : - MangaParser(MangaSource.NICOVIDEO_SEIGA), - MangaParserAuthProvider { +class NicovideoSeigaParser(context: MangaLoaderContext) : + MangaParser(context, MangaSource.NICOVIDEO_SEIGA), + MangaParserAuthProvider { - override val authUrl: String - get() = "https://${getDomain("account")}/login?site=seiga" + override val authUrl: String + get() = "https://${getDomain("account")}/login?site=seiga" - override val isAuthorized: Boolean - get() = context.cookieJar.getCookies(getDomain("seiga")).any { - it.name == "user_session" - } + override val isAuthorized: Boolean + get() = context.cookieJar.getCookies(getDomain("seiga")).any { + it.name == "user_session" + } - override suspend fun getUsername(): String { - val body = context.httpGet("https://${getDomain("app")}/my/apps").parseHtml().body() - return body.selectFirst("#userinfo > div > div > strong")?.text() ?: throw AuthRequiredException(source) - } + override suspend fun getUsername(): String { + val body = webClient.httpGet("https://${getDomain("app")}/my/apps").parseHtml().body() + return body.selectFirst("#userinfo > div > div > strong")?.text() ?: throw AuthRequiredException(source) + } - override val sortOrders: Set = EnumSet.of( - SortOrder.UPDATED, - SortOrder.POPULARITY, - ) + override val sortOrders: Set = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + ) - override val configKeyDomain: ConfigKey.Domain = ConfigKey.Domain("nicovideo.jp", null) + override val configKeyDomain: ConfigKey.Domain = ConfigKey.Domain("nicovideo.jp", null) - @InternalParsersApi - override suspend fun getList( - offset: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - val page = (offset / 20f).toIntUp().inc() - val domain = getDomain("seiga") - val url = when { - !query.isNullOrEmpty() -> return if (offset == 0) getSearchList(query, page) else emptyList() - tags.isNullOrEmpty() -> "https://$domain/manga/list?page=$page&sort=${getSortKey(sortOrder)}" - tags.size == 1 -> "https://$domain/manga/list?category=${tags.first().key}&page=$page" + - "&sort=${getSortKey(sortOrder)}" + @InternalParsersApi + override suspend fun getList( + offset: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + val page = (offset / 20f).toIntUp().inc() + val domain = getDomain("seiga") + val url = when { + !query.isNullOrEmpty() -> return if (offset == 0) getSearchList(query, page) else emptyList() + tags.isNullOrEmpty() -> "https://$domain/manga/list?page=$page&sort=${getSortKey(sortOrder)}" + tags.size == 1 -> "https://$domain/manga/list?category=${tags.first().key}&page=$page" + + "&sort=${getSortKey(sortOrder)}" - tags.size > 1 -> throw IllegalArgumentException("This source supports only 1 category") - else -> "https://$domain/manga/list?page=$page&sort=${getSortKey(sortOrder)}" - } - val doc = context.httpGet(url).parseHtml() - val comicList = doc.body().select("#comic_list > ul > li") ?: doc.parseFailed("Container not found") - val items = comicList.select("div > .description > div > div") - return items.mapNotNull { item -> - val href = - item.selectFirst(".comic_icon > div > a")?.attrAsRelativeUrlOrNull("href") ?: return@mapNotNull null - val statusText = item.selectFirst(".mg_description_header > .mg_icon > .content_status > span")?.text() - Manga( - id = generateUid(href), - title = item.selectFirst(".mg_body > .title > a")?.text() ?: return@mapNotNull null, - coverUrl = item.selectFirst(".comic_icon > div > a > img")?.attrAsAbsoluteUrl("src").orEmpty(), - altTitle = null, - author = item.selectFirst(".mg_description_header > .mg_author > a")?.text(), - rating = RATING_UNKNOWN, - url = href, - isNsfw = false, - tags = item.getElementsByAttributeValueContaining("href", "?category=").mapToSet { a -> - MangaTag( - key = a.attr("href").substringAfterLast('='), - title = a.ownText().trim(), - source = source, - ) - }, - state = when (statusText) { - STATUS_ONGOING -> MangaState.ONGOING - STATUS_FINISHED -> MangaState.FINISHED - else -> null - }, - publicUrl = href.toAbsoluteUrl(item.host ?: getDomain("seiga")), - source = source, - ) - } - } + tags.size > 1 -> throw IllegalArgumentException("This source supports only 1 category") + else -> "https://$domain/manga/list?page=$page&sort=${getSortKey(sortOrder)}" + } + val doc = webClient.httpGet(url).parseHtml() + val comicList = doc.body().select("#comic_list > ul > li") ?: doc.parseFailed("Container not found") + val items = comicList.select("div > .description > div > div") + return items.mapNotNull { item -> + val href = + item.selectFirst(".comic_icon > div > a")?.attrAsRelativeUrlOrNull("href") ?: return@mapNotNull null + val statusText = item.selectFirst(".mg_description_header > .mg_icon > .content_status > span")?.text() + Manga( + id = generateUid(href), + title = item.selectFirst(".mg_body > .title > a")?.text() ?: return@mapNotNull null, + coverUrl = item.selectFirst(".comic_icon > div > a > img")?.attrAsAbsoluteUrl("src").orEmpty(), + altTitle = null, + author = item.selectFirst(".mg_description_header > .mg_author > a")?.text(), + rating = RATING_UNKNOWN, + url = href, + isNsfw = false, + tags = item.getElementsByAttributeValueContaining("href", "?category=").mapToSet { a -> + MangaTag( + key = a.attr("href").substringAfterLast('='), + title = a.ownText().trim(), + source = source, + ) + }, + state = when (statusText) { + STATUS_ONGOING -> MangaState.ONGOING + STATUS_FINISHED -> MangaState.FINISHED + else -> null + }, + publicUrl = href.toAbsoluteUrl(item.host ?: getDomain("seiga")), + source = source, + ) + } + } - override suspend fun getDetails(manga: Manga): Manga { - val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain("seiga"))).parseHtml() - val contents = doc.body().selectFirstOrThrow("#contents") - val statusText = contents - .select("div.mg_work_detail > div > div:nth-child(2) > div.tip.content_status.status_series > span") - .text() - return manga.copy( - description = contents.selectFirst("div.mg_work_detail > div > div.row > div.description_text")?.html(), - largeCoverUrl = contents.selectFirst("div.primaries > div.main_visual > a > img") - ?.attrAsAbsoluteUrlOrNull("src"), - state = when (statusText) { - STATUS_ONGOING -> MangaState.ONGOING - STATUS_FINISHED -> MangaState.FINISHED - else -> null - }, - isNsfw = contents.select(".icon_adult").isNotEmpty(), - chapters = contents.select("#episode_list > ul > li").mapChapters { i, li -> - val href = li.selectFirst("div > div.description > div.title > a") - ?.attrAsRelativeUrl("href") ?: li.parseFailed() - MangaChapter( - id = generateUid(href), - name = li.select("div > div.description > div.title > a").text(), - number = i + 1, - url = href, - scanlator = null, - branch = null, - uploadDate = 0, - source = source, - ) - }, - ) - } + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(getDomain("seiga"))).parseHtml() + val contents = doc.body().selectFirstOrThrow("#contents") + val statusText = contents + .select("div.mg_work_detail > div > div:nth-child(2) > div.tip.content_status.status_series > span") + .text() + return manga.copy( + description = contents.selectFirst("div.mg_work_detail > div > div.row > div.description_text")?.html(), + largeCoverUrl = contents.selectFirst("div.primaries > div.main_visual > a > img") + ?.attrAsAbsoluteUrlOrNull("src"), + state = when (statusText) { + STATUS_ONGOING -> MangaState.ONGOING + STATUS_FINISHED -> MangaState.FINISHED + else -> null + }, + isNsfw = contents.select(".icon_adult").isNotEmpty(), + chapters = contents.select("#episode_list > ul > li").mapChapters { i, li -> + val href = li.selectFirst("div > div.description > div.title > a") + ?.attrAsRelativeUrl("href") ?: li.parseFailed() + MangaChapter( + id = generateUid(href), + name = li.select("div > div.description > div.title > a").text(), + number = i + 1, + url = href, + scanlator = null, + branch = null, + uploadDate = 0, + source = source, + ) + }, + ) + } - override suspend fun getPages(chapter: MangaChapter): List { - val fullUrl = chapter.url.toAbsoluteUrl(getDomain("seiga")) - val doc = context.httpGet(fullUrl).parseHtml() - if (!doc.select("#login_manga").isEmpty()) - throw AuthRequiredException(source) - val root = doc.body().select("#page_contents > li") - return root.map { li -> - val url = li.select("div > img").attr("data-original") - MangaPage( - id = generateUid(url), - url = url, - preview = null, - referer = fullUrl, - source = source, - ) - } - } + override suspend fun getPages(chapter: MangaChapter): List { + val fullUrl = chapter.url.toAbsoluteUrl(getDomain("seiga")) + val doc = webClient.httpGet(fullUrl).parseHtml() + if (!doc.select("#login_manga").isEmpty()) + throw AuthRequiredException(source) + val root = doc.body().select("#page_contents > li") + return root.map { li -> + val url = li.select("div > img").attr("data-original") + MangaPage( + id = generateUid(url), + url = url, + preview = null, + referer = fullUrl, + source = source, + ) + } + } - override suspend fun getTags(): Set { - val doc = context.httpGet("https://${getDomain("seiga")}/manga/list").parseHtml() - val root = doc.body().selectOrThrow("#mg_category_list > ul > li") - return root.mapToSet { li -> - val a = li.selectFirstOrThrow("a") - MangaTag( - title = a.text(), - key = a.attrAsRelativeUrlOrNull("href").orEmpty(), - source = source, - ) - } - } + override suspend fun getTags(): Set { + val doc = webClient.httpGet("https://${getDomain("seiga")}/manga/list").parseHtml() + val root = doc.body().selectOrThrow("#mg_category_list > ul > li") + return root.mapToSet { li -> + val a = li.selectFirstOrThrow("a") + MangaTag( + title = a.text(), + key = a.attrAsRelativeUrlOrNull("href").orEmpty(), + source = source, + ) + } + } - private suspend fun getSearchList(query: String, page: Int): List { - val domain = getDomain("seiga") - val doc = context.httpGet("https://$domain/manga/search/?q=$query&page=$page&sort=score").parseHtml() - val root = doc.body().select(".search_result__item") - return root.mapNotNull { item -> - val href = item.selectFirst(".search_result__item__thumbnail > a") - ?.attrAsRelativeUrl("href") ?: doc.parseFailed() - Manga( - id = generateUid(href), - url = href, - publicUrl = href.toAbsoluteUrl(item.host ?: domain), - title = item.selectFirst(".search_result__item__info > .search_result__item__info--title > a") - ?.text()?.trim() ?: return@mapNotNull null, - altTitle = null, - author = null, - tags = emptySet(), - rating = RATING_UNKNOWN, - state = null, - isNsfw = false, - source = source, - coverUrl = item.selectFirst(".search_result__item__thumbnail > a > img") - ?.attrAsAbsoluteUrl("data-original").orEmpty(), - ) - } - } + private suspend fun getSearchList(query: String, page: Int): List { + val domain = getDomain("seiga") + val doc = webClient.httpGet("https://$domain/manga/search/?q=$query&page=$page&sort=score").parseHtml() + val root = doc.body().select(".search_result__item") + return root.mapNotNull { item -> + val href = item.selectFirst(".search_result__item__thumbnail > a") + ?.attrAsRelativeUrl("href") ?: doc.parseFailed() + Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(item.host ?: domain), + title = item.selectFirst(".search_result__item__info > .search_result__item__info--title > a") + ?.text()?.trim() ?: return@mapNotNull null, + altTitle = null, + author = null, + tags = emptySet(), + rating = RATING_UNKNOWN, + state = null, + isNsfw = false, + source = source, + coverUrl = item.selectFirst(".search_result__item__thumbnail > a > img") + ?.attrAsAbsoluteUrl("data-original").orEmpty(), + ) + } + } - private fun getSortKey(sortOrder: SortOrder) = when (sortOrder) { - SortOrder.POPULARITY -> "manga_view" - SortOrder.UPDATED -> "manga_updated" - else -> "manga_view" - } -} \ No newline at end of file + private fun getSortKey(sortOrder: SortOrder) = when (sortOrder) { + SortOrder.POPULARITY -> "manga_view" + SortOrder.UPDATED -> "manga_updated" + else -> "manga_view" + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NineMangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NineMangaParser.kt index 0918ac86..f3d6d2ce 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NineMangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NineMangaParser.kt @@ -12,249 +12,248 @@ import java.text.SimpleDateFormat import java.util.* internal abstract class NineMangaParser( - final override val context: MangaLoaderContext, - source: MangaSource, - defaultDomain: String, -) : PagedMangaParser(source, pageSize = 26) { + context: MangaLoaderContext, + source: MangaSource, + defaultDomain: String, +) : PagedMangaParser(context, source, pageSize = 26) { - override val configKeyDomain = ConfigKey.Domain(defaultDomain, null) + override val configKeyDomain = ConfigKey.Domain(defaultDomain, null) - init { - context.cookieJar.insertCookies(getDomain(), "ninemanga_template_desk=yes") - } + init { + context.cookieJar.insertCookies(domain, "ninemanga_template_desk=yes") + } - override val headers = Headers.Builder() - .add("Accept-Language", "en-US;q=0.7,en;q=0.3") - .build() + override val headers = Headers.Builder() + .add("Accept-Language", "en-US;q=0.7,en;q=0.3") + .build() - override val sortOrders: Set = Collections.singleton( - SortOrder.POPULARITY, - ) + override val sortOrders: Set = Collections.singleton( + SortOrder.POPULARITY, + ) - override suspend fun getListPage( - page: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - val url = buildString { - append("https://") - append(getDomain()) - when { - !query.isNullOrEmpty() -> { - append("/search/?name_sel=&wd=") - append(query.urlEncoded()) - append("&page=") - } + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + val url = buildString { + append("https://") + append(domain) + when { + !query.isNullOrEmpty() -> { + append("/search/?name_sel=&wd=") + append(query.urlEncoded()) + append("&page=") + } - !tags.isNullOrEmpty() -> { - append("/search/?category_id=") - for (tag in tags) { - append(tag.key) - append(',') - } - append("&page=") - } + !tags.isNullOrEmpty() -> { + append("/search/?category_id=") + for (tag in tags) { + append(tag.key) + append(',') + } + append("&page=") + } - else -> { - append("/category/index_") - } - } - append(page) - append(".html") - } - val doc = context.httpGet(url, headers).parseHtml() - val root = doc.body().selectFirst("ul.direlist") - ?: doc.parseFailed("Cannot find root") - val baseHost = root.baseUri().toHttpUrl().host - return root.select("li").map { node -> - val href = node.selectFirst("a")?.absUrl("href") - ?: node.parseFailed("Link not found") - val relUrl = href.toRelativeUrl(baseHost) - val dd = node.selectFirst("dd") - Manga( - id = generateUid(relUrl), - url = relUrl, - publicUrl = href, - title = dd?.selectFirst("a.bookname")?.text()?.toCamelCase().orEmpty(), - altTitle = null, - coverUrl = node.selectFirst("img")?.absUrl("src").orEmpty(), - rating = RATING_UNKNOWN, - author = null, - isNsfw = false, - tags = emptySet(), - state = null, - source = source, - description = dd?.selectFirst("p")?.html(), - ) - } - } + else -> { + append("/category/index_") + } + } + append(page) + append(".html") + } + val doc = webClient.httpGet(url).parseHtml() + val root = doc.body().selectFirst("ul.direlist") + ?: doc.parseFailed("Cannot find root") + val baseHost = root.baseUri().toHttpUrl().host + return root.select("li").map { node -> + val href = node.selectFirst("a")?.absUrl("href") + ?: node.parseFailed("Link not found") + val relUrl = href.toRelativeUrl(baseHost) + val dd = node.selectFirst("dd") + Manga( + id = generateUid(relUrl), + url = relUrl, + publicUrl = href, + title = dd?.selectFirst("a.bookname")?.text()?.toCamelCase().orEmpty(), + altTitle = null, + coverUrl = node.selectFirst("img")?.absUrl("src").orEmpty(), + rating = RATING_UNKNOWN, + author = null, + isNsfw = false, + tags = emptySet(), + state = null, + source = source, + description = dd?.selectFirst("p")?.html(), + ) + } + } - override suspend fun getDetails(manga: Manga): Manga { - val doc = context.httpGet( - manga.url.toAbsoluteUrl(getDomain()) + "?waring=1", - headers, - ).parseHtml() - val root = doc.body().selectFirstOrThrow("div.manga") - val infoRoot = root.selectFirstOrThrow("div.bookintro") - return manga.copy( - tags = infoRoot.getElementsByAttributeValue("itemprop", "genre").first() - ?.select("a")?.mapToSet { a -> - MangaTag( - title = a.text().toTitleCase(), - key = a.attr("href").substringBetween("/", "."), - source = source, - ) - }.orEmpty(), - author = infoRoot.getElementsByAttributeValue("itemprop", "author").first()?.text(), - state = parseStatus(infoRoot.select("li a.red").text()), - description = infoRoot.getElementsByAttributeValue("itemprop", "description").first() - ?.html()?.substringAfter(""), - chapters = root.selectFirst("div.chapterbox")?.select("ul.sub_vol_ul > li") - ?.asReversed()?.mapChapters { i, li -> - val a = li.selectFirst("a.chapter_list_a") - val href = a?.attrAsRelativeUrlOrNull("href") - ?.replace("%20", " ") ?: li.parseFailed("Link not found") - MangaChapter( - id = generateUid(href), - name = a.text(), - number = i + 1, - url = href, - uploadDate = parseChapterDateByLang(li.selectFirst("span")?.text().orEmpty()), - source = source, - scanlator = null, - branch = null, - ) - }, - ) - } + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet( + manga.url.toAbsoluteUrl(domain) + "?waring=1", + ).parseHtml() + val root = doc.body().selectFirstOrThrow("div.manga") + val infoRoot = root.selectFirstOrThrow("div.bookintro") + return manga.copy( + tags = infoRoot.getElementsByAttributeValue("itemprop", "genre").first() + ?.select("a")?.mapToSet { a -> + MangaTag( + title = a.text().toTitleCase(), + key = a.attr("href").substringBetween("/", "."), + source = source, + ) + }.orEmpty(), + author = infoRoot.getElementsByAttributeValue("itemprop", "author").first()?.text(), + state = parseStatus(infoRoot.select("li a.red").text()), + description = infoRoot.getElementsByAttributeValue("itemprop", "description").first() + ?.html()?.substringAfter(""), + chapters = root.selectFirst("div.chapterbox")?.select("ul.sub_vol_ul > li") + ?.asReversed()?.mapChapters { i, li -> + val a = li.selectFirst("a.chapter_list_a") + val href = a?.attrAsRelativeUrlOrNull("href") + ?.replace("%20", " ") ?: li.parseFailed("Link not found") + MangaChapter( + id = generateUid(href), + name = a.text(), + number = i + 1, + url = href, + uploadDate = parseChapterDateByLang(li.selectFirst("span")?.text().orEmpty()), + source = source, + scanlator = null, + branch = null, + ) + }, + ) + } - override suspend fun getPages(chapter: MangaChapter): List { - val doc = context.httpGet(chapter.url.toAbsoluteUrl(getDomain()), headers).parseHtml() - return doc.body().getElementById("page")?.select("option")?.map { option -> - val url = option.attr("value") - MangaPage( - id = generateUid(url), - url = url, - referer = chapter.url.toAbsoluteUrl(getDomain()), - preview = null, - source = source, - ) - } ?: doc.parseFailed("Pages list not found") - } + override suspend fun getPages(chapter: MangaChapter): List { + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + return doc.body().getElementById("page")?.select("option")?.map { option -> + val url = option.attr("value") + MangaPage( + id = generateUid(url), + url = url, + referer = chapter.url.toAbsoluteUrl(domain), + preview = null, + source = source, + ) + } ?: doc.parseFailed("Pages list not found") + } - override suspend fun getPageUrl(page: MangaPage): String { - val doc = context.httpGet(page.url.toAbsoluteUrl(getDomain()), headers).parseHtml() - val root = doc.body() - return root.selectFirst("a.pic_download")?.absUrl("href") - ?: doc.parseFailed("Page image not found") - } + override suspend fun getPageUrl(page: MangaPage): String { + val doc = webClient.httpGet(page.url.toAbsoluteUrl(domain)).parseHtml() + val root = doc.body() + return root.selectFirst("a.pic_download")?.absUrl("href") + ?: doc.parseFailed("Page image not found") + } - override suspend fun getTags(): Set { - val doc = context.httpGet("https://${getDomain()}/search/?type=high", headers) - .parseHtml() - val root = doc.body().getElementById("search_form") - return root?.select("li.cate_list")?.mapNotNullToSet { li -> - val cateId = li.attr("cate_id") ?: return@mapNotNullToSet null - val a = li.selectFirst("a") ?: return@mapNotNullToSet null - MangaTag( - title = a.text().toTitleCase(), - key = cateId, - source = source, - ) - } ?: doc.parseFailed("Root not found") - } + override suspend fun getTags(): Set { + val doc = webClient.httpGet("https://${domain}/search/?type=high") + .parseHtml() + val root = doc.body().getElementById("search_form") + return root?.select("li.cate_list")?.mapNotNullToSet { li -> + val cateId = li.attr("cate_id") ?: return@mapNotNullToSet null + val a = li.selectFirst("a") ?: return@mapNotNullToSet null + MangaTag( + title = a.text().toTitleCase(), + key = cateId, + source = source, + ) + } ?: doc.parseFailed("Root not found") + } - private fun parseStatus(status: String) = when { - status.contains("Ongoing") -> MangaState.ONGOING - status.contains("Completed") -> MangaState.FINISHED - else -> null - } + private fun parseStatus(status: String) = when { + status.contains("Ongoing") -> MangaState.ONGOING + status.contains("Completed") -> MangaState.FINISHED + else -> null + } - private fun parseChapterDateByLang(date: String): Long { - val dateWords = date.split(" ") + private fun parseChapterDateByLang(date: String): Long { + val dateWords = date.split(" ") - if (dateWords.size == 3) { - if (dateWords[1].contains(",")) { - SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH).tryParse(date) - } else { - val timeAgo = Integer.parseInt(dateWords[0]) - return Calendar.getInstance().apply { - when (dateWords[1]) { - "minutes" -> Calendar.MINUTE // EN-FR - "hours" -> Calendar.HOUR // EN + if (dateWords.size == 3) { + if (dateWords[1].contains(",")) { + SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH).tryParse(date) + } else { + val timeAgo = Integer.parseInt(dateWords[0]) + return Calendar.getInstance().apply { + when (dateWords[1]) { + "minutes" -> Calendar.MINUTE // EN-FR + "hours" -> Calendar.HOUR // EN - "minutos" -> Calendar.MINUTE // ES - "horas" -> Calendar.HOUR + "minutos" -> Calendar.MINUTE // ES + "horas" -> Calendar.HOUR - // "minutos" -> Calendar.MINUTE // BR - "hora" -> Calendar.HOUR + // "minutos" -> Calendar.MINUTE // BR + "hora" -> Calendar.HOUR - "минут" -> Calendar.MINUTE // RU - "часа" -> Calendar.HOUR + "минут" -> Calendar.MINUTE // RU + "часа" -> Calendar.HOUR - "Stunden" -> Calendar.HOUR // DE + "Stunden" -> Calendar.HOUR // DE - "minuti" -> Calendar.MINUTE // IT - "ore" -> Calendar.HOUR + "minuti" -> Calendar.MINUTE // IT + "ore" -> Calendar.HOUR - "heures" -> Calendar.HOUR // FR ("minutes" also French word) - else -> null - }?.let { - add(it, -timeAgo) - } - }.timeInMillis - } - } - return 0L - } + "heures" -> Calendar.HOUR // FR ("minutes" also French word) + else -> null + }?.let { + add(it, -timeAgo) + } + }.timeInMillis + } + } + return 0L + } - @MangaSourceParser("NINEMANGA_EN", "NineManga English", "en") - class English(context: MangaLoaderContext) : NineMangaParser( - context, - MangaSource.NINEMANGA_EN, - "www.ninemanga.com", - ) + @MangaSourceParser("NINEMANGA_EN", "NineManga English", "en") + class English(context: MangaLoaderContext) : NineMangaParser( + context, + MangaSource.NINEMANGA_EN, + "www.ninemanga.com", + ) - @MangaSourceParser("NINEMANGA_ES", "NineManga Español", "es") - class Spanish(context: MangaLoaderContext) : NineMangaParser( - context, - MangaSource.NINEMANGA_ES, - "es.ninemanga.com", - ) + @MangaSourceParser("NINEMANGA_ES", "NineManga Español", "es") + class Spanish(context: MangaLoaderContext) : NineMangaParser( + context, + MangaSource.NINEMANGA_ES, + "es.ninemanga.com", + ) - @MangaSourceParser("NINEMANGA_RU", "NineManga Русский", "ru") - class Russian(context: MangaLoaderContext) : NineMangaParser( - context, - MangaSource.NINEMANGA_RU, - "ru.ninemanga.com", - ) + @MangaSourceParser("NINEMANGA_RU", "NineManga Русский", "ru") + class Russian(context: MangaLoaderContext) : NineMangaParser( + context, + MangaSource.NINEMANGA_RU, + "ru.ninemanga.com", + ) - @MangaSourceParser("NINEMANGA_DE", "NineManga Deutsch", "de") - class Deutsch(context: MangaLoaderContext) : NineMangaParser( - context, - MangaSource.NINEMANGA_DE, - "de.ninemanga.com", - ) + @MangaSourceParser("NINEMANGA_DE", "NineManga Deutsch", "de") + class Deutsch(context: MangaLoaderContext) : NineMangaParser( + context, + MangaSource.NINEMANGA_DE, + "de.ninemanga.com", + ) - @MangaSourceParser("NINEMANGA_BR", "NineManga Brasil", "pt") - class Brazil(context: MangaLoaderContext) : NineMangaParser( - context, - MangaSource.NINEMANGA_BR, - "br.ninemanga.com", - ) + @MangaSourceParser("NINEMANGA_BR", "NineManga Brasil", "pt") + class Brazil(context: MangaLoaderContext) : NineMangaParser( + context, + MangaSource.NINEMANGA_BR, + "br.ninemanga.com", + ) - @MangaSourceParser("NINEMANGA_IT", "NineManga Italiano", "it") - class Italiano(context: MangaLoaderContext) : NineMangaParser( - context, - MangaSource.NINEMANGA_IT, - "it.ninemanga.com", - ) + @MangaSourceParser("NINEMANGA_IT", "NineManga Italiano", "it") + class Italiano(context: MangaLoaderContext) : NineMangaParser( + context, + MangaSource.NINEMANGA_IT, + "it.ninemanga.com", + ) - @MangaSourceParser("NINEMANGA_FR", "NineManga Français", "fr") - class Francais(context: MangaLoaderContext) : NineMangaParser( - context, - MangaSource.NINEMANGA_FR, - "fr.ninemanga.com", - ) -} \ No newline at end of file + @MangaSourceParser("NINEMANGA_FR", "NineManga Français", "fr") + class Francais(context: MangaLoaderContext) : NineMangaParser( + context, + MangaSource.NINEMANGA_FR, + "fr.ninemanga.com", + ) +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NudeMoonParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NudeMoonParser.kt index 0bde4f09..6341dcd7 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NudeMoonParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NudeMoonParser.kt @@ -15,217 +15,217 @@ private const val MAX_THUMB_INDEX = 19 @MangaSourceParser("NUDEMOON", "Nude-Moon", "ru") internal class NudeMoonParser( - override val context: MangaLoaderContext, -) : MangaParser(MangaSource.NUDEMOON), MangaParserAuthProvider { - - override val configKeyDomain = ConfigKey.Domain( - defaultValue = "nude-moon.org", - presetValues = arrayOf("nude-moon.org", "nude-moon.net"), - ) - override val authUrl: String - get() = "https://${getDomain()}/index.php" - - override val isAuthorized: Boolean - get() { - return context.cookieJar.getCookies(getDomain()).any { - it.name == "fusion_user" - } - } - - override val sortOrders: Set = EnumSet.of( - SortOrder.NEWEST, - SortOrder.POPULARITY, - SortOrder.RATING, - ) - - init { - context.cookieJar.insertCookies( - getDomain(), - "NMfYa=1;", - "nm_mobile=0;", - ) - } - - override suspend fun getList( - offset: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - val domain = getDomain() - val url = when { - !query.isNullOrEmpty() -> "https://$domain/search?stext=${query.urlEncoded()}&rowstart=$offset" - !tags.isNullOrEmpty() -> tags.joinToString( - separator = "_", - prefix = "https://$domain/tags/", - postfix = "&rowstart=$offset", - transform = { it.key.urlEncoded() }, - ) - - else -> "https://$domain/all_manga?${getSortKey(sortOrder)}&rowstart=$offset" - } - val doc = context.httpGet(url).parseHtml() - val root = doc.body().run { - selectFirst("td.main-bg") ?: selectFirst("td.main-body") - } ?: doc.parseFailed("Cannot find root") - return root.select("table.news_pic2").mapNotNull { row -> - val a = row.selectFirst("td.bg_style1")?.selectFirst("a") - ?: return@mapNotNull null - val href = a.attrAsRelativeUrl("href") - val title = a.selectFirst("h2")?.text().orEmpty() - val info = row.selectFirst("td[width=100%]") ?: return@mapNotNull null - Manga( - id = generateUid(href), - url = href, - title = title.substringAfter(" / "), - altTitle = title.substringBefore(" / ", "") - .takeUnless { it.isBlank() }, - author = info.getElementsContainingOwnText("Автор:").firstOrNull() - ?.nextElementSibling()?.ownText(), - coverUrl = row.selectFirst("img.news_pic2")?.absUrl("data-src") - .orEmpty(), - tags = row.selectFirst("span.tag-links")?.select("a") - ?.mapToSet { - MangaTag( - title = it.text().toTitleCase(), - key = it.attr("href").substringAfterLast('/'), - source = source, - ) - }.orEmpty(), - source = source, - publicUrl = a.absUrl("href"), - rating = RATING_UNKNOWN, - isNsfw = true, - description = row.selectFirst("div.description")?.html(), - state = null, - ) - } - } - - override suspend fun getDetails(manga: Manga): Manga { - val body = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml().body() - val root = body.selectFirst("table.shoutbox") - ?: body.parseFailed("Cannot find root") - val info = root.select("div.tbl2") - val lastInfo = info.last() - return manga.copy( - largeCoverUrl = body.selectFirst("img.news_pic2")?.absUrl("src"), - description = info.select("div.blockquote").lastOrNull()?.html() ?: manga.description, - tags = info.select("span.tag-links").firstOrNull()?.select("a")?.mapToSet { - MangaTag( - title = it.text().toTitleCase(), - key = it.attr("href").substringAfterLast('/'), - source = source, - ) - }?.plus(manga.tags) ?: manga.tags, - author = lastInfo?.getElementsByAttributeValueContaining("href", "mangaka/")?.text() - ?: manga.author, - chapters = listOf( - MangaChapter( - id = manga.id, - url = getReadLink(manga.url), - source = source, - number = 1, - name = manga.title, - scanlator = lastInfo?.getElementsByAttributeValueContaining("href", "perevod/")?.text(), - uploadDate = lastInfo?.getElementsContainingOwnText("Дата:") - ?.firstOrNull() - ?.html() - ?.parseDate() ?: 0L, - branch = null, - ), - ), - ) - } - - override suspend fun getPages(chapter: MangaChapter): List { - val fullUrl = chapter.url.toAbsoluteUrl(getDomain()) - val doc = context.httpGet(fullUrl).parseHtml() - val mangaId = chapter.url.substringAfterLast('/').substringBefore('-').toIntOrNull() - - val script = doc.select("script").firstNotNullOfOrNull { - it.html().takeIf { x -> x.contains(" images = new ") } - } ?: if (isAuthorized) { - doc.parseFailed("Cannot find pages list") - } else { - throw AuthRequiredException(source) - } - val pagesRegex = Regex("images\\[(\\d+)].src\\s*=\\s*'([^']+)'", RegexOption.MULTILINE) - return pagesRegex.findAll(script).map { match -> - val i = match.groupValues[1].toInt() - val url = match.groupValues[2] - MangaPage( - id = generateUid(url), - url = url, - referer = fullUrl, - preview = if (i <= MAX_THUMB_INDEX && mangaId != null) { - val part2 = url.substringBeforeLast('/') - val part3 = url.substringAfterLast('/') - val part1 = part2.substringBeforeLast('/') - "$part1/thumb/$mangaId/thumb_$part3" - } else { - null - }, - source = source, - ) - }.toList() - } - - override suspend fun getTags(): Set { - val domain = getDomain() - val doc = context.httpGet("https://$domain/all_manga").parseHtml() - val root = doc.body().getElementsContainingOwnText("Поиск манги по тегам") - .firstOrNull()?.parents()?.find { it.tag().normalName() == "tbody" } - ?.selectFirst("td.textbox")?.selectFirst("td.small") - ?: doc.parseFailed("Tags root not found") - return root.select("a").mapToSet { - MangaTag( - title = it.text().toTitleCase(), - key = it.attr("href").substringAfterLast('/') - .removeSuffix("+"), - source = source, - ) - } - } - - override suspend fun getUsername(): String { - val body = context.httpGet("https://${getDomain()}/").parseHtml() - .body() - return body - .getElementsContainingOwnText("Профиль") - .firstOrNull() - ?.attr("href") - ?.substringAfterLast('/') - ?: run { - throw if (body.selectFirst("form[name=\"loginform\"]") != null) { - AuthRequiredException(source) - } else { - body.parseFailed("Cannot find username") - } - } - } - - override fun getFaviconUrl(): String { - return "https://${getDomain()}/favicon.jpg" - } - - private fun getSortKey(sortOrder: SortOrder) = - when (sortOrder) { - SortOrder.POPULARITY -> "views" - SortOrder.NEWEST -> "date" - SortOrder.RATING -> "like" - else -> "like" - } - - private fun String.parseDate(): Long { - val dateString = substringBetweenFirst("Дата:", "<")?.trim() ?: return 0 - val dateFormat = SimpleDateFormat("d MMMM yyyy", Locale("ru")) - return dateFormat.tryParse(dateString) - } - - private fun getReadLink(url: String): String { - val prefix = url.substringBefore('-', "") - val suffix = url.substringAfter('-').trimStart('-') - return "$prefix-online-$suffix" - } -} \ No newline at end of file + context: MangaLoaderContext, +) : MangaParser(context, MangaSource.NUDEMOON), MangaParserAuthProvider { + + override val configKeyDomain = ConfigKey.Domain( + defaultValue = "nude-moon.org", + presetValues = arrayOf("nude-moon.org", "nude-moon.net"), + ) + override val authUrl: String + get() = "https://${domain}/index.php" + + override val isAuthorized: Boolean + get() { + return context.cookieJar.getCookies(domain).any { + it.name == "fusion_user" + } + } + + override val sortOrders: Set = EnumSet.of( + SortOrder.NEWEST, + SortOrder.POPULARITY, + SortOrder.RATING, + ) + + init { + context.cookieJar.insertCookies( + domain, + "NMfYa=1;", + "nm_mobile=0;", + ) + } + + override suspend fun getList( + offset: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + val domain = domain + val url = when { + !query.isNullOrEmpty() -> "https://$domain/search?stext=${query.urlEncoded()}&rowstart=$offset" + !tags.isNullOrEmpty() -> tags.joinToString( + separator = "_", + prefix = "https://$domain/tags/", + postfix = "&rowstart=$offset", + transform = { it.key.urlEncoded() }, + ) + + else -> "https://$domain/all_manga?${getSortKey(sortOrder)}&rowstart=$offset" + } + val doc = webClient.httpGet(url).parseHtml() + val root = doc.body().run { + selectFirst("td.main-bg") ?: selectFirst("td.main-body") + } ?: doc.parseFailed("Cannot find root") + return root.select("table.news_pic2").mapNotNull { row -> + val a = row.selectFirst("td.bg_style1")?.selectFirst("a") + ?: return@mapNotNull null + val href = a.attrAsRelativeUrl("href") + val title = a.selectFirst("h2")?.text().orEmpty() + val info = row.selectFirst("td[width=100%]") ?: return@mapNotNull null + Manga( + id = generateUid(href), + url = href, + title = title.substringAfter(" / "), + altTitle = title.substringBefore(" / ", "") + .takeUnless { it.isBlank() }, + author = info.getElementsContainingOwnText("Автор:").firstOrNull() + ?.nextElementSibling()?.ownText(), + coverUrl = row.selectFirst("img.news_pic2")?.absUrl("data-src") + .orEmpty(), + tags = row.selectFirst("span.tag-links")?.select("a") + ?.mapToSet { + MangaTag( + title = it.text().toTitleCase(), + key = it.attr("href").substringAfterLast('/'), + source = source, + ) + }.orEmpty(), + source = source, + publicUrl = a.absUrl("href"), + rating = RATING_UNKNOWN, + isNsfw = true, + description = row.selectFirst("div.description")?.html(), + state = null, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val body = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml().body() + val root = body.selectFirst("table.shoutbox") + ?: body.parseFailed("Cannot find root") + val info = root.select("div.tbl2") + val lastInfo = info.last() + return manga.copy( + largeCoverUrl = body.selectFirst("img.news_pic2")?.absUrl("src"), + description = info.select("div.blockquote").lastOrNull()?.html() ?: manga.description, + tags = info.select("span.tag-links").firstOrNull()?.select("a")?.mapToSet { + MangaTag( + title = it.text().toTitleCase(), + key = it.attr("href").substringAfterLast('/'), + source = source, + ) + }?.plus(manga.tags) ?: manga.tags, + author = lastInfo?.getElementsByAttributeValueContaining("href", "mangaka/")?.text() + ?: manga.author, + chapters = listOf( + MangaChapter( + id = manga.id, + url = getReadLink(manga.url), + source = source, + number = 1, + name = manga.title, + scanlator = lastInfo?.getElementsByAttributeValueContaining("href", "perevod/")?.text(), + uploadDate = lastInfo?.getElementsContainingOwnText("Дата:") + ?.firstOrNull() + ?.html() + ?.parseDate() ?: 0L, + branch = null, + ), + ), + ) + } + + override suspend fun getPages(chapter: MangaChapter): List { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + val mangaId = chapter.url.substringAfterLast('/').substringBefore('-').toIntOrNull() + + val script = doc.select("script").firstNotNullOfOrNull { + it.html().takeIf { x -> x.contains(" images = new ") } + } ?: if (isAuthorized) { + doc.parseFailed("Cannot find pages list") + } else { + throw AuthRequiredException(source) + } + val pagesRegex = Regex("images\\[(\\d+)].src\\s*=\\s*'([^']+)'", RegexOption.MULTILINE) + return pagesRegex.findAll(script).map { match -> + val i = match.groupValues[1].toInt() + val url = match.groupValues[2] + MangaPage( + id = generateUid(url), + url = url, + referer = fullUrl, + preview = if (i <= MAX_THUMB_INDEX && mangaId != null) { + val part2 = url.substringBeforeLast('/') + val part3 = url.substringAfterLast('/') + val part1 = part2.substringBeforeLast('/') + "$part1/thumb/$mangaId/thumb_$part3" + } else { + null + }, + source = source, + ) + }.toList() + } + + override suspend fun getTags(): Set { + val domain = domain + val doc = webClient.httpGet("https://$domain/all_manga").parseHtml() + val root = doc.body().getElementsContainingOwnText("Поиск манги по тегам") + .firstOrNull()?.parents()?.find { it.tag().normalName() == "tbody" } + ?.selectFirst("td.textbox")?.selectFirst("td.small") + ?: doc.parseFailed("Tags root not found") + return root.select("a").mapToSet { + MangaTag( + title = it.text().toTitleCase(), + key = it.attr("href").substringAfterLast('/') + .removeSuffix("+"), + source = source, + ) + } + } + + override suspend fun getUsername(): String { + val body = webClient.httpGet("https://${domain}/").parseHtml() + .body() + return body + .getElementsContainingOwnText("Профиль") + .firstOrNull() + ?.attr("href") + ?.substringAfterLast('/') + ?: run { + throw if (body.selectFirst("form[name=\"loginform\"]") != null) { + AuthRequiredException(source) + } else { + body.parseFailed("Cannot find username") + } + } + } + + override fun getFaviconUrl(): String { + return "https://${domain}/favicon.jpg" + } + + private fun getSortKey(sortOrder: SortOrder) = + when (sortOrder) { + SortOrder.POPULARITY -> "views" + SortOrder.NEWEST -> "date" + SortOrder.RATING -> "like" + else -> "like" + } + + private fun String.parseDate(): Long { + val dateString = substringBetweenFirst("Дата:", "<")?.trim() ?: return 0 + val dateFormat = SimpleDateFormat("d MMMM yyyy", Locale("ru")) + return dateFormat.tryParse(dateString) + } + + private fun getReadLink(url: String): String { + val prefix = url.substringBefore('-', "") + val suffix = url.substringAfter('-').trimStart('-') + return "$prefix-online-$suffix" + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/RemangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/RemangaParser.kt index 48f4530e..fa3a24dc 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/RemangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/RemangaParser.kt @@ -28,259 +28,259 @@ private const val STATUS_FINISHED = 0 @MangaSourceParser("REMANGA", "Remanga", "ru") internal class RemangaParser( - override val context: MangaLoaderContext, -) : PagedMangaParser(MangaSource.REMANGA, PAGE_SIZE), MangaParserAuthProvider { + context: MangaLoaderContext, +) : PagedMangaParser(context, MangaSource.REMANGA, PAGE_SIZE), MangaParserAuthProvider { - override val headers = Headers.Builder() - .add("User-Agent", "Mozilla/5.0 (Android 13; Mobile; rv:68.0) Gecko/68.0 Firefox/109.0") - .build() + private val baseHeaders = Headers.Builder() + .add("User-Agent", "Mozilla/5.0 (Android 13; Mobile; rv:68.0) Gecko/68.0 Firefox/109.0") + .build() - override val configKeyDomain = ConfigKey.Domain("remanga.org", null) - override val authUrl: String - get() = "https://${getDomain()}/user/login" + override val headers + get() = getApiHeaders() - override val sortOrders: Set = EnumSet.of( - SortOrder.UPDATED, - SortOrder.POPULARITY, - SortOrder.RATING, - SortOrder.NEWEST, - ) + override val configKeyDomain = ConfigKey.Domain("remanga.org", null) + override val authUrl: String + get() = "https://${domain}/user/login" - override val isAuthorized: Boolean - get() { - return context.cookieJar.getCookies(getDomain()).any { - it.name == "user" - } - } + override val sortOrders: Set = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.RATING, + SortOrder.NEWEST, + ) - private val regexLastUrlPath = Regex("/[^/]+/?$") + override val isAuthorized: Boolean + get() { + return context.cookieJar.getCookies(domain).any { + it.name == "user" + } + } - override suspend fun getListPage( - page: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - copyCookies() - val domain = getDomain() - val urlBuilder = StringBuilder() - .append("https://api.") - .append(domain) - if (query != null) { - urlBuilder.append("/api/search/?query=") - .append(query.urlEncoded()) - } else { - urlBuilder.append("/api/search/catalog/?ordering=") - .append(getSortKey(sortOrder)) - tags?.forEach { tag -> - urlBuilder.append("&genres=") - urlBuilder.append(tag.key) - } - } - urlBuilder - .append("&page=") - .append(page) - .append("&count=") - .append(PAGE_SIZE) - val content = context.httpGet(urlBuilder.toString(), getApiHeaders()).parseJson() - .getJSONArray("content") - return content.mapJSON { jo -> - val url = "/manga/${jo.getString("dir")}" - val img = jo.getJSONObject("img") - Manga( - id = generateUid(url), - url = url, - publicUrl = "https://$domain$url", - title = jo.getString("rus_name"), - altTitle = jo.getString("en_name"), - rating = jo.getString("avg_rating").toFloatOrNull()?.div(10f) ?: RATING_UNKNOWN, - coverUrl = "https://api.$domain${img.getString("mid")}", - largeCoverUrl = "https://api.$domain${img.getString("high")}", - author = null, - isNsfw = false, - state = null, - tags = jo.optJSONArray("genres")?.mapJSONToSet { g -> - MangaTag( - title = g.getString("name").toTitleCase(), - key = g.getInt("id").toString(), - source = MangaSource.REMANGA, - ) - }.orEmpty(), - source = MangaSource.REMANGA, - ) - } - } + private val regexLastUrlPath = Regex("/[^/]+/?$") - override suspend fun getDetails(manga: Manga): Manga { - copyCookies() - val domain = getDomain() - val slug = manga.url.find(regexLastUrlPath) - ?: throw ParseException("Cannot obtain slug from ${manga.url}", manga.publicUrl) - val data = context.httpGet( - url = "https://api.$domain/api/titles$slug/", - headers = getApiHeaders(), - ).handle401().parseJson() - val content = try { - data.getJSONObject("content") - } catch (e: JSONException) { - throw ParseException(data.optString("msg"), manga.publicUrl, e) - } - val branchId = content.getJSONArray("branches").optJSONObject(0) - ?.getLong("id") ?: throw ParseException("No branches found", manga.publicUrl) - val chapters = grabChapters(domain, branchId) - val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.US) - return manga.copy( - description = content.getString("description"), - state = when (content.optJSONObject("status")?.getInt("id")) { - STATUS_ONGOING -> MangaState.ONGOING - STATUS_FINISHED -> MangaState.FINISHED - else -> null - }, - tags = content.getJSONArray("genres").mapJSONToSet { g -> - MangaTag( - title = g.getString("name").toTitleCase(), - key = g.getInt("id").toString(), - source = MangaSource.REMANGA, - ) - }, - chapters = chapters.mapChapters { i, jo -> - if ( - jo.getBooleanOrDefault("is_paid", false) && - !jo.getBooleanOrDefault("is_bought", false) - ) { - return@mapChapters null - } - val id = jo.getLong("id") - val name = jo.getString("name").toTitleCase(Locale.ROOT) - val publishers = jo.optJSONArray("publishers") - MangaChapter( - id = generateUid(id), - url = "/api/titles/chapters/$id/", - number = chapters.size - i, - name = buildString { - append("Том ") - append(jo.optString("tome", "0")) - append(". ") - append("Глава ") - append(jo.optString("chapter", "0")) - if (name.isNotEmpty()) { - append(" - ") - append(name) - } - }, - uploadDate = dateFormat.tryParse(jo.getString("upload_date")), - scanlator = publishers?.optJSONObject(0)?.getStringOrNull("name"), - source = MangaSource.REMANGA, - branch = null, - ) - }.asReversed(), - ) - } + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + copyCookies() + val domain = domain + val urlBuilder = StringBuilder() + .append("https://api.") + .append(domain) + if (query != null) { + urlBuilder.append("/api/search/?query=") + .append(query.urlEncoded()) + } else { + urlBuilder.append("/api/search/catalog/?ordering=") + .append(getSortKey(sortOrder)) + tags?.forEach { tag -> + urlBuilder.append("&genres=") + urlBuilder.append(tag.key) + } + } + urlBuilder + .append("&page=") + .append(page) + .append("&count=") + .append(PAGE_SIZE) + val content = webClient.httpGet(urlBuilder.toString()).parseJson() + .getJSONArray("content") + return content.mapJSON { jo -> + val url = "/manga/${jo.getString("dir")}" + val img = jo.getJSONObject("img") + Manga( + id = generateUid(url), + url = url, + publicUrl = "https://$domain$url", + title = jo.getString("rus_name"), + altTitle = jo.getString("en_name"), + rating = jo.getString("avg_rating").toFloatOrNull()?.div(10f) ?: RATING_UNKNOWN, + coverUrl = "https://api.$domain${img.getString("mid")}", + largeCoverUrl = "https://api.$domain${img.getString("high")}", + author = null, + isNsfw = false, + state = null, + tags = jo.optJSONArray("genres")?.mapJSONToSet { g -> + MangaTag( + title = g.getString("name").toTitleCase(), + key = g.getInt("id").toString(), + source = MangaSource.REMANGA, + ) + }.orEmpty(), + source = MangaSource.REMANGA, + ) + } + } - override suspend fun getPages(chapter: MangaChapter): List { - val referer = "https://${getDomain()}/" - val content = context.httpGet(chapter.url.toAbsoluteUrl(getDomain("api")), getApiHeaders()) - .handle401() - .parseJson() - .getJSONObject("content") - val pages = content.optJSONArray("pages") - if (pages == null) { - val pubDate = content.getStringOrNull("pub_date")?.let { - SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.US).tryParse(it) - } - if (pubDate != null && pubDate > System.currentTimeMillis()) { - val at = SimpleDateFormat.getDateInstance(DateFormat.LONG).format(Date(pubDate)) - throw ContentUnavailableException("Глава станет доступной $at") - } else { - throw ContentUnavailableException("Глава недоступна") - } - } - val result = ArrayList(pages.length()) - for (i in 0 until pages.length()) { - when (val item = pages.get(i)) { - is JSONObject -> result += parsePage(item, referer) - is JSONArray -> item.mapJSONTo(result) { parsePage(it, referer) } - else -> throw ParseException("Unknown json item $item", chapter.url) - } - } - return result - } + override suspend fun getDetails(manga: Manga): Manga { + copyCookies() + val domain = domain + val slug = manga.url.find(regexLastUrlPath) + ?: throw ParseException("Cannot obtain slug from ${manga.url}", manga.publicUrl) + val data = webClient.httpGet( + url = "https://api.$domain/api/titles$slug/", + ).handle401().parseJson() + val content = try { + data.getJSONObject("content") + } catch (e: JSONException) { + throw ParseException(data.optString("msg"), manga.publicUrl, e) + } + val branchId = content.getJSONArray("branches").optJSONObject(0) + ?.getLong("id") ?: throw ParseException("No branches found", manga.publicUrl) + val chapters = grabChapters(domain, branchId) + val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.US) + return manga.copy( + description = content.getString("description"), + state = when (content.optJSONObject("status")?.getInt("id")) { + STATUS_ONGOING -> MangaState.ONGOING + STATUS_FINISHED -> MangaState.FINISHED + else -> null + }, + tags = content.getJSONArray("genres").mapJSONToSet { g -> + MangaTag( + title = g.getString("name").toTitleCase(), + key = g.getInt("id").toString(), + source = MangaSource.REMANGA, + ) + }, + chapters = chapters.mapChapters { i, jo -> + if ( + jo.getBooleanOrDefault("is_paid", false) && + !jo.getBooleanOrDefault("is_bought", false) + ) { + return@mapChapters null + } + val id = jo.getLong("id") + val name = jo.getString("name").toTitleCase(Locale.ROOT) + val publishers = jo.optJSONArray("publishers") + MangaChapter( + id = generateUid(id), + url = "/api/titles/chapters/$id/", + number = chapters.size - i, + name = buildString { + append("Том ") + append(jo.optString("tome", "0")) + append(". ") + append("Глава ") + append(jo.optString("chapter", "0")) + if (name.isNotEmpty()) { + append(" - ") + append(name) + } + }, + uploadDate = dateFormat.tryParse(jo.getString("upload_date")), + scanlator = publishers?.optJSONObject(0)?.getStringOrNull("name"), + source = MangaSource.REMANGA, + branch = null, + ) + }.asReversed(), + ) + } - override suspend fun getTags(): Set { - val domain = getDomain() - val content = context.httpGet("https://api.$domain/api/forms/titles/?get=genres", getApiHeaders()) - .parseJson().getJSONObject("content").getJSONArray("genres") - return content.mapJSONToSet { jo -> - MangaTag( - title = jo.getString("name").toTitleCase(), - key = jo.getInt("id").toString(), - source = source, - ) - } - } + override suspend fun getPages(chapter: MangaChapter): List { + val referer = "https://${domain}/" + val content = webClient.httpGet(chapter.url.toAbsoluteUrl(getDomain("api"))) + .handle401() + .parseJson() + .getJSONObject("content") + val pages = content.optJSONArray("pages") + if (pages == null) { + val pubDate = content.getStringOrNull("pub_date")?.let { + SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.US).tryParse(it) + } + if (pubDate != null && pubDate > System.currentTimeMillis()) { + val at = SimpleDateFormat.getDateInstance(DateFormat.LONG).format(Date(pubDate)) + throw ContentUnavailableException("Глава станет доступной $at") + } else { + throw ContentUnavailableException("Глава недоступна") + } + } + val result = ArrayList(pages.length()) + for (i in 0 until pages.length()) { + when (val item = pages.get(i)) { + is JSONObject -> result += parsePage(item, referer) + is JSONArray -> item.mapJSONTo(result) { parsePage(it, referer) } + else -> throw ParseException("Unknown json item $item", chapter.url) + } + } + return result + } - override suspend fun getUsername(): String { - val jo = context.httpGet( - url = "https://api.${getDomain()}/api/users/current/", - headers = getApiHeaders(), - ).handle401().parseJson() - return jo.getJSONObject("content").getString("username") - } + override suspend fun getTags(): Set { + val domain = domain + val content = webClient.httpGet("https://api.$domain/api/forms/titles/?get=genres") + .parseJson().getJSONObject("content").getJSONArray("genres") + return content.mapJSONToSet { jo -> + MangaTag( + title = jo.getString("name").toTitleCase(), + key = jo.getInt("id").toString(), + source = source, + ) + } + } - private fun getApiHeaders(): Headers { - val userCookie = context.cookieJar.getCookies(getDomain()).find { - it.name == "user" - } ?: return headers - val jo = JSONObject(URLDecoder.decode(userCookie.value, Charsets.UTF_8.name())) - val accessToken = jo.getStringOrNull("access_token") ?: return headers - return headers.newBuilder().add("authorization", "bearer $accessToken").build() - } + override suspend fun getUsername(): String { + val jo = webClient.httpGet( + url = "https://api.${domain}/api/users/current/", + ).handle401().parseJson() + return jo.getJSONObject("content").getString("username") + } - private fun copyCookies() { - val domain = getDomain() - context.cookieJar.copyCookies(domain, "api.$domain") - } + private fun getApiHeaders(): Headers { + val userCookie = context.cookieJar.getCookies(domain).find { + it.name == "user" + } ?: return baseHeaders + val jo = JSONObject(URLDecoder.decode(userCookie.value, Charsets.UTF_8.name())) + val accessToken = jo.getStringOrNull("access_token") ?: return baseHeaders + return baseHeaders.newBuilder().add("authorization", "bearer $accessToken").build() + } - private fun getSortKey(order: SortOrder?) = when (order) { - SortOrder.UPDATED -> "-chapter_date" - SortOrder.POPULARITY -> "-rating" - SortOrder.RATING -> "-votes" - SortOrder.NEWEST -> "-id" - else -> "-chapter_date" - } + private fun copyCookies() { + val domain = domain + context.cookieJar.copyCookies(domain, "api.$domain") + } - private fun parsePage(jo: JSONObject, referer: String) = MangaPage( - id = generateUid(jo.getLong("id")), - url = jo.getString("link"), - preview = null, - referer = referer, - source = source, - ) + private fun getSortKey(order: SortOrder?) = when (order) { + SortOrder.UPDATED -> "-chapter_date" + SortOrder.POPULARITY -> "-rating" + SortOrder.RATING -> "-votes" + SortOrder.NEWEST -> "-id" + else -> "-chapter_date" + } - private suspend fun grabChapters(domain: String, branchId: Long): List { - val result = ArrayList(100) - var page = 1 - while (true) { - val content = context.httpGet( - url = "https://api.$domain/api/titles/chapters/?branch_id=$branchId&page=$page&count=100", - headers = getApiHeaders(), - ).handle401().parseJson().getJSONArray("content") - val len = content.length() - if (len == 0) { - break - } - result.ensureCapacity(result.size + len) - for (i in 0 until len) { - result.add(content.getJSONObject(i)) - } - page++ - } - return result - } + private fun parsePage(jo: JSONObject, referer: String) = MangaPage( + id = generateUid(jo.getLong("id")), + url = jo.getString("link"), + preview = null, + referer = referer, + source = source, + ) - private fun Response.handle401() = apply { - if (code == HttpURLConnection.HTTP_UNAUTHORIZED) { - throw AuthRequiredException(source) - } - } -} \ No newline at end of file + private suspend fun grabChapters(domain: String, branchId: Long): List { + val result = ArrayList(100) + var page = 1 + while (true) { + val content = webClient.httpGet( + url = "https://api.$domain/api/titles/chapters/?branch_id=$branchId&page=$page&count=100", + ).handle401().parseJson().getJSONArray("content") + val len = content.length() + if (len == 0) { + break + } + result.ensureCapacity(result.size + len) + for (i in 0 until len) { + result.add(content.getJSONObject(i)) + } + page++ + } + return result + } + + private fun Response.handle401() = apply { + if (code == HttpURLConnection.HTTP_UNAUTHORIZED) { + throw AuthRequiredException(source) + } + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/UnionMangasParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/UnionMangasParser.kt index 7e1eb643..ea07c0fc 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/UnionMangasParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/UnionMangasParser.kt @@ -13,165 +13,165 @@ import java.text.SimpleDateFormat import java.util.* @MangaSourceParser("UNION_MANGAS", "Union Mangás", "pt") -class UnionMangasParser(override val context: MangaLoaderContext) : PagedMangaParser(MangaSource.UNION_MANGAS, 40) { +class UnionMangasParser(context: MangaLoaderContext) : PagedMangaParser(context, MangaSource.UNION_MANGAS, 40) { - override val sortOrders = EnumSet.of( - SortOrder.ALPHABETICAL, - SortOrder.POPULARITY, - ) + override val sortOrders = EnumSet.of( + SortOrder.ALPHABETICAL, + SortOrder.POPULARITY, + ) - override val configKeyDomain = ConfigKey.Domain("unionleitor.top", emptyArray()) + override val configKeyDomain = ConfigKey.Domain("unionleitor.top", emptyArray()) - override suspend fun getListPage( - page: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - if (!query.isNullOrEmpty()) { - return if (page == searchPaginator.firstPage) { - search(query) - } else { - emptyList() - } - } - val tag = tags.oneOrThrowIfMany() - val url = urlBuilder() - .addPathSegment("lista-mangas") - .addPathSegment( - when { - tag != null -> tag.key - sortOrder == SortOrder.ALPHABETICAL -> "a-z" - else -> "visualizacoes" - }, - ).addPathSegment(page.toString()) - val doc = context.httpGet(url.build()).parseHtml() - val root = doc.selectFirstOrThrow("div.tamanho-bloco-perfil") - return root.select(".lista-mangas-novos").map { div -> - val a = div.selectFirstOrThrow("a") - val img = div.selectFirstOrThrow("img") - val href = a.attrAsRelativeUrl("href") - Manga( - id = generateUid(href), - url = href, - publicUrl = a.attrAsAbsoluteUrl("href"), - title = div.selectLastOrThrow("a").text(), - coverUrl = img.attrAsAbsoluteUrl("src"), - altTitle = null, - rating = RATING_UNKNOWN, - tags = emptySet(), - description = div.selectLast("div")?.ownText(), - state = null, - author = null, - isNsfw = false, - source = source, - ) - } - } + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + if (!query.isNullOrEmpty()) { + return if (page == searchPaginator.firstPage) { + search(query) + } else { + emptyList() + } + } + val tag = tags.oneOrThrowIfMany() + val url = urlBuilder() + .addPathSegment("lista-mangas") + .addPathSegment( + when { + tag != null -> tag.key + sortOrder == SortOrder.ALPHABETICAL -> "a-z" + else -> "visualizacoes" + }, + ).addPathSegment(page.toString()) + val doc = webClient.httpGet(url.build()).parseHtml() + val root = doc.selectFirstOrThrow("div.tamanho-bloco-perfil") + return root.select(".lista-mangas-novos").map { div -> + val a = div.selectFirstOrThrow("a") + val img = div.selectFirstOrThrow("img") + val href = a.attrAsRelativeUrl("href") + Manga( + id = generateUid(href), + url = href, + publicUrl = a.attrAsAbsoluteUrl("href"), + title = div.selectLastOrThrow("a").text(), + coverUrl = img.attrAsAbsoluteUrl("src"), + altTitle = null, + rating = RATING_UNKNOWN, + tags = emptySet(), + description = div.selectLast("div")?.ownText(), + state = null, + author = null, + isNsfw = false, + source = source, + ) + } + } - override suspend fun getDetails(manga: Manga): Manga { - val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml() - val root = doc.selectFirstOrThrow(".perfil-manga") - val dateFormat = SimpleDateFormat("dd/MM/yyyy", Locale.ROOT) - return manga.copy( - rating = root.select("h2") - .find { it.ownText().startsWith('#') } - ?.ownText()?.drop(1)?.toFloatOrNull()?.div(10f) ?: manga.rating, - largeCoverUrl = root.selectFirst("img.img-thumbnail")?.attrAsAbsoluteUrlOrNull("src"), - description = root.selectFirst(".panel-default")?.selectFirst(".panel-body")?.html(), - author = root.tableValue("Autor")?.ownText(), - altTitle = root.tableValue("Título(s) Alternativo(s)")?.ownText(), - state = when (root.tableValue("Status")?.selectLast(".label")?.text()) { - "Completo" -> MangaState.FINISHED - "Ativo" -> MangaState.ONGOING - else -> null - }, - tags = root.tableValue("Gênero(s)")?.select("a")?.mapToSet { - it.toMangaTag() - } ?: manga.tags, - isNsfw = root.selectFirst(".alert-danger")?.html()?.contains("18 anos") == true, - chapters = root.select("div.row.capitulos").asReversed().mapChapters { i, div -> - val a = div.selectFirstOrThrow("a") - val href = a.attrAsRelativeUrl("href") - val title = a.text() - MangaChapter( - id = generateUid(href), - name = title, - number = i + 1, - url = href, - scanlator = div.selectLast("a")?.text()?.takeUnless { it == title }, - uploadDate = dateFormat.tryParse( - a.nextElementSibling()?.text()?.removeSurrounding("(", ")"), - ), - branch = null, - source = source, - ) - }, - ) - } + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val root = doc.selectFirstOrThrow(".perfil-manga") + val dateFormat = SimpleDateFormat("dd/MM/yyyy", Locale.ROOT) + return manga.copy( + rating = root.select("h2") + .find { it.ownText().startsWith('#') } + ?.ownText()?.drop(1)?.toFloatOrNull()?.div(10f) ?: manga.rating, + largeCoverUrl = root.selectFirst("img.img-thumbnail")?.attrAsAbsoluteUrlOrNull("src"), + description = root.selectFirst(".panel-default")?.selectFirst(".panel-body")?.html(), + author = root.tableValue("Autor")?.ownText(), + altTitle = root.tableValue("Título(s) Alternativo(s)")?.ownText(), + state = when (root.tableValue("Status")?.selectLast(".label")?.text()) { + "Completo" -> MangaState.FINISHED + "Ativo" -> MangaState.ONGOING + else -> null + }, + tags = root.tableValue("Gênero(s)")?.select("a")?.mapToSet { + it.toMangaTag() + } ?: manga.tags, + isNsfw = root.selectFirst(".alert-danger")?.html()?.contains("18 anos") == true, + chapters = root.select("div.row.capitulos").asReversed().mapChapters { i, div -> + val a = div.selectFirstOrThrow("a") + val href = a.attrAsRelativeUrl("href") + val title = a.text() + MangaChapter( + id = generateUid(href), + name = title, + number = i + 1, + url = href, + scanlator = div.selectLast("a")?.text()?.takeUnless { it == title }, + uploadDate = dateFormat.tryParse( + a.nextElementSibling()?.text()?.removeSurrounding("(", ")"), + ), + branch = null, + source = source, + ) + }, + ) + } - override suspend fun getPages(chapter: MangaChapter): List { - val fullUrl = chapter.url.toAbsoluteUrl(getDomain()) - val doc = context.httpGet(fullUrl).parseHtml() - val root = doc.body().selectFirstOrThrow("article") - return root.selectOrThrow("img[pag]").mapNotNull { img -> - val href = img.attrAsRelativeUrl("src") - if (href.startsWith("/images/banner")) { - return@mapNotNull null - } - MangaPage( - id = generateUid(href), - url = href, - referer = fullUrl, - preview = null, - source = source, - ) - } - } + override suspend fun getPages(chapter: MangaChapter): List { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + val root = doc.body().selectFirstOrThrow("article") + return root.selectOrThrow("img[pag]").mapNotNull { img -> + val href = img.attrAsRelativeUrl("src") + if (href.startsWith("/images/banner")) { + return@mapNotNull null + } + MangaPage( + id = generateUid(href), + url = href, + referer = fullUrl, + preview = null, + source = source, + ) + } + } - override suspend fun getTags(): Set { - val doc = context.httpGet(urlBuilder().addPathSegment("lista-mangas").build()).parseHtml() - val ul = doc.body().selectFirstOrThrow(".nav-tabs").selectFirstOrThrow("ul.dropdown-menu") - return ul.select("li").mapToSet { li -> - li.selectFirstOrThrow("a").toMangaTag() - } - } + override suspend fun getTags(): Set { + val doc = webClient.httpGet(urlBuilder().addPathSegment("lista-mangas").build()).parseHtml() + val ul = doc.body().selectFirstOrThrow(".nav-tabs").selectFirstOrThrow("ul.dropdown-menu") + return ul.select("li").mapToSet { li -> + li.selectFirstOrThrow("a").toMangaTag() + } + } - private suspend fun search(query: String): List { - val domain = getDomain() - val json = context.httpGet( - urlBuilder() - .addPathSegments("assets/busca.php") - .addQueryParameter("nomeManga", query) - .build(), - ).parseJson() - return json.getJSONArray("items").mapJSON { jo -> - val href = "/pagina-manga/" + jo.getString("url") - Manga( - id = generateUid(href), - url = href, - publicUrl = href.toAbsoluteUrl(domain), - title = jo.getString("titulo"), - rating = RATING_UNKNOWN, - tags = emptySet(), - author = jo.getStringOrNull("autor"), - coverUrl = jo.getString("imagem"), - state = null, - isNsfw = false, - altTitle = null, - source = source, - ) - } - } + private suspend fun search(query: String): List { + val domain = domain + val json = webClient.httpGet( + urlBuilder() + .addPathSegments("assets/busca.php") + .addQueryParameter("nomeManga", query) + .build(), + ).parseJson() + return json.getJSONArray("items").mapJSON { jo -> + val href = "/pagina-manga/" + jo.getString("url") + Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + title = jo.getString("titulo"), + rating = RATING_UNKNOWN, + tags = emptySet(), + author = jo.getStringOrNull("autor"), + coverUrl = jo.getString("imagem"), + state = null, + isNsfw = false, + altTitle = null, + source = source, + ) + } + } - private fun Element.tableValue(title: String): Element? { - return select("h4.media-heading") - .find { it.selectFirst("label.subtit-manga")?.text()?.contains(title, ignoreCase = true) == true } - } + private fun Element.tableValue(title: String): Element? { + return select("h4.media-heading") + .find { it.selectFirst("label.subtit-manga")?.text()?.contains(title, ignoreCase = true) == true } + } - private fun Element.toMangaTag() = MangaTag( - title = text().toTitleCase(sourceLocale ?: Locale.ROOT), - key = attr("href").removeSuffix('/').substringAfterLast('/'), - source = source, - ) -} \ No newline at end of file + private fun Element.toMangaTag() = MangaTag( + title = text().toTitleCase(sourceLocale ?: Locale.ROOT), + key = attr("href").removeSuffix('/').substringAfterLast('/'), + source = source, + ) +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt index 04fb602d..e8f3a94b 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt @@ -4,6 +4,7 @@ import okhttp3.Headers import okhttp3.HttpUrl.Companion.toHttpUrl import okhttp3.Response import org.json.JSONArray +import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaParser import org.koitharu.kotatsu.parsers.MangaParserAuthProvider import org.koitharu.kotatsu.parsers.exception.AuthRequiredException @@ -20,301 +21,298 @@ private const val NSFW_ALERT = "сексуальные сцены" private const val NOTHING_FOUND = "Ничего не найдено" internal abstract class GroupleParser( - source: MangaSource, - userAgent: String, - private val siteId: Int, -) : MangaParser(source), MangaParserAuthProvider { + context: MangaLoaderContext, + source: MangaSource, + userAgent: String, + private val siteId: Int, +) : MangaParser(context, source), MangaParserAuthProvider { - override val headers = Headers.Builder() - .add("User-Agent", userAgent) - .build() + override val headers = Headers.Builder() + .add("User-Agent", userAgent) + .build() - override val sortOrders: Set = EnumSet.of( - SortOrder.UPDATED, - SortOrder.POPULARITY, - SortOrder.NEWEST, - SortOrder.RATING, - ) + override val sortOrders: Set = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.NEWEST, + SortOrder.RATING, + ) - override val authUrl: String - get() { - val targetUri = "https://${getDomain()}/".urlEncoded() - return "https://grouple.co/internal/auth/sso?siteId=$siteId&=targetUri=$targetUri" - } + override val authUrl: String + get() { + val targetUri = "https://${domain}/".urlEncoded() + return "https://grouple.co/internal/auth/sso?siteId=$siteId&=targetUri=$targetUri" + } - override val isAuthorized: Boolean - get() = context.cookieJar.getCookies(getDomain()).any { it.name == "gwt" } + override val isAuthorized: Boolean + get() = context.cookieJar.getCookies(domain).any { it.name == "gwt" } - override suspend fun getList( - offset: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - val domain = getDomain() - val doc = when { - !query.isNullOrEmpty() -> context.httpPost( - "https://$domain/search", - mapOf( - "q" to query.urlEncoded(), - "offset" to (offset upBy PAGE_SIZE_SEARCH).toString(), - ), - headers, - ) + override suspend fun getList( + offset: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + val domain = domain + val doc = when { + !query.isNullOrEmpty() -> webClient.httpPost( + "https://$domain/search", + mapOf( + "q" to query.urlEncoded(), + "offset" to (offset upBy PAGE_SIZE_SEARCH).toString(), + ), + ) - tags.isNullOrEmpty() -> context.httpGet( - "https://$domain/list?sortType=${ - getSortKey(sortOrder) - }&offset=${offset upBy PAGE_SIZE}", - headers, - ) + tags.isNullOrEmpty() -> webClient.httpGet( + "https://$domain/list?sortType=${ + getSortKey(sortOrder) + }&offset=${offset upBy PAGE_SIZE}", + ) - tags.size == 1 -> context.httpGet( - "https://$domain/list/genre/${tags.first().key}?sortType=${ - getSortKey(sortOrder) - }&offset=${offset upBy PAGE_SIZE}", - headers, - ) + tags.size == 1 -> webClient.httpGet( + "https://$domain/list/genre/${tags.first().key}?sortType=${ + getSortKey(sortOrder) + }&offset=${offset upBy PAGE_SIZE}", + ) - offset > 0 -> return emptyList() - else -> advancedSearch(domain, tags) - }.parseHtml().body() - val root = (doc.getElementById("mangaBox") ?: doc.getElementById("mangaResults")) - ?: doc.parseFailed("Cannot find root") - val tiles = root.selectFirst("div.tiles.row") ?: if ( - root.select(".alert").any { it.ownText() == NOTHING_FOUND } - ) { - return emptyList() - } else { - doc.parseFailed("No tiles found") - } - val baseHost = root.baseUri().toHttpUrl().host - return tiles.select("div.tile").mapNotNull { node -> - val imgDiv = node.selectFirst("div.img") ?: return@mapNotNull null - val descDiv = node.selectFirst("div.desc") ?: return@mapNotNull null - if (descDiv.selectFirst("i.fa-user") != null) { - return@mapNotNull null // skip author - } - val href = imgDiv.selectFirst("a")?.attrAsAbsoluteUrlOrNull("href") - if (href == null || href.toHttpUrl().host != baseHost) { - return@mapNotNull null // skip external links - } - val title = descDiv.selectFirst("h3")?.selectFirst("a")?.text() - ?: return@mapNotNull null - val tileInfo = descDiv.selectFirst("div.tile-info") - val relUrl = href.toRelativeUrl(baseHost) - Manga( - id = generateUid(relUrl), - url = relUrl, - publicUrl = href, - title = title, - altTitle = descDiv.selectFirst("h4")?.text(), - coverUrl = imgDiv.selectFirst("img.lazy")?.attr("data-original")?.replace("_p.", ".").orEmpty(), - rating = runCatching { - node.selectFirst(".compact-rate") - ?.attr("title") - ?.toFloatOrNull() - ?.div(5f) - }.getOrNull() ?: RATING_UNKNOWN, - author = tileInfo?.selectFirst("a.person-link")?.text(), - isNsfw = false, - tags = runCatching { - tileInfo?.select("a.element-link") - ?.mapToSet { - MangaTag( - title = it.text().toTitleCase(), - key = it.attr("href").substringAfterLast('/'), - source = source, - ) - } - }.getOrNull().orEmpty(), - state = when { - node.selectFirst("div.tags") - ?.selectFirst("span.mangaCompleted") != null -> MangaState.FINISHED + offset > 0 -> return emptyList() + else -> advancedSearch(domain, tags) + }.parseHtml().body() + val root = (doc.getElementById("mangaBox") ?: doc.getElementById("mangaResults")) + ?: doc.parseFailed("Cannot find root") + val tiles = root.selectFirst("div.tiles.row") ?: if ( + root.select(".alert").any { it.ownText() == NOTHING_FOUND } + ) { + return emptyList() + } else { + doc.parseFailed("No tiles found") + } + val baseHost = root.baseUri().toHttpUrl().host + return tiles.select("div.tile").mapNotNull { node -> + val imgDiv = node.selectFirst("div.img") ?: return@mapNotNull null + val descDiv = node.selectFirst("div.desc") ?: return@mapNotNull null + if (descDiv.selectFirst("i.fa-user") != null) { + return@mapNotNull null // skip author + } + val href = imgDiv.selectFirst("a")?.attrAsAbsoluteUrlOrNull("href") + if (href == null || href.toHttpUrl().host != baseHost) { + return@mapNotNull null // skip external links + } + val title = descDiv.selectFirst("h3")?.selectFirst("a")?.text() + ?: return@mapNotNull null + val tileInfo = descDiv.selectFirst("div.tile-info") + val relUrl = href.toRelativeUrl(baseHost) + Manga( + id = generateUid(relUrl), + url = relUrl, + publicUrl = href, + title = title, + altTitle = descDiv.selectFirst("h4")?.text(), + coverUrl = imgDiv.selectFirst("img.lazy")?.attr("data-original")?.replace("_p.", ".").orEmpty(), + rating = runCatching { + node.selectFirst(".compact-rate") + ?.attr("title") + ?.toFloatOrNull() + ?.div(5f) + }.getOrNull() ?: RATING_UNKNOWN, + author = tileInfo?.selectFirst("a.person-link")?.text(), + isNsfw = false, + tags = runCatching { + tileInfo?.select("a.element-link") + ?.mapToSet { + MangaTag( + title = it.text().toTitleCase(), + key = it.attr("href").substringAfterLast('/'), + source = source, + ) + } + }.getOrNull().orEmpty(), + state = when { + node.selectFirst("div.tags") + ?.selectFirst("span.mangaCompleted") != null -> MangaState.FINISHED - else -> null - }, - source = source, - ) - } - } + else -> null + }, + source = source, + ) + } + } - override suspend fun getDetails(manga: Manga): Manga { - val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain()), headers).checkAuthRequired().parseHtml() - val root = doc.body().getElementById("mangaBox")?.selectFirst("div.leftContent") - ?: doc.parseFailed("Cannot find root") - val dateFormat = SimpleDateFormat("dd.MM.yy", Locale.US) - val coverImg = root.selectFirst("div.subject-cover")?.selectFirst("img") - return manga.copy( - description = root.selectFirst("div.manga-description")?.html(), - largeCoverUrl = coverImg?.attr("data-full"), - coverUrl = coverImg?.attr("data-thumb") ?: manga.coverUrl, - tags = manga.tags + root.select("div.subject-meta").select("span.elem_genre ") - .mapNotNull { - val a = it.selectFirst("a.element-link") ?: return@mapNotNull null - MangaTag( - title = a.text().toTitleCase(), - key = a.attr("href").substringAfterLast('/'), - source = source, - ) - }, - author = root.selectFirst("a.person-link")?.text() ?: manga.author, - isNsfw = root.select(".alert-warning").any { it.ownText().contains(NSFW_ALERT) }, - chapters = root.selectFirst("div.chapters-link")?.selectFirst("table") - ?.select("tr:has(td > a)")?.asReversed()?.mapChapters { i, tr -> - val a = tr.selectFirst("a.chapter-link") ?: return@mapChapters null - val href = a.attrAsRelativeUrl("href") - var translators = "" - val translatorElement = a.attr("title") - if (!translatorElement.isNullOrBlank()) { - translators = translatorElement - .replace("(Переводчик),", "&") - .removeSuffix(" (Переводчик)") - } - MangaChapter( - id = generateUid(href), - name = tr.selectFirst("a")?.text().orEmpty().removePrefix(manga.title).trim(), - number = i + 1, - url = href, - uploadDate = dateFormat.tryParse(tr.selectFirst("td.date")?.text()), - scanlator = translators, - source = source, - branch = null, - ) - }, - ) - } + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).checkAuthRequired().parseHtml() + val root = doc.body().getElementById("mangaBox")?.selectFirst("div.leftContent") + ?: doc.parseFailed("Cannot find root") + val dateFormat = SimpleDateFormat("dd.MM.yy", Locale.US) + val coverImg = root.selectFirst("div.subject-cover")?.selectFirst("img") + return manga.copy( + description = root.selectFirst("div.manga-description")?.html(), + largeCoverUrl = coverImg?.attr("data-full"), + coverUrl = coverImg?.attr("data-thumb") ?: manga.coverUrl, + tags = manga.tags + root.select("div.subject-meta").select("span.elem_genre ") + .mapNotNull { + val a = it.selectFirst("a.element-link") ?: return@mapNotNull null + MangaTag( + title = a.text().toTitleCase(), + key = a.attr("href").substringAfterLast('/'), + source = source, + ) + }, + author = root.selectFirst("a.person-link")?.text() ?: manga.author, + isNsfw = root.select(".alert-warning").any { it.ownText().contains(NSFW_ALERT) }, + chapters = root.selectFirst("div.chapters-link")?.selectFirst("table") + ?.select("tr:has(td > a)")?.asReversed()?.mapChapters { i, tr -> + val a = tr.selectFirst("a.chapter-link") ?: return@mapChapters null + val href = a.attrAsRelativeUrl("href") + var translators = "" + val translatorElement = a.attr("title") + if (!translatorElement.isNullOrBlank()) { + translators = translatorElement + .replace("(Переводчик),", "&") + .removeSuffix(" (Переводчик)") + } + MangaChapter( + id = generateUid(href), + name = tr.selectFirst("a")?.text().orEmpty().removePrefix(manga.title).trim(), + number = i + 1, + url = href, + uploadDate = dateFormat.tryParse(tr.selectFirst("td.date")?.text()), + scanlator = translators, + source = source, + branch = null, + ) + }, + ) + } - override suspend fun getPages(chapter: MangaChapter): List { - val doc = context.httpGet(chapter.url.toAbsoluteUrl(getDomain()) + "?mtr=1", headers) - .checkAuthRequired() - .parseHtml() - val scripts = doc.select("script") - for (script in scripts) { - val data = script.html() - val pos = data.indexOf("rm_h.readerInit( 0,") - if (pos == -1) { - continue - } - val json = data.substring(pos) - .substringAfter('(') - .substringBefore('\n') - .substringBeforeLast(')') - if (json.isEmpty()) { - continue - } - val ja = JSONArray("[$json]") - val pages = ja.getJSONArray(1) - val servers = ja.getJSONArray(3).mapJSON { it.getString("path") } - val serversStr = servers.joinToString("|") - return (0 until pages.length()).map { i -> - val page = pages.getJSONArray(i) - val primaryServer = page.getString(0) - val url = page.getString(2) - MangaPage( - id = generateUid(url), - url = "$primaryServer|$serversStr|$url", - preview = null, - referer = chapter.url, - source = source, - ) - } - } - doc.parseFailed("Pages list not found at ${chapter.url}") - } + override suspend fun getPages(chapter: MangaChapter): List { + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain) + "?mtr=1") + .checkAuthRequired() + .parseHtml() + val scripts = doc.select("script") + for (script in scripts) { + val data = script.html() + val pos = data.indexOf("rm_h.readerInit( 0,") + if (pos == -1) { + continue + } + val json = data.substring(pos) + .substringAfter('(') + .substringBefore('\n') + .substringBeforeLast(')') + if (json.isEmpty()) { + continue + } + val ja = JSONArray("[$json]") + val pages = ja.getJSONArray(1) + val servers = ja.getJSONArray(3).mapJSON { it.getString("path") } + val serversStr = servers.joinToString("|") + return (0 until pages.length()).map { i -> + val page = pages.getJSONArray(i) + val primaryServer = page.getString(0) + val url = page.getString(2) + MangaPage( + id = generateUid(url), + url = "$primaryServer|$serversStr|$url", + preview = null, + referer = chapter.url, + source = source, + ) + } + } + doc.parseFailed("Pages list not found at ${chapter.url}") + } - override suspend fun getPageUrl(page: MangaPage): String { - val parts = page.url.split('|') - val path = parts.last() - val servers = parts.dropLast(1).toSet() - val headers = Headers.headersOf("Referer", page.referer) - for (server in servers) { - val url = server + path - if (tryHead(url, headers)) { - return url - } - } - val fallbackServer = servers.firstOrNull() ?: throw ParseException("Cannot find any page url", page.url) - return fallbackServer + path - } + override suspend fun getPageUrl(page: MangaPage): String { + val parts = page.url.split('|') + val path = parts.last() + val servers = parts.dropLast(1).toSet() + for (server in servers) { + val url = server + path + if (tryHead(url)) { + return url + } + } + val fallbackServer = servers.firstOrNull() ?: throw ParseException("Cannot find any page url", page.url) + return fallbackServer + path + } - override suspend fun getTags(): Set { - val doc = context.httpGet("https://${getDomain()}/list/genres/sort_name", headers).parseHtml() - val root = doc.body().getElementById("mangaBox")?.selectFirst("div.leftContent") - ?.selectFirst("table.table") ?: doc.parseFailed("Cannot find root") - return root.select("a.element-link").mapToSet { a -> - MangaTag( - title = a.text().toTitleCase(), - key = a.attr("href").substringAfterLast('/'), - source = source, - ) - } - } + override suspend fun getTags(): Set { + val doc = webClient.httpGet("https://${domain}/list/genres/sort_name").parseHtml() + val root = doc.body().getElementById("mangaBox")?.selectFirst("div.leftContent") + ?.selectFirst("table.table") ?: doc.parseFailed("Cannot find root") + return root.select("a.element-link").mapToSet { a -> + MangaTag( + title = a.text().toTitleCase(), + key = a.attr("href").substringAfterLast('/'), + source = source, + ) + } + } - override suspend fun getUsername(): String { - val root = context.httpGet("https://grouple.co/").parseHtml().body() - val element = root.selectFirst("img.user-avatar") ?: throw AuthRequiredException(source) - val res = element.parent()?.text() - return if (res.isNullOrEmpty()) { - root.parseFailed("Cannot find username") - } else res - } + override suspend fun getUsername(): String { + val root = webClient.httpGet("https://grouple.co/").parseHtml().body() + val element = root.selectFirst("img.user-avatar") ?: throw AuthRequiredException(source) + val res = element.parent()?.text() + return if (res.isNullOrEmpty()) { + root.parseFailed("Cannot find username") + } else res + } - private fun getSortKey(sortOrder: SortOrder) = - when (sortOrder) { - SortOrder.ALPHABETICAL -> "name" - SortOrder.POPULARITY -> "rate" - SortOrder.UPDATED -> "updated" - SortOrder.NEWEST -> "created" - SortOrder.RATING -> "votes" - } + private fun getSortKey(sortOrder: SortOrder) = + when (sortOrder) { + SortOrder.ALPHABETICAL -> "name" + SortOrder.POPULARITY -> "rate" + SortOrder.UPDATED -> "updated" + SortOrder.NEWEST -> "created" + SortOrder.RATING -> "votes" + } - private suspend fun advancedSearch(domain: String, tags: Set): Response { - val url = "https://$domain/search/advanced" - // Step 1: map catalog genres names to advanced-search genres ids - val tagsIndex = context.httpGet(url, headers).parseHtml() - .body().selectFirst("form.search-form") - ?.select("div.form-group") - ?.get(1) ?: throw ParseException("Genres filter element not found", url) - val tagNames = tags.map { it.title.lowercase() } - val payload = HashMap() - var foundGenres = 0 - tagsIndex.select("li.property").forEach { li -> - val name = li.text().trim().lowercase() - val id = li.selectFirst("input")?.id() - ?: li.parseFailed("Id for tag $name not found") - payload[id] = if (name in tagNames) { - foundGenres++ - "in" - } else "" - } - if (foundGenres != tags.size) { - tagsIndex.parseFailed("Some genres are not found") - } - // Step 2: advanced search - payload["q"] = "" - payload["s_high_rate"] = "" - payload["s_single"] = "" - payload["s_mature"] = "" - payload["s_completed"] = "" - payload["s_translated"] = "" - payload["s_many_chapters"] = "" - payload["s_wait_upload"] = "" - payload["s_sale"] = "" - payload["years"] = "1900,2099" - payload["+"] = "Искать".urlEncoded() - return context.httpPost(url, payload, headers) - } + private suspend fun advancedSearch(domain: String, tags: Set): Response { + val url = "https://$domain/search/advanced" + // Step 1: map catalog genres names to advanced-search genres ids + val tagsIndex = webClient.httpGet(url).parseHtml() + .body().selectFirst("form.search-form") + ?.select("div.form-group") + ?.get(1) ?: throw ParseException("Genres filter element not found", url) + val tagNames = tags.map { it.title.lowercase() } + val payload = HashMap() + var foundGenres = 0 + tagsIndex.select("li.property").forEach { li -> + val name = li.text().trim().lowercase() + val id = li.selectFirst("input")?.id() + ?: li.parseFailed("Id for tag $name not found") + payload[id] = if (name in tagNames) { + foundGenres++ + "in" + } else "" + } + if (foundGenres != tags.size) { + tagsIndex.parseFailed("Some genres are not found") + } + // Step 2: advanced search + payload["q"] = "" + payload["s_high_rate"] = "" + payload["s_single"] = "" + payload["s_mature"] = "" + payload["s_completed"] = "" + payload["s_translated"] = "" + payload["s_many_chapters"] = "" + payload["s_wait_upload"] = "" + payload["s_sale"] = "" + payload["years"] = "1900,2099" + payload["+"] = "Искать".urlEncoded() + return webClient.httpPost(url, payload) + } - private suspend fun tryHead(url: String, headers: Headers): Boolean = runCatchingCancellable { - context.httpHead(url, headers).isSuccessful - }.getOrDefault(false) + private suspend fun tryHead(url: String): Boolean = runCatchingCancellable { + webClient.httpHead(url).isSuccessful + }.getOrDefault(false) - private fun Response.checkAuthRequired(): Response { - val lastPathSegment = request.url.pathSegments.lastOrNull() ?: return this - if (lastPathSegment == "login") { - throw AuthRequiredException(source) - } - return this - } -} \ No newline at end of file + private fun Response.checkAuthRequired(): Response { + val lastPathSegment = request.url.pathSegments.lastOrNull() ?: return this + if (lastPathSegment == "login") { + throw AuthRequiredException(source) + } + return this + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/MintMangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/MintMangaParser.kt index ecce1f71..ddbbdd3b 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/MintMangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/MintMangaParser.kt @@ -7,15 +7,15 @@ import org.koitharu.kotatsu.parsers.model.MangaSource @MangaSourceParser("MINTMANGA", "MintManga", "ru") internal class MintMangaParser( - override val context: MangaLoaderContext, -) : GroupleParser(MangaSource.MINTMANGA, "mintmangafun", 2) { + context: MangaLoaderContext, +) : GroupleParser(context, MangaSource.MINTMANGA, "mintmangafun", 2) { - override val configKeyDomain = ConfigKey.Domain( - "mintmanga.live", - arrayOf("mintmanga.live", "mintmanga.com"), - ) + override val configKeyDomain = ConfigKey.Domain( + "mintmanga.live", + arrayOf("mintmanga.live", "mintmanga.com"), + ) - override fun getFaviconUrl(): String { - return "https://resmm.rmr.rocks/static/apple-touch-icon-8fff291039c140493adb0c7ba81065ad.png" - } -} \ No newline at end of file + override fun getFaviconUrl(): String { + return "https://resmm.rmr.rocks/static/apple-touch-icon-8fff291039c140493adb0c7ba81065ad.png" + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/ReadmangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/ReadmangaParser.kt index 6ab26771..f4fe90d4 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/ReadmangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/ReadmangaParser.kt @@ -7,15 +7,15 @@ import org.koitharu.kotatsu.parsers.model.MangaSource @MangaSourceParser("READMANGA_RU", "ReadManga", "ru") internal class ReadmangaParser( - override val context: MangaLoaderContext, -) : GroupleParser(MangaSource.READMANGA_RU, "readmangafun", 1) { + context: MangaLoaderContext, +) : GroupleParser(context, MangaSource.READMANGA_RU, "readmangafun", 1) { - override val configKeyDomain = ConfigKey.Domain( - "readmanga.live", - arrayOf("readmanga.io", "readmanga.live", "readmanga.me"), - ) + override val configKeyDomain = ConfigKey.Domain( + "readmanga.live", + arrayOf("readmanga.io", "readmanga.live", "readmanga.me"), + ) - override fun getFaviconUrl(): String { - return "https://resrm.rmr.rocks/static/apple-touch-icon-3162037c9df9f28dca0f9a4092cb0f65.png" - } -} \ No newline at end of file + override fun getFaviconUrl(): String { + return "https://resrm.rmr.rocks/static/apple-touch-icon-3162037c9df9f28dca0f9a4092cb0f65.png" + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/SelfMangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/SelfMangaParser.kt index 3591eb71..91295808 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/SelfMangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/SelfMangaParser.kt @@ -7,12 +7,12 @@ import org.koitharu.kotatsu.parsers.model.MangaSource @MangaSourceParser("SELFMANGA", "SelfManga", "ru") internal class SelfMangaParser( - override val context: MangaLoaderContext, -) : GroupleParser(MangaSource.SELFMANGA, "selfmangafun", 3) { + context: MangaLoaderContext, +) : GroupleParser(context, MangaSource.SELFMANGA, "selfmangafun", 3) { - override val configKeyDomain = ConfigKey.Domain("selfmanga.live", null) + override val configKeyDomain = ConfigKey.Domain("selfmanga.live", null) - override fun getFaviconUrl(): String { - return "https://ressm.rmr.rocks/static/apple-touch-icon-a769ea533d811b73ac3eedde658bb1d3.png" - } -} \ No newline at end of file + override fun getFaviconUrl(): String { + return "https://ressm.rmr.rocks/static/apple-touch-icon-a769ea533d811b73ac3eedde658bb1d3.png" + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/IsekaiScanEu.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/IsekaiScanEu.kt index f2b2c6b9..526353e5 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/IsekaiScanEu.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/IsekaiScanEu.kt @@ -12,36 +12,36 @@ import java.util.* @MangaSourceParser("ISEKAISCAN_EU", "IsekaiScan (eu)", "en") internal class IsekaiScanEuParser(context: MangaLoaderContext) : - MadaraParser(context, MangaSource.ISEKAISCAN_EU, "isekaiscan.eu") { + MadaraParser(context, MangaSource.ISEKAISCAN_EU, "isekaiscan.eu") { - override val datePattern = "MM/dd/yyyy" + override val datePattern = "MM/dd/yyyy" - override suspend fun getChapters(manga: Manga, doc: Document): List { - val mangaId = doc.body().requireElementById("manga-chapters-holder").attr("data-id") - val ul = context.httpPost( - "https://${getDomain()}/wp-admin/admin-ajax.php", - mapOf( - "action" to "manga_get_chapters", - "manga" to mangaId, - ), - ).parseHtml().body().selectFirstOrThrow("ul") - val dateFormat = SimpleDateFormat(datePattern, Locale.US) - return ul.select("li").asReversed().mapChapters { i, li -> - val a = li.selectFirst("a") - val href = a?.attrAsRelativeUrlOrNull("href") ?: li.parseFailed("Link is missing") - MangaChapter( - id = generateUid(href), - name = a.ownText(), - number = i + 1, - url = href, - uploadDate = parseChapterDate( - dateFormat, - li.selectFirst("span.chapter-release-date i")?.text(), - ), - source = source, - scanlator = null, - branch = null, - ) - } - } -} \ No newline at end of file + override suspend fun getChapters(manga: Manga, doc: Document): List { + val mangaId = doc.body().requireElementById("manga-chapters-holder").attr("data-id") + val ul = webClient.httpPost( + "https://${domain}/wp-admin/admin-ajax.php", + mapOf( + "action" to "manga_get_chapters", + "manga" to mangaId, + ), + ).parseHtml().body().selectFirstOrThrow("ul") + val dateFormat = SimpleDateFormat(datePattern, Locale.US) + return ul.select("li").asReversed().mapChapters { i, li -> + val a = li.selectFirst("a") + val href = a?.attrAsRelativeUrlOrNull("href") ?: li.parseFailed("Link is missing") + MangaChapter( + id = generateUid(href), + name = a.ownText(), + number = i + 1, + url = href, + uploadDate = parseChapterDate( + dateFormat, + li.selectFirst("span.chapter-release-date i")?.text(), + ), + source = source, + scanlator = null, + branch = null, + ) + } + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Madara5Parser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Madara5Parser.kt index 26072e61..ae2e4110 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Madara5Parser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Madara5Parser.kt @@ -13,175 +13,175 @@ import java.text.SimpleDateFormat import java.util.* abstract class Madara5Parser @InternalParsersApi constructor( - override val context: MangaLoaderContext, - source: MangaSource, - domain: String, -) : PagedMangaParser(source, pageSize = 22) { - - protected open val datePattern = "MMMM dd, HH:mm" - protected open val tagPrefix = "/mangas/" - protected open val nsfwTags = arraySetOf("yaoi", "yuri", "mature") - - override val sortOrders: Set = EnumSet.of(SortOrder.UPDATED) - - override val configKeyDomain = ConfigKey.Domain(domain, null) - - override suspend fun getListPage( - page: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - val domain = getDomain() - val url = buildString { - append("https://") - append(domain) - append("/search?s=") - if (!query.isNullOrEmpty()) { - append(query.urlEncoded()) - } - append("&post_type=wp-manga") - if (!tags.isNullOrEmpty()) { - for (tag in tags) { - append("&genre%5B%5D=") - append(tag.key) - } - } - append("&op=1&author=&artist=&page=") - append(page) - } - val root = context.httpGet(url).parseHtml().body().selectFirstOrThrow(".search-wrap") - return root.select(".c-tabs-item__content").map { div -> - val a = div.selectFirstOrThrow("a") - val img = div.selectLastOrThrow("img") - val href = a.attrAsRelativeUrl("href") - val postContent = root.selectFirstOrThrow(".post-content") - val tagSet = postContent.getElementsContainingOwnText("Genre") - .firstOrNull()?.tableValue() - ?.getElementsByAttributeValueContaining("href", tagPrefix) - ?.mapToSet { it.asMangaTag() }.orEmpty() - Manga( - id = generateUid(href), - title = a.attr("title"), - altTitle = postContent.getElementsContainingOwnText("Alternative") - .firstOrNull()?.tableValue()?.text()?.trim(), - url = href, - publicUrl = a.attrAsAbsoluteUrl("href"), - coverUrl = img.src().orEmpty(), - author = postContent.getElementsContainingOwnText("Author") - .firstOrNull()?.tableValue()?.text()?.trim(), - state = postContent.getElementsContainingOwnText("Status") - .firstOrNull()?.tableValue()?.text()?.asMangaState(), - isNsfw = isNsfw(tagSet), - rating = div.selectFirstOrThrow(".score").text() - .toFloatOrNull()?.div(5f) ?: RATING_UNKNOWN, - tags = tagSet, - source = source, - ) - } - } - - override suspend fun getDetails(manga: Manga): Manga { - val root = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml().body() - .selectFirstOrThrow(".site-content") - val postContent = root.selectFirstOrThrow(".post-content") - val tags = postContent.getElementsContainingOwnText("Genre") - .firstOrNull()?.tableValue() - ?.getElementsByAttributeValueContaining("href", tagPrefix) - ?.mapToSet { a -> a.asMangaTag() } ?: manga.tags - val mangaId = root.getElementById("manga-chapters-holder")?.attr("data-id")?.toLongOrNull() - ?: root.parseFailed("Cannot find mangaId") - return manga.copy( - description = (root.selectFirst(".detail-content") - ?: root.selectFirstOrThrow(".description-summary")).html(), - author = postContent.getElementsContainingOwnText("Author") - .firstOrNull()?.tableValue()?.text()?.trim(), - state = postContent.getElementsContainingOwnText("Status") - .firstOrNull()?.tableValue()?.text()?.asMangaState(), - tags = tags, - isNsfw = isNsfw(tags), - chapters = loadChapters(mangaId), - ) - } - - override suspend fun getPages(chapter: MangaChapter): List { - val fullUrl = chapter.url.toAbsoluteUrl(getDomain()) - val doc = context.httpGet(fullUrl).parseHtml() - val arrayData = doc.getElementById("arraydata") ?: doc.parseFailed("#arraydata not found") - return arrayData.html().split(',').map { url -> - MangaPage( - id = generateUid(url), - url = url, - referer = fullUrl, - preview = null, - source = source, - ) - } - } - - override suspend fun getTags(): Set { - val doc = context.httpGet("http://${getDomain()}/").parseHtml().body() - return doc.getElementsByAttributeValueContaining("href", tagPrefix) - .mapToSet { it.asMangaTag() } - } - - private suspend fun loadChapters(mangaId: Long): List { - val dateFormat = SimpleDateFormat(datePattern, sourceLocale ?: Locale.US) - val doc = context.httpGet("https://${getDomain()}/ajax-list-chapter?mangaID=$mangaId").parseHtml() - return doc.select("li.wp-manga-chapter").asReversed().mapChapters { i, li -> - val a = li.selectFirstOrThrow("a") - val href = a.attrAsRelativeUrl("href") - MangaChapter( - id = generateUid(href), - url = href, - name = a.text(), - number = i + 1, - branch = null, - uploadDate = dateFormat.tryParse( - li.selectFirst(".chapter-release-date")?.text()?.trim(), - ), - scanlator = null, - source = source, - ) - } - } - - protected fun isNsfw(tags: Set): Boolean { - return tags.any { it.key in nsfwTags } - } - - private fun Element.src(): String? { - return absUrl("data-src").ifEmpty { - absUrl("src") - }.takeUnless { it.isEmpty() } - } - - private fun Element.tableValue(): Element { - for (p in parents()) { - val children = p.children() - if (children.size == 2) { - return children[1] - } - } - parseFailed("Cannot find tableValue for node ${text()}") - } - - private fun String.asMangaState() = when (trim().lowercase(sourceLocale ?: Locale.US)) { - "ongoing" -> MangaState.ONGOING - "completed" -> MangaState.FINISHED - else -> null - } - - private fun Element.asMangaTag() = MangaTag( - title = ownText(), - key = attr("href").removeSuffix('/').substringAfterLast('/') - .replace('-', '+'), - source = source, - ) - - @MangaSourceParser("MANGAOWLS", "BeautyManga", "en") - class BeautyManga(context: MangaLoaderContext) : Madara5Parser(context, MangaSource.MANGAOWLS, "beautymanga.com") { - - override fun getFaviconUrl() = "http://${getDomain()}/frontend/images/favico.png" - } + context: MangaLoaderContext, + source: MangaSource, + domain: String, +) : PagedMangaParser(context, source, pageSize = 22) { + + protected open val datePattern = "MMMM dd, HH:mm" + protected open val tagPrefix = "/mangas/" + protected open val nsfwTags = arraySetOf("yaoi", "yuri", "mature") + + override val sortOrders: Set = EnumSet.of(SortOrder.UPDATED) + + override val configKeyDomain = ConfigKey.Domain(domain, null) + + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + val domain = domain + val url = buildString { + append("https://") + append(domain) + append("/search?s=") + if (!query.isNullOrEmpty()) { + append(query.urlEncoded()) + } + append("&post_type=wp-manga") + if (!tags.isNullOrEmpty()) { + for (tag in tags) { + append("&genre%5B%5D=") + append(tag.key) + } + } + append("&op=1&author=&artist=&page=") + append(page) + } + val root = webClient.httpGet(url).parseHtml().body().selectFirstOrThrow(".search-wrap") + return root.select(".c-tabs-item__content").map { div -> + val a = div.selectFirstOrThrow("a") + val img = div.selectLastOrThrow("img") + val href = a.attrAsRelativeUrl("href") + val postContent = root.selectFirstOrThrow(".post-content") + val tagSet = postContent.getElementsContainingOwnText("Genre") + .firstOrNull()?.tableValue() + ?.getElementsByAttributeValueContaining("href", tagPrefix) + ?.mapToSet { it.asMangaTag() }.orEmpty() + Manga( + id = generateUid(href), + title = a.attr("title"), + altTitle = postContent.getElementsContainingOwnText("Alternative") + .firstOrNull()?.tableValue()?.text()?.trim(), + url = href, + publicUrl = a.attrAsAbsoluteUrl("href"), + coverUrl = img.src().orEmpty(), + author = postContent.getElementsContainingOwnText("Author") + .firstOrNull()?.tableValue()?.text()?.trim(), + state = postContent.getElementsContainingOwnText("Status") + .firstOrNull()?.tableValue()?.text()?.asMangaState(), + isNsfw = isNsfw(tagSet), + rating = div.selectFirstOrThrow(".score").text() + .toFloatOrNull()?.div(5f) ?: RATING_UNKNOWN, + tags = tagSet, + source = source, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val root = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml().body() + .selectFirstOrThrow(".site-content") + val postContent = root.selectFirstOrThrow(".post-content") + val tags = postContent.getElementsContainingOwnText("Genre") + .firstOrNull()?.tableValue() + ?.getElementsByAttributeValueContaining("href", tagPrefix) + ?.mapToSet { a -> a.asMangaTag() } ?: manga.tags + val mangaId = root.getElementById("manga-chapters-holder")?.attr("data-id")?.toLongOrNull() + ?: root.parseFailed("Cannot find mangaId") + return manga.copy( + description = (root.selectFirst(".detail-content") + ?: root.selectFirstOrThrow(".description-summary")).html(), + author = postContent.getElementsContainingOwnText("Author") + .firstOrNull()?.tableValue()?.text()?.trim(), + state = postContent.getElementsContainingOwnText("Status") + .firstOrNull()?.tableValue()?.text()?.asMangaState(), + tags = tags, + isNsfw = isNsfw(tags), + chapters = loadChapters(mangaId), + ) + } + + override suspend fun getPages(chapter: MangaChapter): List { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + val arrayData = doc.getElementById("arraydata") ?: doc.parseFailed("#arraydata not found") + return arrayData.html().split(',').map { url -> + MangaPage( + id = generateUid(url), + url = url, + referer = fullUrl, + preview = null, + source = source, + ) + } + } + + override suspend fun getTags(): Set { + val doc = webClient.httpGet("http://${domain}/").parseHtml().body() + return doc.getElementsByAttributeValueContaining("href", tagPrefix) + .mapToSet { it.asMangaTag() } + } + + private suspend fun loadChapters(mangaId: Long): List { + val dateFormat = SimpleDateFormat(datePattern, sourceLocale ?: Locale.US) + val doc = webClient.httpGet("https://${domain}/ajax-list-chapter?mangaID=$mangaId").parseHtml() + return doc.select("li.wp-manga-chapter").asReversed().mapChapters { i, li -> + val a = li.selectFirstOrThrow("a") + val href = a.attrAsRelativeUrl("href") + MangaChapter( + id = generateUid(href), + url = href, + name = a.text(), + number = i + 1, + branch = null, + uploadDate = dateFormat.tryParse( + li.selectFirst(".chapter-release-date")?.text()?.trim(), + ), + scanlator = null, + source = source, + ) + } + } + + protected fun isNsfw(tags: Set): Boolean { + return tags.any { it.key in nsfwTags } + } + + private fun Element.src(): String? { + return absUrl("data-src").ifEmpty { + absUrl("src") + }.takeUnless { it.isEmpty() } + } + + private fun Element.tableValue(): Element { + for (p in parents()) { + val children = p.children() + if (children.size == 2) { + return children[1] + } + } + parseFailed("Cannot find tableValue for node ${text()}") + } + + private fun String.asMangaState() = when (trim().lowercase(sourceLocale ?: Locale.US)) { + "ongoing" -> MangaState.ONGOING + "completed" -> MangaState.FINISHED + else -> null + } + + private fun Element.asMangaTag() = MangaTag( + title = ownText(), + key = attr("href").removeSuffix('/').substringAfterLast('/') + .replace('-', '+'), + source = source, + ) + + @MangaSourceParser("MANGAOWLS", "BeautyManga", "en") + class BeautyManga(context: MangaLoaderContext) : Madara5Parser(context, MangaSource.MANGAOWLS, "beautymanga.com") { + + override fun getFaviconUrl() = "http://${domain}/frontend/images/favico.png" + } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Madara6Parser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Madara6Parser.kt index 5fa04a6c..a38f4b97 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Madara6Parser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Madara6Parser.kt @@ -10,61 +10,61 @@ import java.text.SimpleDateFormat import java.util.* internal abstract class Madara6Parser( - context: MangaLoaderContext, - source: MangaSource, - domain: String, + context: MangaLoaderContext, + source: MangaSource, + domain: String, ) : MadaraParser(context, source, domain) { - override val datePattern: String = "dd MMMM yyyy" + override val datePattern: String = "dd MMMM yyyy" - override suspend fun getDetails(manga: Manga): Manga { - return coroutineScope { - val chapters = async { loadChapters(manga.url) } - val body = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml().body() - parseDetails(manga, body, chapters.await()) - } - } + override suspend fun getDetails(manga: Manga): Manga { + return coroutineScope { + val chapters = async { loadChapters(manga.url) } + val body = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml().body() + parseDetails(manga, body, chapters.await()) + } + } - protected fun Element.tableValue(): Element { - for (p in parents()) { - val children = p.children() - if (children.size == 2) { - return children[1] - } - } - parseFailed("Cannot find tableValue for node ${text()}") - } + protected fun Element.tableValue(): Element { + for (p in parents()) { + val children = p.children() + if (children.size == 2) { + return children[1] + } + } + parseFailed("Cannot find tableValue for node ${text()}") + } - protected abstract fun String.asMangaState(): MangaState? + protected abstract fun String.asMangaState(): MangaState? - protected fun Element.asMangaTag() = MangaTag( - title = ownText(), - key = attr("href").removeSuffix('/').substringAfterLast('/') - .replace('-', '+'), - source = source, - ) + protected fun Element.asMangaTag() = MangaTag( + title = ownText(), + key = attr("href").removeSuffix('/').substringAfterLast('/') + .replace('-', '+'), + source = source, + ) - protected open suspend fun loadChapters(mangaUrl: String): List { - val url = mangaUrl.toAbsoluteUrl(getDomain()).removeSuffix('/') + "/ajax/chapters/" - val dateFormat = SimpleDateFormat(datePattern, sourceLocale ?: Locale.ROOT) - val doc = context.httpPost(url, emptyMap()).parseHtml() - return doc.select("li.wp-manga-chapter").asReversed().mapChapters { i, li -> - val a = li.selectFirstOrThrow("a") - val href = a.attrAsRelativeUrl("href") - MangaChapter( - id = generateUid(href), - url = href, - name = a.text(), - number = i + 1, - branch = null, - uploadDate = dateFormat.tryParse( - li.selectFirst(".chapter-release-date")?.text()?.trim(), - ), - scanlator = null, - source = source, - ) - } - } + protected open suspend fun loadChapters(mangaUrl: String): List { + val url = mangaUrl.toAbsoluteUrl(domain).removeSuffix('/') + "/ajax/chapters/" + val dateFormat = SimpleDateFormat(datePattern, sourceLocale ?: Locale.ROOT) + val doc = webClient.httpPost(url, emptyMap()).parseHtml() + return doc.select("li.wp-manga-chapter").asReversed().mapChapters { i, li -> + val a = li.selectFirstOrThrow("a") + val href = a.attrAsRelativeUrl("href") + MangaChapter( + id = generateUid(href), + url = href, + name = a.text(), + number = i + 1, + branch = null, + uploadDate = dateFormat.tryParse( + li.selectFirst(".chapter-release-date")?.text()?.trim(), + ), + scanlator = null, + source = source, + ) + } + } - protected abstract fun parseDetails(manga: Manga, body: Element, chapters: List): Manga -} \ No newline at end of file + protected abstract fun parseDetails(manga: Manga, body: Element, chapters: List): Manga +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt index 3885663a..ad7a6b59 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt @@ -16,503 +16,503 @@ import java.text.SimpleDateFormat import java.util.* internal abstract class MadaraParser( - override val context: MangaLoaderContext, - source: MangaSource, - domain: String, -) : PagedMangaParser(source, pageSize = 12) { - - override val configKeyDomain = ConfigKey.Domain(domain, null) - - override val sortOrders: Set = EnumSet.of( - SortOrder.UPDATED, - SortOrder.POPULARITY, - ) - - protected open val tagPrefix = "manga-genre/" - protected open val isNsfwSource = false - protected open val datePattern = "MMMM dd, yyyy" - - init { - paginator.firstPage = 0 - searchPaginator.firstPage = 0 - } - - override suspend fun getListPage( - page: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - val tag = tags.oneOrThrowIfMany() - val payload = createRequestTemplate() - payload["page"] = page.toString() - payload["vars[meta_key]"] = when (sortOrder) { - SortOrder.POPULARITY -> "_wp_manga_views" - SortOrder.UPDATED -> "_latest_update" - else -> "_wp_manga_views" - } - payload["vars[wp-manga-genre]"] = tag?.key.orEmpty() - payload["vars[s]"] = query?.urlEncoded().orEmpty() - val doc = context.httpPost( - "https://${getDomain()}/wp-admin/admin-ajax.php", - payload, - ).parseHtml() - return doc.select("div.row.c-tabs-item__content").ifEmpty { - doc.select("div.page-item-detail.manga") - }.map { div -> - val href = div.selectFirst("a")?.attrAsRelativeUrlOrNull("href") - ?: div.parseFailed("Link not found") - val summary = div.selectFirst(".tab-summary") ?: div.selectFirst(".item-summary") - Manga( - id = generateUid(href), - url = href, - publicUrl = href.toAbsoluteUrl(div.host ?: getDomain()), - coverUrl = div.selectFirst("img")?.src().orEmpty(), - title = (summary?.selectFirst("h3") ?: summary?.selectFirst("h4"))?.text().orEmpty(), - altTitle = null, - rating = div.selectFirst("span.total_votes")?.ownText() - ?.toFloatOrNull()?.div(5f) ?: -1f, - tags = summary?.selectFirst(".mg_genres")?.select("a")?.mapToSet { a -> - MangaTag( - key = a.attr("href").removeSuffix("/").substringAfterLast('/'), - title = a.text().toTitleCase(), - source = source, - ) - }.orEmpty(), - author = summary?.selectFirst(".mg_author")?.selectFirst("a")?.ownText(), - state = when ( - summary?.selectFirst(".mg_status")?.selectFirst(".summary-content") - ?.ownText()?.trim()?.lowercase() - ) { - "ongoing" -> MangaState.ONGOING - "completed" -> MangaState.FINISHED - else -> null - }, - source = source, - isNsfw = isNsfwSource, - ) - } - } - - override suspend fun getTags(): Set { - val doc = context.httpGet("https://${getDomain()}/manga/").parseHtml() - val body = doc.body() - val root1 = body.selectFirst("header")?.selectFirst("ul.second-menu") - val root2 = body.selectFirst("div.genres_wrap")?.selectFirst("ul.list-unstyled") - if (root1 == null && root2 == null) { - doc.parseFailed("Root not found") - } - val list = root1?.select("li").orEmpty() + root2?.select("li").orEmpty() - val keySet = HashSet(list.size) - return list.mapNotNullToSet { li -> - val a = li.selectFirst("a") ?: return@mapNotNullToSet null - val href = a.attr("href").removeSuffix("/") - .substringAfterLast(tagPrefix, "") - if (href.isEmpty() || !keySet.add(href)) { - return@mapNotNullToSet null - } - MangaTag( - key = href, - title = a.ownText().trim().ifEmpty { - a.selectFirst(".menu-image-title")?.text()?.trim() ?: return@mapNotNullToSet null - }.toTitleCase(), - source = source, - ) - } - } - - override suspend fun getDetails(manga: Manga): Manga = coroutineScope { - val fullUrl = manga.url.toAbsoluteUrl(getDomain()) - val doc = context.httpGet(fullUrl).parseHtml() - val chaptersDeferred = async { getChapters(manga, doc) } - val root = doc.body().selectFirst("div.profile-manga") - ?.selectFirst("div.summary_content") - ?.selectFirst("div.post-content") - ?: throw ParseException("Root not found", fullUrl) - val root2 = doc.body().selectFirst("div.content-area") - ?.selectFirst("div.c-page") - ?: throw ParseException("Root2 not found", fullUrl) - manga.copy( - tags = root.selectFirst("div.genres-content")?.select("a") - ?.mapNotNullToSet { a -> - MangaTag( - key = a.attr("href").removeSuffix("/").substringAfterLast('/'), - title = a.text().toTitleCase(), - source = source, - ) - } ?: manga.tags, - description = root2.selectFirst("div.description-summary") - ?.selectFirst("div.summary__content") - ?.select("p") - ?.filterNot { it.ownText().startsWith("A brief description") } - ?.joinToString { it.html() }, - chapters = chaptersDeferred.await(), - ) - } - - protected open suspend fun getChapters(manga: Manga, doc: Document): List { - val root2 = doc.body().selectFirstOrThrow("div.content-area") - .selectFirstOrThrow("div.c-page") - val dateFormat = SimpleDateFormat(datePattern, sourceLocale ?: Locale.US) - return root2.select("li").asReversed().mapChapters { i, li -> - val a = li.selectFirst("a") - val href = a?.attrAsRelativeUrlOrNull("href") ?: li.parseFailed("Link is missing") - MangaChapter( - id = generateUid(href), - name = a.ownText(), - number = i + 1, - url = href, - uploadDate = parseChapterDate( - dateFormat, - li.selectFirst("span.chapter-release-date i")?.text(), - ), - source = source, - scanlator = null, - branch = null, - ) - } - } - - override suspend fun getPages(chapter: MangaChapter): List { - val fullUrl = chapter.url.toAbsoluteUrl(getDomain()) - val doc = context.httpGet(fullUrl).parseHtml() - val root = doc.body().selectFirst("div.main-col-inner") - ?.selectFirst("div.reading-content") - ?: throw ParseException("Root not found", fullUrl) - return root.select("div.page-break").map { div -> - val img = div.selectFirst("img") ?: div.parseFailed("Page image not found") - val url = img.src()?.toRelativeUrl(getDomain()) ?: div.parseFailed("Image src not found") - MangaPage( - id = generateUid(url), - url = url, - preview = null, - referer = fullUrl, - source = source, - ) - } - } - - protected fun parseChapterDate(dateFormat: DateFormat, date: String?): Long { - date ?: return 0 - return when { - date.endsWith(" ago", ignoreCase = true) -> { - parseRelativeDate(date) - } - // Handle translated 'ago' in Portuguese. - date.endsWith(" atrás", ignoreCase = true) -> { - parseRelativeDate(date) - } - // Handle translated 'ago' in Turkish. - date.endsWith(" önce", ignoreCase = true) -> { - parseRelativeDate(date) - } - // Handle 'yesterday' and 'today', using midnight - date.startsWith("year", ignoreCase = true) -> { - Calendar.getInstance().apply { - add(Calendar.DAY_OF_MONTH, -1) // yesterday - set(Calendar.HOUR_OF_DAY, 0) - set(Calendar.MINUTE, 0) - set(Calendar.SECOND, 0) - set(Calendar.MILLISECOND, 0) - }.timeInMillis - } - - date.startsWith("today", ignoreCase = true) -> { - Calendar.getInstance().apply { - set(Calendar.HOUR_OF_DAY, 0) - set(Calendar.MINUTE, 0) - set(Calendar.SECOND, 0) - set(Calendar.MILLISECOND, 0) - }.timeInMillis - } - - date.contains(Regex("""\d(st|nd|rd|th)""")) -> { - // Clean date (e.g. 5th December 2019 to 5 December 2019) before parsing it - date.split(" ").map { - if (it.contains(Regex("""\d\D\D"""))) { - it.replace(Regex("""\D"""), "") - } else { - it - } - } - .let { dateFormat.tryParse(it.joinToString(" ")) } - } - - else -> dateFormat.tryParse(date) - } - } - - // Parses dates in this form: - // 21 hours ago - private fun parseRelativeDate(date: String): Long { - val number = Regex("""(\d+)""").find(date)?.value?.toIntOrNull() ?: return 0 - val cal = Calendar.getInstance() - - return when { - WordSet( - "hari", - "gün", - "jour", - "día", - "dia", - "day", - ).anyWordIn(date) -> cal.apply { add(Calendar.DAY_OF_MONTH, -number) }.timeInMillis - - WordSet("jam", "saat", "heure", "hora", "hour").anyWordIn(date) -> cal.apply { - add( - Calendar.HOUR, - -number, - ) - }.timeInMillis - - WordSet("menit", "dakika", "min", "minute", "minuto").anyWordIn(date) -> cal.apply { - add( - Calendar.MINUTE, - -number, - ) - }.timeInMillis - - WordSet("detik", "segundo", "second").anyWordIn(date) -> cal.apply { - add( - Calendar.SECOND, - -number, - ) - }.timeInMillis - - WordSet("month").anyWordIn(date) -> cal.apply { add(Calendar.MONTH, -number) }.timeInMillis - WordSet("year").anyWordIn(date) -> cal.apply { add(Calendar.YEAR, -number) }.timeInMillis - else -> 0 - } - } - - private fun Element.src(): String? { - var result = absUrl("data-src") - if (result.isEmpty()) result = absUrl("data-cfsrc") - if (result.isEmpty()) result = absUrl("src") - return if (result.isEmpty()) null else result - } - - private fun createRequestTemplate() = - ( - "action=madara_load_more&page=1&template=madara-core%2Fcontent%2Fcontent-search&vars%5Bs%5D=&vars%5B" + - "orderby%5D=meta_value_num&vars%5Bpaged%5D=1&vars%5Btemplate%5D=search&vars%5Bmeta_query" + - "%5D%5B0%5D%5Brelation%5D=AND&vars%5Bmeta_query%5D%5Brelation%5D=OR&vars%5Bpost_type" + - "%5D=wp-manga&vars%5Bpost_status%5D=publish&vars%5Bmeta_key%5D=_latest_update&vars%5Border" + - "%5D=desc&vars%5Bmanga_archives_item_layout%5D=default" - ).split('&') - .map { - val pos = it.indexOf('=') - it.substring(0, pos) to it.substring(pos + 1) - }.toMutableMap() - - @MangaSourceParser("MANGAREAD", "MangaRead", "en") - class MangaRead(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGAREAD, "www.mangaread.org") { - override val tagPrefix = "genres/" - override val datePattern = "dd.MM.yyyy" - } - - @MangaSourceParser("MANGAWEEBS", "MangaWeebs", "en") - class MangaWeebs(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGAWEEBS, "mangaweebs.in") { - override val datePattern = "dd MMMM HH:mm" - } - - @MangaSourceParser("HachiManga", "HACHIMANGA", "ja") - class HachiManga(context: MangaLoaderContext) : - MadaraParser(context, MangaSource.HACHIMANGA, "hachimanga.com") { - - override val sourceLocale: Locale = Locale.ENGLISH - } - - @MangaSourceParser("PIANMANGA", "PianManga", "en") - class PianManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.PIANMANGA, "pianmanga.me") - - @MangaSourceParser("MANGAROSIE", "MangaRosie", "en") - class MangaRosie(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGAROSIE, "mangarosie.in") - - @MangaSourceParser("MANGATX", "MangaTx", "en") - class MangaTx(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGATX, "mangatx.com") - - @MangaSourceParser("MANGAEFFECT", "MangaEffect", "en") - class MangaEffect(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGAEFFECT, "mangaeffect.com") { - override val datePattern = "dd.MM.yyyy" - } - - @MangaSourceParser("AQUAMANGA", "AquaManga", "en") - class AquaManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.AQUAMANGA, "aquamanga.com") { - - override fun getFaviconUrl(): String { - return "https://${getDomain()}/wp-content/uploads/2021/03/cropped-cropped-favicon-1-32x32.png" - } - } - - @MangaSourceParser("MANGALEK", "MangaLek", "ar") - class MangaLek(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGALEK, "mangalek.com") { - - override fun getFaviconUrl(): String { - return "https://${getDomain()}/wp-content/logo192.png" - } - } - - @MangaSourceParser("HARIMANGA", "HariManga", "en") - class HariManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.HARIMANGA, "harimanga.com") { - override val datePattern = "MM/dd/yyyy" - } - - @MangaSourceParser("KISSMANGA", "KissManga", "en") - class KissManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.KISSMANGA, "kissmanga.in") - - @MangaSourceParser("MANGAROCK", "MangaRock", "en") - class MangaRock(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGAROCK, "mangarockteam.com") - - @MangaSourceParser("FREEMANGA", "FreeManga", "en") - class FreeManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.FREEMANGA, "freemanga.me") - - @MangaSourceParser("MANGA_KOMI", "MangaKomi", "en") - class MangaKomi(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGA_KOMI, "mangakomi.io") - - @MangaSourceParser("NEATMANGA", "NeatManga", "en") - class NeatManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.NEATMANGA, "neatmangas.com") - - @MangaSourceParser("MANHWACLAN", "ManhwaClan", "en") - class ManhwaClan(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANHWACLAN, "manhwaclan.com") - - @MangaSourceParser("MANGA_3S", "Manga3s", "en") - class Manga3s(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGA_3S, "manga3s.com") { - override val tagPrefix = "manhwa-genre/" - } - - @MangaSourceParser("MANHWAKOOL", "Manhwa Kool", "en") - class ManhwaKool(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANHWAKOOL, "manhwakool.com") { - override val datePattern: String = "MM/dd" - override fun getFaviconUrl(): String { - return "https://${getDomain()}/wp-content/uploads/2021/10/cropped-logo-kool-32x32.jpeg" - } - } - - @MangaSourceParser("TOPMANHUA", "Top Manhua", "en") - class TopManhua(context: MangaLoaderContext) : MadaraParser(context, MangaSource.TOPMANHUA, "www.topmanhua.com") { - override val tagPrefix = "manhua-genre/" - override val datePattern = "MM/dd/yyyy" - } - - @MangaSourceParser("S2MANGA", "S2Manga", "en") - class S2Manga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.S2MANGA, "s2manga.com") - - @MangaSourceParser("SKY_MANGA", "Sky Manga", "en") - class SkyManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.SKY_MANGA, "skymanga.xyz") { + context: MangaLoaderContext, + source: MangaSource, + domain: String, +) : PagedMangaParser(context, source, pageSize = 12) { + + override val configKeyDomain = ConfigKey.Domain(domain, null) + + override val sortOrders: Set = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + ) + + protected open val tagPrefix = "manga-genre/" + protected open val isNsfwSource = false + protected open val datePattern = "MMMM dd, yyyy" + + init { + paginator.firstPage = 0 + searchPaginator.firstPage = 0 + } + + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + val tag = tags.oneOrThrowIfMany() + val payload = createRequestTemplate() + payload["page"] = page.toString() + payload["vars[meta_key]"] = when (sortOrder) { + SortOrder.POPULARITY -> "_wp_manga_views" + SortOrder.UPDATED -> "_latest_update" + else -> "_wp_manga_views" + } + payload["vars[wp-manga-genre]"] = tag?.key.orEmpty() + payload["vars[s]"] = query?.urlEncoded().orEmpty() + val doc = webClient.httpPost( + "https://$domain/wp-admin/admin-ajax.php", + payload, + ).parseHtml() + return doc.select("div.row.c-tabs-item__content").ifEmpty { + doc.select("div.page-item-detail.manga") + }.map { div -> + val href = div.selectFirst("a")?.attrAsRelativeUrlOrNull("href") + ?: div.parseFailed("Link not found") + val summary = div.selectFirst(".tab-summary") ?: div.selectFirst(".item-summary") + Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(div.host ?: domain), + coverUrl = div.selectFirst("img")?.src().orEmpty(), + title = (summary?.selectFirst("h3") ?: summary?.selectFirst("h4"))?.text().orEmpty(), + altTitle = null, + rating = div.selectFirst("span.total_votes")?.ownText() + ?.toFloatOrNull()?.div(5f) ?: -1f, + tags = summary?.selectFirst(".mg_genres")?.select("a")?.mapToSet { a -> + MangaTag( + key = a.attr("href").removeSuffix("/").substringAfterLast('/'), + title = a.text().toTitleCase(), + source = source, + ) + }.orEmpty(), + author = summary?.selectFirst(".mg_author")?.selectFirst("a")?.ownText(), + state = when ( + summary?.selectFirst(".mg_status")?.selectFirst(".summary-content") + ?.ownText()?.trim()?.lowercase() + ) { + "ongoing" -> MangaState.ONGOING + "completed" -> MangaState.FINISHED + else -> null + }, + source = source, + isNsfw = isNsfwSource, + ) + } + } + + override suspend fun getTags(): Set { + val doc = webClient.httpGet("https://$domain/manga/").parseHtml() + val body = doc.body() + val root1 = body.selectFirst("header")?.selectFirst("ul.second-menu") + val root2 = body.selectFirst("div.genres_wrap")?.selectFirst("ul.list-unstyled") + if (root1 == null && root2 == null) { + doc.parseFailed("Root not found") + } + val list = root1?.select("li").orEmpty() + root2?.select("li").orEmpty() + val keySet = HashSet(list.size) + return list.mapNotNullToSet { li -> + val a = li.selectFirst("a") ?: return@mapNotNullToSet null + val href = a.attr("href").removeSuffix("/") + .substringAfterLast(tagPrefix, "") + if (href.isEmpty() || !keySet.add(href)) { + return@mapNotNullToSet null + } + MangaTag( + key = href, + title = a.ownText().trim().ifEmpty { + a.selectFirst(".menu-image-title")?.text()?.trim() ?: return@mapNotNullToSet null + }.toTitleCase(), + source = source, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { + val fullUrl = manga.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + val chaptersDeferred = async { getChapters(manga, doc) } + val root = doc.body().selectFirst("div.profile-manga") + ?.selectFirst("div.summary_content") + ?.selectFirst("div.post-content") + ?: throw ParseException("Root not found", fullUrl) + val root2 = doc.body().selectFirst("div.content-area") + ?.selectFirst("div.c-page") + ?: throw ParseException("Root2 not found", fullUrl) + manga.copy( + tags = root.selectFirst("div.genres-content")?.select("a") + ?.mapNotNullToSet { a -> + MangaTag( + key = a.attr("href").removeSuffix("/").substringAfterLast('/'), + title = a.text().toTitleCase(), + source = source, + ) + } ?: manga.tags, + description = root2.selectFirst("div.description-summary") + ?.selectFirst("div.summary__content") + ?.select("p") + ?.filterNot { it.ownText().startsWith("A brief description") } + ?.joinToString { it.html() }, + chapters = chaptersDeferred.await(), + ) + } + + protected open suspend fun getChapters(manga: Manga, doc: Document): List { + val root2 = doc.body().selectFirstOrThrow("div.content-area") + .selectFirstOrThrow("div.c-page") + val dateFormat = SimpleDateFormat(datePattern, sourceLocale ?: Locale.US) + return root2.select("li").asReversed().mapChapters { i, li -> + val a = li.selectFirst("a") + val href = a?.attrAsRelativeUrlOrNull("href") ?: li.parseFailed("Link is missing") + MangaChapter( + id = generateUid(href), + name = a.ownText(), + number = i + 1, + url = href, + uploadDate = parseChapterDate( + dateFormat, + li.selectFirst("span.chapter-release-date i")?.text(), + ), + source = source, + scanlator = null, + branch = null, + ) + } + } + + override suspend fun getPages(chapter: MangaChapter): List { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + val root = doc.body().selectFirst("div.main-col-inner") + ?.selectFirst("div.reading-content") + ?: throw ParseException("Root not found", fullUrl) + return root.select("div.page-break").map { div -> + val img = div.selectFirst("img") ?: div.parseFailed("Page image not found") + val url = img.src()?.toRelativeUrl(domain) ?: div.parseFailed("Image src not found") + MangaPage( + id = generateUid(url), + url = url, + preview = null, + referer = fullUrl, + source = source, + ) + } + } + + protected fun parseChapterDate(dateFormat: DateFormat, date: String?): Long { + date ?: return 0 + return when { + date.endsWith(" ago", ignoreCase = true) -> { + parseRelativeDate(date) + } + // Handle translated 'ago' in Portuguese. + date.endsWith(" atrás", ignoreCase = true) -> { + parseRelativeDate(date) + } + // Handle translated 'ago' in Turkish. + date.endsWith(" önce", ignoreCase = true) -> { + parseRelativeDate(date) + } + // Handle 'yesterday' and 'today', using midnight + date.startsWith("year", ignoreCase = true) -> { + Calendar.getInstance().apply { + add(Calendar.DAY_OF_MONTH, -1) // yesterday + set(Calendar.HOUR_OF_DAY, 0) + set(Calendar.MINUTE, 0) + set(Calendar.SECOND, 0) + set(Calendar.MILLISECOND, 0) + }.timeInMillis + } + + date.startsWith("today", ignoreCase = true) -> { + Calendar.getInstance().apply { + set(Calendar.HOUR_OF_DAY, 0) + set(Calendar.MINUTE, 0) + set(Calendar.SECOND, 0) + set(Calendar.MILLISECOND, 0) + }.timeInMillis + } + + date.contains(Regex("""\d(st|nd|rd|th)""")) -> { + // Clean date (e.g. 5th December 2019 to 5 December 2019) before parsing it + date.split(" ").map { + if (it.contains(Regex("""\d\D\D"""))) { + it.replace(Regex("""\D"""), "") + } else { + it + } + } + .let { dateFormat.tryParse(it.joinToString(" ")) } + } + + else -> dateFormat.tryParse(date) + } + } + + // Parses dates in this form: + // 21 hours ago + private fun parseRelativeDate(date: String): Long { + val number = Regex("""(\d+)""").find(date)?.value?.toIntOrNull() ?: return 0 + val cal = Calendar.getInstance() + + return when { + WordSet( + "hari", + "gün", + "jour", + "día", + "dia", + "day", + ).anyWordIn(date) -> cal.apply { add(Calendar.DAY_OF_MONTH, -number) }.timeInMillis + + WordSet("jam", "saat", "heure", "hora", "hour").anyWordIn(date) -> cal.apply { + add( + Calendar.HOUR, + -number, + ) + }.timeInMillis + + WordSet("menit", "dakika", "min", "minute", "minuto").anyWordIn(date) -> cal.apply { + add( + Calendar.MINUTE, + -number, + ) + }.timeInMillis + + WordSet("detik", "segundo", "second").anyWordIn(date) -> cal.apply { + add( + Calendar.SECOND, + -number, + ) + }.timeInMillis + + WordSet("month").anyWordIn(date) -> cal.apply { add(Calendar.MONTH, -number) }.timeInMillis + WordSet("year").anyWordIn(date) -> cal.apply { add(Calendar.YEAR, -number) }.timeInMillis + else -> 0 + } + } + + private fun Element.src(): String? { + var result = absUrl("data-src") + if (result.isEmpty()) result = absUrl("data-cfsrc") + if (result.isEmpty()) result = absUrl("src") + return if (result.isEmpty()) null else result + } + + private fun createRequestTemplate() = + ( + "action=madara_load_more&page=1&template=madara-core%2Fcontent%2Fcontent-search&vars%5Bs%5D=&vars%5B" + + "orderby%5D=meta_value_num&vars%5Bpaged%5D=1&vars%5Btemplate%5D=search&vars%5Bmeta_query" + + "%5D%5B0%5D%5Brelation%5D=AND&vars%5Bmeta_query%5D%5Brelation%5D=OR&vars%5Bpost_type" + + "%5D=wp-manga&vars%5Bpost_status%5D=publish&vars%5Bmeta_key%5D=_latest_update&vars%5Border" + + "%5D=desc&vars%5Bmanga_archives_item_layout%5D=default" + ).split('&') + .map { + val pos = it.indexOf('=') + it.substring(0, pos) to it.substring(pos + 1) + }.toMutableMap() + + @MangaSourceParser("MANGAREAD", "MangaRead", "en") + class MangaRead(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGAREAD, "www.mangaread.org") { + override val tagPrefix = "genres/" + override val datePattern = "dd.MM.yyyy" + } + + @MangaSourceParser("MANGAWEEBS", "MangaWeebs", "en") + class MangaWeebs(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGAWEEBS, "mangaweebs.in") { + override val datePattern = "dd MMMM HH:mm" + } + + @MangaSourceParser("HachiManga", "HACHIMANGA", "ja") + class HachiManga(context: MangaLoaderContext) : + MadaraParser(context, MangaSource.HACHIMANGA, "hachimanga.com") { + + override val sourceLocale: Locale = Locale.ENGLISH + } + + @MangaSourceParser("PIANMANGA", "PianManga", "en") + class PianManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.PIANMANGA, "pianmanga.me") + + @MangaSourceParser("MANGAROSIE", "MangaRosie", "en") + class MangaRosie(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGAROSIE, "mangarosie.in") + + @MangaSourceParser("MANGATX", "MangaTx", "en") + class MangaTx(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGATX, "mangatx.com") + + @MangaSourceParser("MANGAEFFECT", "MangaEffect", "en") + class MangaEffect(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGAEFFECT, "mangaeffect.com") { + override val datePattern = "dd.MM.yyyy" + } + + @MangaSourceParser("AQUAMANGA", "AquaManga", "en") + class AquaManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.AQUAMANGA, "aquamanga.com") { + + override fun getFaviconUrl(): String { + return "https://$domain/wp-content/uploads/2021/03/cropped-cropped-favicon-1-32x32.png" + } + } + + @MangaSourceParser("MANGALEK", "MangaLek", "ar") + class MangaLek(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGALEK, "mangalek.com") { + + override fun getFaviconUrl(): String { + return "https://$domain/wp-content/logo192.png" + } + } + + @MangaSourceParser("HARIMANGA", "HariManga", "en") + class HariManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.HARIMANGA, "harimanga.com") { + override val datePattern = "MM/dd/yyyy" + } + + @MangaSourceParser("KISSMANGA", "KissManga", "en") + class KissManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.KISSMANGA, "kissmanga.in") + + @MangaSourceParser("MANGAROCK", "MangaRock", "en") + class MangaRock(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGAROCK, "mangarockteam.com") + + @MangaSourceParser("FREEMANGA", "FreeManga", "en") + class FreeManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.FREEMANGA, "freemanga.me") + + @MangaSourceParser("MANGA_KOMI", "MangaKomi", "en") + class MangaKomi(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGA_KOMI, "mangakomi.io") + + @MangaSourceParser("NEATMANGA", "NeatManga", "en") + class NeatManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.NEATMANGA, "neatmangas.com") + + @MangaSourceParser("MANHWACLAN", "ManhwaClan", "en") + class ManhwaClan(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANHWACLAN, "manhwaclan.com") + + @MangaSourceParser("MANGA_3S", "Manga3s", "en") + class Manga3s(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGA_3S, "manga3s.com") { + override val tagPrefix = "manhwa-genre/" + } + + @MangaSourceParser("MANHWAKOOL", "Manhwa Kool", "en") + class ManhwaKool(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANHWAKOOL, "manhwakool.com") { + override val datePattern: String = "MM/dd" + override fun getFaviconUrl(): String { + return "https://$domain/wp-content/uploads/2021/10/cropped-logo-kool-32x32.jpeg" + } + } + + @MangaSourceParser("TOPMANHUA", "Top Manhua", "en") + class TopManhua(context: MangaLoaderContext) : MadaraParser(context, MangaSource.TOPMANHUA, "www.topmanhua.com") { + override val tagPrefix = "manhua-genre/" + override val datePattern = "MM/dd/yyyy" + } + + @MangaSourceParser("S2MANGA", "S2Manga", "en") + class S2Manga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.S2MANGA, "s2manga.com") + + @MangaSourceParser("SKY_MANGA", "Sky Manga", "en") + class SkyManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.SKY_MANGA, "skymanga.xyz") { - override val isNsfwSource = true + override val isNsfwSource = true - override fun getFaviconUrl(): String { - return "https://${getDomain()}/wp-content/uploads/cropped-sky-tv-1-32x32.png" - } - } + override fun getFaviconUrl(): String { + return "https://$domain/wp-content/uploads/cropped-sky-tv-1-32x32.png" + } + } - @MangaSourceParser("BAKAMAN", "BakaMan", "th") - class BakaMan(context: MangaLoaderContext) : MadaraParser(context, MangaSource.BAKAMAN, "bakaman.net") { + @MangaSourceParser("BAKAMAN", "BakaMan", "th") + class BakaMan(context: MangaLoaderContext) : MadaraParser(context, MangaSource.BAKAMAN, "bakaman.net") { - override val isNsfwSource = true - } + override val isNsfwSource = true + } - @MangaSourceParser("MANGA_DISTRICT", "Manga District", "en") - class MangaDistrict(context: MangaLoaderContext) : - MadaraParser(context, MangaSource.MANGA_DISTRICT, "mangadistrict.com") { + @MangaSourceParser("MANGA_DISTRICT", "Manga District", "en") + class MangaDistrict(context: MangaLoaderContext) : + MadaraParser(context, MangaSource.MANGA_DISTRICT, "mangadistrict.com") { - override val tagPrefix = "publication-genre/" + override val tagPrefix = "publication-genre/" - override val isNsfwSource = true - } + override val isNsfwSource = true + } - @MangaSourceParser("HENTAI_4FREE", "Hentai4Free", "en") - class Hentai4Free(context: MangaLoaderContext) : - MadaraParser(context, MangaSource.HENTAI_4FREE, "hentai4free.net") { + @MangaSourceParser("HENTAI_4FREE", "Hentai4Free", "en") + class Hentai4Free(context: MangaLoaderContext) : + MadaraParser(context, MangaSource.HENTAI_4FREE, "hentai4free.net") { - override val tagPrefix = "hentai-tag/" + override val tagPrefix = "hentai-tag/" - override val isNsfwSource = true + override val isNsfwSource = true - override suspend fun getTags(): Set { - val doc = context.httpGet("https://${getDomain()}/").parseHtml() - val body = doc.body() - val root1 = body.selectFirst("header")?.selectFirst("ul.second-menu") - val list = root1?.select("li").orEmpty() - val keySet = HashSet(list.size) - return list.mapNotNullToSet { li -> - val a = li.selectFirst("a") ?: return@mapNotNullToSet null - val href = a.attr("href").removeSuffix("/") - .substringAfterLast(tagPrefix, "") - if (href.isEmpty() || !keySet.add(href)) { - return@mapNotNullToSet null - } - MangaTag( - key = href, - title = a.ownText().trim().toTitleCase(), - source = source, - ) - } - } - } + override suspend fun getTags(): Set { + val doc = webClient.httpGet("https://$domain/").parseHtml() + val body = doc.body() + val root1 = body.selectFirst("header")?.selectFirst("ul.second-menu") + val list = root1?.select("li").orEmpty() + val keySet = HashSet(list.size) + return list.mapNotNullToSet { li -> + val a = li.selectFirst("a") ?: return@mapNotNullToSet null + val href = a.attr("href").removeSuffix("/") + .substringAfterLast(tagPrefix, "") + if (href.isEmpty() || !keySet.add(href)) { + return@mapNotNullToSet null + } + MangaTag( + key = href, + title = a.ownText().trim().toTitleCase(), + source = source, + ) + } + } + } - @MangaSourceParser("ALLPORN_COMIC", "All Porn Comic", "en") - class AllPornComic(context: MangaLoaderContext) : - MadaraParser(context, MangaSource.ALLPORN_COMIC, "allporncomic.com") { + @MangaSourceParser("ALLPORN_COMIC", "All Porn Comic", "en") + class AllPornComic(context: MangaLoaderContext) : + MadaraParser(context, MangaSource.ALLPORN_COMIC, "allporncomic.com") { - override val tagPrefix = "porncomic-genre/" + override val tagPrefix = "porncomic-genre/" - override val isNsfwSource = true + override val isNsfwSource = true - override fun getFaviconUrl(): String { - return "https://cdn.${getDomain()}/wp-content/uploads/2019/01/cropped-cropped-pcround-32x32.png" - } - } + override fun getFaviconUrl(): String { + return "https://cdn.$domain/wp-content/uploads/2019/01/cropped-cropped-pcround-32x32.png" + } + } - @MangaSourceParser("CAT_300", "Cat300", "th") - class Cat300(context: MangaLoaderContext) : MadaraParser(context, MangaSource.CAT_300, "cat300.com") { + @MangaSourceParser("CAT_300", "Cat300", "th") + class Cat300(context: MangaLoaderContext) : MadaraParser(context, MangaSource.CAT_300, "cat300.com") { - override val isNsfwSource = true - } - - @MangaSourceParser("BIBIMANGA", "BibiManga", "en") - class BibiManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.BIBIMANGA, "bibimanga.com") { - - override val isNsfwSource = true - } - - @MangaSourceParser("TREE_MANGA", "Tree Manga", "en") - class TreeManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.TREE_MANGA, "treemanga.com") { + override val isNsfwSource = true + } + + @MangaSourceParser("BIBIMANGA", "BibiManga", "en") + class BibiManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.BIBIMANGA, "bibimanga.com") { + + override val isNsfwSource = true + } + + @MangaSourceParser("TREE_MANGA", "Tree Manga", "en") + class TreeManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.TREE_MANGA, "treemanga.com") { - override val datePattern = "MM/dd/yyyy" - - override fun getFaviconUrl(): String { - return "https://${getDomain()}/wp-content/uploads/2017/10/lgoo-treemanga-2-1.jpg" - } - } + override val datePattern = "MM/dd/yyyy" + + override fun getFaviconUrl(): String { + return "https://$domain/wp-content/uploads/2017/10/lgoo-treemanga-2-1.jpg" + } + } - @MangaSourceParser("MANGACV", "Manga Cv", "en") - class MangaCv(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGACV, "mangacv.com") { + @MangaSourceParser("MANGACV", "Manga Cv", "en") + class MangaCv(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGACV, "mangacv.com") { - override fun getFaviconUrl(): String { - return "https://${getDomain()}/wp-content/uploads/2020/10/cropped-mangaCV-32x32.png" - } - } + override fun getFaviconUrl(): String { + return "https://$domain/wp-content/uploads/2020/10/cropped-mangaCV-32x32.png" + } + } - @MangaSourceParser("MANGA_MANHUA", "Manga Manhua", "en") - class MangaManhua(context: MangaLoaderContext) : - MadaraParser(context, MangaSource.MANGA_MANHUA, "mangamanhua.online") + @MangaSourceParser("MANGA_MANHUA", "Manga Manhua", "en") + class MangaManhua(context: MangaLoaderContext) : + MadaraParser(context, MangaSource.MANGA_MANHUA, "mangamanhua.online") - @MangaSourceParser("MANGA_247", "247MANGA", "en") - class Manga247(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGA_247, "247manga.com") { - override val tagPrefix = "manhwa-genre/" - } + @MangaSourceParser("MANGA_247", "247MANGA", "en") + class Manga247(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGA_247, "247manga.com") { + override val tagPrefix = "manhwa-genre/" + } - @MangaSourceParser("MANGA_365", "365Manga", "en") - class Manga365(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGA_365, "365manga.com") + @MangaSourceParser("MANGA_365", "365Manga", "en") + class Manga365(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGA_365, "365manga.com") - @MangaSourceParser("MANGACLASH", "Mangaclash", "en") - class Mangaclash(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGACLASH, "mangaclash.com") { - override val datePattern = "MM/dd/yyyy" - } + @MangaSourceParser("MANGACLASH", "Mangaclash", "en") + class Mangaclash(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGACLASH, "mangaclash.com") { + override val datePattern = "MM/dd/yyyy" + } - @MangaSourceParser("ZINMANGA", "ZINMANGA", "en") - class Zinmanga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.ZINMANGA, "zinmanga.com") + @MangaSourceParser("ZINMANGA", "ZINMANGA", "en") + class Zinmanga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.ZINMANGA, "zinmanga.com") } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangalinkParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangalinkParser.kt index 6303c813..ff78aab4 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangalinkParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangalinkParser.kt @@ -8,44 +8,41 @@ import org.koitharu.kotatsu.parsers.exception.ParseException import org.koitharu.kotatsu.parsers.model.Manga import org.koitharu.kotatsu.parsers.model.MangaSource import org.koitharu.kotatsu.parsers.model.MangaTag -import org.koitharu.kotatsu.parsers.util.mapNotNullToSet -import org.koitharu.kotatsu.parsers.util.parseHtml -import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl -import org.koitharu.kotatsu.parsers.util.toTitleCase +import org.koitharu.kotatsu.parsers.util.* @MangaSourceParser("MANGALINK_AR", "Mangalink", "ar") internal class MangalinkParser(context: MangaLoaderContext) : - MadaraParser(context, MangaSource.MANGALINK_AR, "mangalink.online") { + MadaraParser(context, MangaSource.MANGALINK_AR, "mangalink.online") { - override suspend fun getDetails(manga: Manga): Manga = coroutineScope { - val fullUrl = manga.url.toAbsoluteUrl(getDomain()) - val doc = context.httpGet(fullUrl).parseHtml() - val chaptersDeferred = async { getChapters(manga, doc) } - val root = doc.body().selectFirst("div.profile-manga") - ?.selectFirst("div.summary_content") - ?.selectFirst("div.post-content") - ?: throw ParseException("Root not found", fullUrl) - val root2 = doc.body().selectFirst("div.content-area") - ?.selectFirst("div.c-page") - ?: throw ParseException("Root2 not found", fullUrl) - manga.copy( - tags = root.selectFirst("div.genres-content")?.select("a") - ?.mapNotNullToSet { a -> - MangaTag( - key = a.attr("href").removeSuffix("/").substringAfterLast('/'), - title = a.text().toTitleCase(), - source = source, - ) - } ?: manga.tags, - description = root2.selectFirst("div.description-summary") - ?.selectFirst("div.summary__content") - ?.select("p") - ?.filterNot { it.ownText().startsWith("A brief description") } - ?.joinToString { it.html() }, - chapters = chaptersDeferred.await(), - ) - } + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { + val fullUrl = manga.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + val chaptersDeferred = async { getChapters(manga, doc) } + val root = doc.body().selectFirst("div.profile-manga") + ?.selectFirst("div.summary_content") + ?.selectFirst("div.post-content") + ?: throw ParseException("Root not found", fullUrl) + val root2 = doc.body().selectFirst("div.content-area") + ?.selectFirst("div.c-page") + ?: throw ParseException("Root2 not found", fullUrl) + manga.copy( + tags = root.selectFirst("div.genres-content")?.select("a") + ?.mapNotNullToSet { a -> + MangaTag( + key = a.attr("href").removeSuffix("/").substringAfterLast('/'), + title = a.text().toTitleCase(), + source = source, + ) + } ?: manga.tags, + description = root2.selectFirst("div.description-summary") + ?.selectFirst("div.summary__content") + ?.select("p") + ?.filterNot { it.ownText().startsWith("A brief description") } + ?.joinToString { it.html() }, + chapters = chaptersDeferred.await(), + ) + } - override fun getFaviconUrl(): String = - "https://cdn.${getDomain()}/wp-content/uploads/2020/05/cropped-mangalink-180x180.jpg" -} \ No newline at end of file + override fun getFaviconUrl(): String = + "https://cdn.${domain}/wp-content/uploads/2020/05/cropped-mangalink-180x180.jpg" +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangasOriginesParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangasOriginesParser.kt index 8b9b567d..af1c3801 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangasOriginesParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangasOriginesParser.kt @@ -8,49 +8,50 @@ import org.koitharu.kotatsu.parsers.model.MangaChapter import org.koitharu.kotatsu.parsers.model.MangaSource import org.koitharu.kotatsu.parsers.model.MangaState import org.koitharu.kotatsu.parsers.util.attrAsAbsoluteUrlOrNull +import org.koitharu.kotatsu.parsers.util.domain import org.koitharu.kotatsu.parsers.util.mapToSet import org.koitharu.kotatsu.parsers.util.selectFirstOrThrow import java.util.* @MangaSourceParser("MANGAS_ORIGINES", "Mangas Origines", "fr") internal class MangasOriginesParser(context: MangaLoaderContext) : - Madara6Parser(context, MangaSource.MANGAS_ORIGINES, "mangas-origines.fr") { + Madara6Parser(context, MangaSource.MANGAS_ORIGINES, "mangas-origines.fr") { - override val tagPrefix = "catalogues-genre/" + override val tagPrefix = "catalogues-genre/" - override fun getFaviconUrl(): String { - return "https://${getDomain()}/wp-content/uploads/2020/11/Mangas-150x150.png" - } + override fun getFaviconUrl(): String { + return "https://${domain}/wp-content/uploads/2020/11/Mangas-150x150.png" + } - override fun parseDetails(manga: Manga, body: Element, chapters: List): Manga { - val root = body.selectFirstOrThrow(".site-content") - val postContent = root.selectFirstOrThrow(".post-content") - val tags = postContent.getElementsContainingOwnText("Genre") - .firstOrNull()?.tableValue() - ?.getElementsByAttributeValueContaining("href", tagPrefix) - ?.mapToSet { a -> a.asMangaTag() } ?: manga.tags - return manga.copy( - largeCoverUrl = root.selectFirst("picture") - ?.selectFirst("img[data-src]") - ?.attrAsAbsoluteUrlOrNull("data-src"), - description = (root.selectFirst(".detail-content") - ?: root.selectFirstOrThrow(".manga-excerpt")).html(), - author = postContent.getElementsContainingOwnText("Auteur") - .firstOrNull()?.tableValue()?.text()?.trim(), - state = postContent.getElementsContainingOwnText("STATUS") - .firstOrNull()?.tableValue()?.text()?.asMangaState(), - tags = tags, - isNsfw = body.hasClass("adult-content"), - chapters = chapters, - ) - } + override fun parseDetails(manga: Manga, body: Element, chapters: List): Manga { + val root = body.selectFirstOrThrow(".site-content") + val postContent = root.selectFirstOrThrow(".post-content") + val tags = postContent.getElementsContainingOwnText("Genre") + .firstOrNull()?.tableValue() + ?.getElementsByAttributeValueContaining("href", tagPrefix) + ?.mapToSet { a -> a.asMangaTag() } ?: manga.tags + return manga.copy( + largeCoverUrl = root.selectFirst("picture") + ?.selectFirst("img[data-src]") + ?.attrAsAbsoluteUrlOrNull("data-src"), + description = (root.selectFirst(".detail-content") + ?: root.selectFirstOrThrow(".manga-excerpt")).html(), + author = postContent.getElementsContainingOwnText("Auteur") + .firstOrNull()?.tableValue()?.text()?.trim(), + state = postContent.getElementsContainingOwnText("STATUS") + .firstOrNull()?.tableValue()?.text()?.asMangaState(), + tags = tags, + isNsfw = body.hasClass("adult-content"), + chapters = chapters, + ) + } - override fun String.asMangaState() = when (trim().lowercase(Locale.FRANCE)) { - "en cours" -> MangaState.ONGOING - "abandonné", - "terminé", - -> MangaState.FINISHED + override fun String.asMangaState() = when (trim().lowercase(Locale.FRANCE)) { + "en cours" -> MangaState.ONGOING + "abandonné", + "terminé", + -> MangaState.FINISHED - else -> null - } -} \ No newline at end of file + else -> null + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/PrismaScansParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/PrismaScansParser.kt index 6a95d095..b48770b0 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/PrismaScansParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/PrismaScansParser.kt @@ -8,51 +8,52 @@ import org.koitharu.kotatsu.parsers.model.MangaChapter import org.koitharu.kotatsu.parsers.model.MangaSource import org.koitharu.kotatsu.parsers.model.MangaState import org.koitharu.kotatsu.parsers.util.attrAsAbsoluteUrlOrNull +import org.koitharu.kotatsu.parsers.util.domain import org.koitharu.kotatsu.parsers.util.mapToSet import org.koitharu.kotatsu.parsers.util.selectFirstOrThrow import java.util.* @MangaSourceParser("PRISMA_SCANS", "Prisma Scans", "pt") internal class PrismaScansParser(context: MangaLoaderContext) : - Madara6Parser(context, MangaSource.PRISMA_SCANS, "prismascans.net") { + Madara6Parser(context, MangaSource.PRISMA_SCANS, "prismascans.net") { - override val tagPrefix = "manga-genre/" - override val datePattern = "MMM dd, yyyy" + override val tagPrefix = "manga-genre/" + override val datePattern = "MMM dd, yyyy" - override fun getFaviconUrl(): String { - return "https://${getDomain()}/wp-content/uploads/2022/07/cropped-branca-1-192x192.png" - } + override fun getFaviconUrl(): String { + return "https://${domain}/wp-content/uploads/2022/07/cropped-branca-1-192x192.png" + } - override fun parseDetails(manga: Manga, body: Element, chapters: List): Manga { - val root = body.selectFirstOrThrow(".site-content") - val postContent = root.selectFirstOrThrow(".post-content") - val tags = postContent.getElementsContainingOwnText("Gênero") - .firstOrNull()?.tableValue() - ?.getElementsByAttributeValueContaining("href", tagPrefix) - ?.mapToSet { a -> a.asMangaTag() } ?: manga.tags - return manga.copy( - largeCoverUrl = root.selectFirst("picture") - ?.selectFirst("img[data-src]") - ?.attrAsAbsoluteUrlOrNull("data-src"), - description = root.selectFirstOrThrow(".manga-excerpt").firstElementChild()?.html(), - author = postContent.getElementsContainingOwnText("Artista") - .firstOrNull()?.tableValue()?.text()?.trim(), - altTitle = postContent.getElementsContainingOwnText("Título Alternativo") - .firstOrNull()?.tableValue()?.text()?.trim(), - state = postContent.getElementsContainingOwnText("Status") - .firstOrNull()?.tableValue()?.text()?.asMangaState(), - tags = tags, - isNsfw = body.hasClass("adult-content"), - chapters = chapters, - ) - } + override fun parseDetails(manga: Manga, body: Element, chapters: List): Manga { + val root = body.selectFirstOrThrow(".site-content") + val postContent = root.selectFirstOrThrow(".post-content") + val tags = postContent.getElementsContainingOwnText("Gênero") + .firstOrNull()?.tableValue() + ?.getElementsByAttributeValueContaining("href", tagPrefix) + ?.mapToSet { a -> a.asMangaTag() } ?: manga.tags + return manga.copy( + largeCoverUrl = root.selectFirst("picture") + ?.selectFirst("img[data-src]") + ?.attrAsAbsoluteUrlOrNull("data-src"), + description = root.selectFirstOrThrow(".manga-excerpt").firstElementChild()?.html(), + author = postContent.getElementsContainingOwnText("Artista") + .firstOrNull()?.tableValue()?.text()?.trim(), + altTitle = postContent.getElementsContainingOwnText("Título Alternativo") + .firstOrNull()?.tableValue()?.text()?.trim(), + state = postContent.getElementsContainingOwnText("Status") + .firstOrNull()?.tableValue()?.text()?.asMangaState(), + tags = tags, + isNsfw = body.hasClass("adult-content"), + chapters = chapters, + ) + } - override fun String.asMangaState() = when (trim().lowercase(sourceLocale ?: Locale.ROOT)) { - "em lançamento" -> MangaState.ONGOING - "completo", - "cancelado", - -> MangaState.FINISHED + override fun String.asMangaState() = when (trim().lowercase(sourceLocale ?: Locale.ROOT)) { + "em lançamento" -> MangaState.ONGOING + "completo", + "cancelado", + -> MangaState.FINISHED - else -> null - } -} \ No newline at end of file + else -> null + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/ReaperScansParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/ReaperScansParser.kt index 7e3a2002..b161995f 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/ReaperScansParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/ReaperScansParser.kt @@ -9,80 +9,80 @@ import java.util.* @MangaSourceParser("REAPER_SCANS_ID", "ReaperScansID", "in") internal class ReaperScansParser(context: MangaLoaderContext) : - Madara6Parser(context, MangaSource.REAPER_SCANS_ID, "reaperscans.id") { + Madara6Parser(context, MangaSource.REAPER_SCANS_ID, "reaperscans.id") { - override val datePattern = "MMMM dd, yyyy" - override val tagPrefix = "genre/" - override val sourceLocale: Locale = Locale.ENGLISH + override val datePattern = "MMMM dd, yyyy" + override val tagPrefix = "genre/" + override val sourceLocale: Locale = Locale.ENGLISH - override fun String.asMangaState(): MangaState? = when (this) { - "OnGoing", - "Upcoming", - -> MangaState.ONGOING + override fun String.asMangaState(): MangaState? = when (this) { + "OnGoing", + "Upcoming", + -> MangaState.ONGOING - "Completed", - "Dropped", - -> MangaState.FINISHED + "Completed", + "Dropped", + -> MangaState.FINISHED - else -> null - } + else -> null + } - override fun parseDetails(manga: Manga, body: Element, chapters: List): Manga { - val root = body.selectFirstOrThrow(".site-content") - val postContent = root.requireElementById("nav-info") - val tags = postContent.getElementsContainingOwnText("Gênero") - .firstOrNull()?.tableValue() - ?.getElementsByAttributeValueContaining("href", tagPrefix) - ?.mapToSet { a -> a.asMangaTag() } ?: manga.tags - return manga.copy( - rating = postContent.selectFirstOrThrow(".post-rating") - .selectFirstOrThrow(".total_votes").text().toFloat() / 5f, - largeCoverUrl = root.selectFirst(".summary_image") - ?.selectFirst("img[data-src]") - ?.attrAsAbsoluteUrlOrNull("data-src") - .assertNotNull("largeCoverUrl"), - description = root.requireElementById("nav-profile") - .selectFirstOrThrow(".description-summary") - .firstElementChild()?.html(), - author = postContent.getElementsContainingOwnText("Author(s)") - .firstOrNull()?.tableValue()?.text()?.trim(), - altTitle = postContent.getElementsContainingOwnText("Alternative") - .firstOrNull()?.tableValue()?.text()?.trim(), - state = postContent.getElementsContainingOwnText("Status") - .firstOrNull()?.tableValue()?.text()?.asMangaState(), - tags = tags, - isNsfw = body.hasClass("adult-content"), - chapters = chapters, - ) - } + override fun parseDetails(manga: Manga, body: Element, chapters: List): Manga { + val root = body.selectFirstOrThrow(".site-content") + val postContent = root.requireElementById("nav-info") + val tags = postContent.getElementsContainingOwnText("Gênero") + .firstOrNull()?.tableValue() + ?.getElementsByAttributeValueContaining("href", tagPrefix) + ?.mapToSet { a -> a.asMangaTag() } ?: manga.tags + return manga.copy( + rating = postContent.selectFirstOrThrow(".post-rating") + .selectFirstOrThrow(".total_votes").text().toFloat() / 5f, + largeCoverUrl = root.selectFirst(".summary_image") + ?.selectFirst("img[data-src]") + ?.attrAsAbsoluteUrlOrNull("data-src") + .assertNotNull("largeCoverUrl"), + description = root.requireElementById("nav-profile") + .selectFirstOrThrow(".description-summary") + .firstElementChild()?.html(), + author = postContent.getElementsContainingOwnText("Author(s)") + .firstOrNull()?.tableValue()?.text()?.trim(), + altTitle = postContent.getElementsContainingOwnText("Alternative") + .firstOrNull()?.tableValue()?.text()?.trim(), + state = postContent.getElementsContainingOwnText("Status") + .firstOrNull()?.tableValue()?.text()?.asMangaState(), + tags = tags, + isNsfw = body.hasClass("adult-content"), + chapters = chapters, + ) + } - override suspend fun getTags(): Set { - val doc = context.httpGet("https://${getDomain()}/semua-komik/").parseHtml() - val body = doc.body() - val root1 = body.selectFirst("header")?.selectFirst("ul.second-menu") - val root2 = body.selectFirst("div.genres_wrap")?.selectFirst("ul.list-unstyled") - if (root1 == null && root2 == null) { - doc.parseFailed("Root not found") - } - val list = root1?.select("li").orEmpty() + root2?.select("li").orEmpty() - val keySet = HashSet(list.size) - return list.mapNotNullToSet { li -> - val a = li.selectFirst("a") ?: return@mapNotNullToSet null - val href = a.attr("href").removeSuffix("/") - .substringAfterLast(tagPrefix, "") - if (href.isEmpty() || !keySet.add(href)) { - return@mapNotNullToSet null - } - MangaTag( - key = href, - title = a.ownText().trim().ifEmpty { - a.selectFirst(".menu-image-title")?.text()?.trim() ?: return@mapNotNullToSet null - }.toTitleCase(), - source = source, - ) - } - } + override suspend fun getTags(): Set { + val doc = webClient.httpGet("https://${domain}/semua-komik/").parseHtml() + val body = doc.body() + val root1 = body.selectFirst("header")?.selectFirst("ul.second-menu") + val root2 = body.selectFirst("div.genres_wrap")?.selectFirst("ul.list-unstyled") + if (root1 == null && root2 == null) { + doc.parseFailed("Root not found") + } + val list = root1?.select("li").orEmpty() + root2?.select("li").orEmpty() + val keySet = HashSet(list.size) + return list.mapNotNullToSet { li -> + val a = li.selectFirst("a") ?: return@mapNotNullToSet null + val href = a.attr("href").removeSuffix("/") + .substringAfterLast(tagPrefix, "") + if (href.isEmpty() || !keySet.add(href)) { + return@mapNotNullToSet null + } + MangaTag( + key = href, + title = a.ownText().trim().ifEmpty { + a.selectFirst(".menu-image-title")?.text()?.trim() ?: return@mapNotNullToSet null + }.toTitleCase(), + source = source, + ) + } + } - override fun getFaviconUrl(): String = - "https://wuz.${getDomain()}/wp-content/uploads/2022/03/20063242/cropped-ms-icon-310x310-2-192x192.png" -} \ No newline at end of file + override fun getFaviconUrl(): String = + "https://wuz.${domain}/wp-content/uploads/2022/03/20063242/cropped-ms-icon-310x310-2-192x192.png" +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/TatakaeScansParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/TatakaeScansParser.kt index d9366459..4a9a65cb 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/TatakaeScansParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/TatakaeScansParser.kt @@ -3,49 +3,53 @@ package org.koitharu.kotatsu.parsers.site.madara import org.jsoup.nodes.Element import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.* +import org.koitharu.kotatsu.parsers.model.Manga +import org.koitharu.kotatsu.parsers.model.MangaChapter +import org.koitharu.kotatsu.parsers.model.MangaSource +import org.koitharu.kotatsu.parsers.model.MangaState import org.koitharu.kotatsu.parsers.util.attrAsAbsoluteUrlOrNull +import org.koitharu.kotatsu.parsers.util.domain import org.koitharu.kotatsu.parsers.util.mapToSet import org.koitharu.kotatsu.parsers.util.selectFirstOrThrow import java.util.* @MangaSourceParser("TATAKAE_SCANS", "Tatakae Scans", "pt") internal class TatakaeScansParser(context: MangaLoaderContext) : - Madara6Parser(context, MangaSource.TATAKAE_SCANS, "tatakaescan.com") { - - override val datePattern: String = "dd 'de' MMMMM 'de' yyyy" - - override fun parseDetails(manga: Manga, body: Element, chapters: List): Manga { - val root = body.selectFirstOrThrow(".site-content") - val postContent = root.selectFirstOrThrow(".post-content") - val tags = postContent.getElementsContainingOwnText("Gênero") - .firstOrNull()?.tableValue() - ?.getElementsByAttributeValueContaining("href", tagPrefix) - ?.mapToSet { a -> a.asMangaTag() } ?: manga.tags - return manga.copy( - largeCoverUrl = root.selectFirst("picture") - ?.selectFirst("img[data-src]") - ?.attrAsAbsoluteUrlOrNull("data-src"), - description = (root.selectFirst(".detail-content") - ?: root.selectFirstOrThrow(".manga-excerpt")).html(), - author = postContent.getElementsContainingOwnText("Autor") - .firstOrNull()?.tableValue()?.text()?.trim(), - state = postContent.getElementsContainingOwnText("Status") - .firstOrNull()?.tableValue()?.text()?.asMangaState(), - tags = tags, - isNsfw = body.hasClass("adult-content"), - chapters = chapters, - ) - } - - override fun String.asMangaState() = when (trim().lowercase(Locale.ROOT)) { - "em lançamento" -> MangaState.ONGOING - - else -> null - } - - override fun getFaviconUrl(): String { - return "https://${getDomain()}/wp-content/uploads/2022/07/cropped-favicon-180x180.png" - } - -} \ No newline at end of file + Madara6Parser(context, MangaSource.TATAKAE_SCANS, "tatakaescan.com") { + + override val datePattern: String = "dd 'de' MMMMM 'de' yyyy" + + override fun parseDetails(manga: Manga, body: Element, chapters: List): Manga { + val root = body.selectFirstOrThrow(".site-content") + val postContent = root.selectFirstOrThrow(".post-content") + val tags = postContent.getElementsContainingOwnText("Gênero") + .firstOrNull()?.tableValue() + ?.getElementsByAttributeValueContaining("href", tagPrefix) + ?.mapToSet { a -> a.asMangaTag() } ?: manga.tags + return manga.copy( + largeCoverUrl = root.selectFirst("picture") + ?.selectFirst("img[data-src]") + ?.attrAsAbsoluteUrlOrNull("data-src"), + description = (root.selectFirst(".detail-content") + ?: root.selectFirstOrThrow(".manga-excerpt")).html(), + author = postContent.getElementsContainingOwnText("Autor") + .firstOrNull()?.tableValue()?.text()?.trim(), + state = postContent.getElementsContainingOwnText("Status") + .firstOrNull()?.tableValue()?.text()?.asMangaState(), + tags = tags, + isNsfw = body.hasClass("adult-content"), + chapters = chapters, + ) + } + + override fun String.asMangaState() = when (trim().lowercase(Locale.ROOT)) { + "em lançamento" -> MangaState.ONGOING + + else -> null + } + + override fun getFaviconUrl(): String { + return "https://${domain}/wp-content/uploads/2022/07/cropped-favicon-180x180.png" + } + +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt index 4892f507..8482aa43 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt @@ -16,509 +16,529 @@ import java.text.SimpleDateFormat import java.util.* internal abstract class MangaReaderParser( - source: MangaSource, - pageSize: Int, - searchPageSize: Int -) : PagedMangaParser(source, pageSize, searchPageSize) { - - override val sortOrders: Set - get() = EnumSet.of(SortOrder.UPDATED, SortOrder.POPULARITY, SortOrder.ALPHABETICAL, SortOrder.NEWEST) - - protected val idLocale - get() = Locale("in", "ID") - - abstract val listUrl: String - abstract val tableMode: Boolean - open val chapterDateFormat = SimpleDateFormat("MMM d, yyyy", idLocale) - - private var tagCache: ArrayMap? = null - private val mutex = Mutex() - private var lastSearchPage = 1 - - override suspend fun getDetails(manga: Manga): Manga { - val docs = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml() - val chapters = docs.select("#chapterlist > ul > li").reversed().mapChapters { index, element -> - val url = element.selectFirst("a")?.attrAsRelativeUrl("href") ?: return@mapChapters null - MangaChapter( - id = generateUid(url), - name = element.selectFirst(".chapternum")?.text() ?: "Chapter ${index + 1}", - url = url, - number = index + 1, - scanlator = null, - uploadDate = chapterDateFormat.tryParse(element.selectFirst(".chapterdate")?.text()), - branch = null, - source = source - ) - } - return if (tableMode) parseInfoTable(docs, manga, chapters) else parseInfoList(docs, manga, chapters) - } - - open suspend fun parseInfoTable(docs: Document, manga: Manga, chapters: List): Manga { - val mangaInfo = docs.selectFirst("div.seriestucontent > div.seriestucontentr") - val mangaState = mangaInfo?.selectFirst(".infotable td:contains(Status)")?.lastElementSibling()?.let { - when (it.text()) { - "Ongoing" -> MangaState.ONGOING - "Completed" -> MangaState.FINISHED - else -> null - } - } - val tagMap = getOrCreateTagMap() - val tags = mangaInfo?.select(".seriestugenre > a")?.mapNotNullToSet { tagMap[it.text()] } - - return manga.copy( - description = mangaInfo?.selectFirst("div.entry-content")?.html(), - state = mangaState, - author = mangaInfo?.selectFirst(".infotable td:contains(Author)")?.lastElementSibling()?.text(), - isNsfw = docs.selectFirst(".restrictcontainer") != null, - tags = tags.orEmpty(), - chapters = chapters, - ) - } - - open suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { - val mangaState = docs.selectFirst(".info-left .tsinfo div:contains(Status)")?.lastElementChild()?.let { - when (it.text()) { - "Ongoing" -> MangaState.ONGOING - "Completed" -> MangaState.FINISHED - else -> null - } - } - val tagMap = getOrCreateTagMap() - val tags = docs.select(".info-right .mgen > a").mapNotNullToSet { tagMap[it.text()] } - - return manga.copy( - description = docs.selectFirst(".info-right div.entry-content > p")?.html(), - state = mangaState, - author = docs.selectFirst(".info-left .tsinfo div:contains(Author)")?.lastElementChild()?.text(), - isNsfw = docs.selectFirst(".info-right .alr") != null, - tags = tags, - chapters = chapters, - ) - } - - override suspend fun getListPage( - page: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder - ): List { - if (!query.isNullOrEmpty()) { - if (page > lastSearchPage) { - return emptyList() - } - - val url = buildString { - append("https://") - append(getDomain()) - append("/page/") - append(page) - append("/?s=") - append(query.urlEncoded()) - } - - val docs = context.httpGet(url).parseHtml() - lastSearchPage = docs.selectFirst(".pagination .next") - ?.previousElementSibling() - ?.text()?.toIntOrNull() ?: 1 - return parseMangaList(docs) - } - - val sortQuery = when (sortOrder) { - SortOrder.ALPHABETICAL -> "title" - SortOrder.NEWEST -> "latest" - SortOrder.POPULARITY -> "popular" - SortOrder.UPDATED -> "update" - else -> "" - } - val tagKey = "genre[]".urlEncoded() - val tagQuery = if (tags.isNullOrEmpty()) "" else tags.joinToString(separator = "&", prefix = "&") { "$tagKey=${it.key}" } - val url = buildString { - append("https://") - append(getDomain()) - append(listUrl) - append("/?order=") - append(sortQuery) - append(tagQuery) - append("&page=") - append(page) - } - - return parseMangaList(context.httpGet(url).parseHtml()) - } - - private fun parseMangaList(docs: Document): List { - return docs.select(".postbody .listupd .bs .bsx").mapNotNull { - val a = it.selectFirst("a") ?: return@mapNotNull null - val relativeUrl = a.attrAsRelativeUrl("href") - val rating = it.selectFirst(".numscore")?.text() - ?.toFloatOrNull()?.div(10) ?: RATING_UNKNOWN - - Manga( - id = generateUid(relativeUrl), - url = relativeUrl, - title = a.attr("title"), - altTitle = null, - publicUrl = a.attrAsAbsoluteUrl("href"), - rating = rating, - isNsfw = false, - coverUrl = it.selectFirst("img.ts-post-image")?.imageUrl().orEmpty(), - tags = emptySet(), - state = null, - author = null, - source = source - ) - } - } - - override suspend fun getPages(chapter: MangaChapter): List { - val chapterUrl = chapter.url.toAbsoluteUrl(getDomain()) - val docs = context.httpGet(chapterUrl).parseHtml() - val script = docs.selectFirstOrThrow("script:containsData(ts_reader)") - val images = JSONObject(script.data().substringAfter('(').substringBeforeLast(')')) - .getJSONArray("sources") - .getJSONObject(0) - .getJSONArray("images") - - val pages = ArrayList(images.length()) - for (i in 0 until images.length()) { - pages.add( - MangaPage( - id = generateUid(images.getString(i)), - url = images.getString(i), - referer = chapterUrl, - preview = null, - source = source - ) - ) - } - - return pages - } - - override suspend fun getTags(): Set { - return getOrCreateTagMap().values.toSet() - } - - protected suspend fun getOrCreateTagMap(): Map = mutex.withLock { - tagCache?.let { return@withLock it } - val tagMap = ArrayMap() - - val url = listUrl.toAbsoluteUrl(getDomain()) - val tagElements = context.httpGet(url).parseHtml().select("ul.genrez > li") - for (el in tagElements) { - if (el.text().isEmpty()) continue - - tagMap[el.text()] = MangaTag( - title = el.text(), - key = el.selectFirst("input")?.attr("value") ?: continue, - source = source - ) - } - - tagCache = tagMap - return@withLock tagMap - } - - private fun Element.imageUrl(): String { - return attrAsAbsoluteUrlOrNull("src") - ?: attrAsAbsoluteUrlOrNull("data-cfsrc") - ?: "" - } - - @MangaSourceParser("MANHWALAND", "Manhwaland", "id") - class ManhwaLandParser(override val context: MangaLoaderContext) : MangaReaderParser(MangaSource.MANHWALAND, pageSize = 20, searchPageSize = 10) { - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("manhwaland.guru", null) - - override val listUrl: String - get() = "/series" - override val tableMode: Boolean - get() = false - } - - @MangaSourceParser("SEKAIKOMIK", "Sekaikomik", "id") - class SekaikomikParser(override val context: MangaLoaderContext) : MangaReaderParser(MangaSource.SEKAIKOMIK, pageSize = 20, searchPageSize = 100) { - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("sekaikomik.pro", null) - - override val listUrl: String - get() = "/manga" - override val tableMode: Boolean - get() = false - override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMMM D, yyyy", idLocale) - } - - @MangaSourceParser("MANHWAINDO", "Manhwaindo", "id") - class ManhwaIndoParser(override val context: MangaLoaderContext) : MangaReaderParser(MangaSource.MANHWAINDO, pageSize = 20, searchPageSize = 10) { - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("manhwaindo.id", null) - - override val listUrl: String get() = "/series" - override val tableMode: Boolean get() = false - } - - @MangaSourceParser("MANHWALIST", "Manhwalist", "id") - class ManhwalistParser(override val context: MangaLoaderContext) : MangaReaderParser(MangaSource.MANHWALIST, pageSize = 24, searchPageSize = 10) { - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("manhwalist.in", null) - - override val listUrl: String = "/manga" - override val tableMode: Boolean get() = false - override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMMM d, yyyy", Locale.ENGLISH) - } - - @MangaSourceParser("KIRYUU", "Kiryuu", "id") - class KiryuuParser(override val context: MangaLoaderContext) : MangaReaderParser(MangaSource.KIRYUU, pageSize = 20, searchPageSize = 10) { - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("kiryuu.id", null) - - override val listUrl: String - get() = "/manga" - override val tableMode: Boolean - get() = true - override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMMM d, yyyy", idLocale) - } - - @MangaSourceParser("TURKTOON", "Turktoon", "tr") - class TurktoonParser(override val context: MangaLoaderContext) : MangaReaderParser(MangaSource.TURKTOON, pageSize = 20, searchPageSize = 10) { - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("turktoon.com", null) - - override val listUrl: String - get() = "/manga" - override val tableMode: Boolean - get() = false - override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMMM d, yyyy", Locale("tr", "TR")) - - override suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { - val mangaState = docs.selectFirst(".info-left .tsinfo div:contains(Durum)")?.lastElementChild()?.let { - when (it.text()) { - "Devam Ediyor" -> MangaState.ONGOING - "Tamamlandı" -> MangaState.FINISHED - else -> null - } - } - - return super.parseInfoList(docs, manga, chapters).copy(state = mangaState) - } - } - - @MangaSourceParser("WESTMANGA", "Westmanga", "id") - class WestmangaParser(override val context: MangaLoaderContext) : MangaReaderParser(MangaSource.WESTMANGA, pageSize = 20, searchPageSize = 10) { - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("westmanga.info", null) - - override val listUrl: String - get() = "/manga" - override val tableMode: Boolean - get() = true - override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMMM d, yyyy", Locale.ENGLISH) - } - - @MangaSourceParser("TEMPESTFANSUB", "Tempestfansub", "tr") - class TempestfansubParser(override val context: MangaLoaderContext) : MangaReaderParser(MangaSource.TEMPESTFANSUB, pageSize = 40, searchPageSize = 40) { - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("manga.tempestfansub.com", null) - - override val listUrl: String get() = "/manga" - override val tableMode: Boolean get() = true - override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMMM d, yyyy", Locale("tr", "TR")) - - override suspend fun parseInfoTable(docs: Document, manga: Manga, chapters: List): Manga { - val infoElement = docs.selectFirst("div.infox") - return manga.copy( - chapters = chapters, - description = infoElement?.selectFirst("div.entry-content")?.html(), - author = infoElement?.selectFirst(".flex-wrap div:contains(Artist)")?.lastElementSibling()?.text(), - tags = infoElement?.select(".wd-full .mgen > a") - ?.mapNotNullToSet { getOrCreateTagMap()[it.text()] } - .orEmpty(), - isNsfw = docs.selectFirst(".postbody .alr") != null, - ) - } - } - - @MangaSourceParser("MANHWADESU", "ManhwaDesu", "id") - class ManhwadesuParser(override val context: MangaLoaderContext) : MangaReaderParser(MangaSource.MANHWADESU, pageSize = 20, searchPageSize = 10) { - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("manhwadesu.org", null) - - override val listUrl: String get() = "/komik" - override val tableMode: Boolean get() = false - } - - @MangaSourceParser("MANGATALE", "MangaTale", "id") - class MangaTaleParser(override val context: MangaLoaderContext) : MangaReaderParser(MangaSource.MANGATALE, pageSize = 20, searchPageSize = 10) { - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("mangatale.co", null) - - override val listUrl: String get() = "/manga" - override val tableMode: Boolean get() = false - - override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMMM d, yyyy", Locale.ENGLISH) - - override suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { - val infoElement = docs.selectFirst("div.infox") - return manga.copy( - chapters = chapters, - description = infoElement?.selectFirst("div.entry-content")?.html(), - author = infoElement?.selectFirst(".flex-wrap div:contains(Author)")?.lastElementSibling()?.text(), - tags = infoElement?.select(".wd-full .mgen > a") - ?.mapNotNullToSet { getOrCreateTagMap()[it.text()] } - .orEmpty(), - isNsfw = docs.selectFirst(".postbody .alr") != null, - ) - } - } - - @MangaSourceParser("DRAGONTRANSLATION", "DragonTranslation", "es") - class DragonTranslationParser(override val context: MangaLoaderContext) : MangaReaderParser(MangaSource.DRAGONTRANSLATION, pageSize = 20, searchPageSize = 10) { - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("dragontranslation.com", null) - - override val listUrl: String get() = "/manga" - override val tableMode: Boolean get() = false - - override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMMM d, yyyy", Locale.ENGLISH) - - override suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { - val infoElement = docs.selectFirst("div.infox") - return manga.copy( - chapters = chapters, - description = infoElement?.selectFirst("div.entry-content")?.html(), - author = infoElement?.selectFirst(".flex-wrap div:contains(Author)")?.lastElementSibling()?.text(), - tags = infoElement?.select(".wd-full .mgen > a") - ?.mapNotNullToSet { getOrCreateTagMap()[it.text()] } - .orEmpty(), - isNsfw = docs.selectFirst(".postbody .alr") != null, - ) - } - } - - @MangaSourceParser("ASURATR", "Asura Scans (tr)", "tr") - class AsuraTRParser(override val context: MangaLoaderContext) : MangaReaderParser(MangaSource.ASURATR, pageSize = 30, searchPageSize = 10) { - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("asurascanstr.com", null) - - override val listUrl: String get() = "/manga" - override val tableMode: Boolean get() = false - - override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", Locale("tr")) - - override suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { - val infoElement = docs.selectFirst("div.infox") - return manga.copy( - chapters = chapters, - description = infoElement?.selectFirst("div.entry-content")?.html(), - author = infoElement?.selectFirst(".flex-wrap div:contains(Yazar)")?.lastElementSibling()?.text(), - tags = infoElement?.select(".wd-full .mgen > a") - ?.mapNotNullToSet { getOrCreateTagMap()[it.text()] } - .orEmpty(), - isNsfw = docs.selectFirst(".postbody .alr") != null, - ) - } - } - - @MangaSourceParser("KOMIKTAP", "KomikTap", "id") - class KomikTapParser(override val context: MangaLoaderContext) : MangaReaderParser(MangaSource.KOMIKTAP, pageSize = 15, searchPageSize = 10) { - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("92.87.6.124", null) - - override val listUrl: String - get() = "/manga" - override val tableMode: Boolean - get() = true - - override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", idLocale) - } - - @MangaSourceParser("ASURASCANS", "Asura Scans", "en") - class AsuraScansParser(override val context: MangaLoaderContext) : MangaReaderParser(MangaSource.ASURASCANS, pageSize = 20, searchPageSize = 10) { - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("asurascans.com", null) - - override val listUrl: String - get() = "/manga" - override val tableMode: Boolean - get() = false - - override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH) - - override suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { - val infoElement = docs.selectFirst("div.infox") - return manga.copy( - chapters = chapters, - description = infoElement?.selectFirst("div.entry-content")?.html(), - author = infoElement?.selectFirst(".flex-wrap div:contains(Author)")?.lastElementSibling()?.text(), - tags = infoElement?.select(".wd-full .mgen > a") - ?.mapNotNullToSet { getOrCreateTagMap()[it.text()] } - .orEmpty(), - isNsfw = docs.selectFirst(".postbody .alr") != null, - ) - } - } - - @MangaSourceParser("COSMICSCANS", "CosmicScans", "en") - class CosmicScansParser(override val context: MangaLoaderContext) : MangaReaderParser(MangaSource.COSMICSCANS, pageSize = 20, searchPageSize = 10) { - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("cosmicscans.com", null) - - override val listUrl: String - get() = "/manga" - override val tableMode: Boolean - get() = false - - override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH) - - override suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { - val infoElement = docs.selectFirst("div.infox") - return manga.copy( - chapters = chapters, - description = infoElement?.selectFirst("div.entry-content")?.html(), - author = infoElement?.selectFirst(".flex-wrap div:contains(Author)")?.lastElementSibling()?.text(), - tags = infoElement?.select(".wd-full .mgen > a") - ?.mapNotNullToSet { getOrCreateTagMap()[it.text()] } - .orEmpty(), - ) - } - } - - @MangaSourceParser("MANGASUSU", "Mangasusu", "id") - class MangasusuParser(override val context: MangaLoaderContext) : MangaReaderParser(MangaSource.MANGASUSU, pageSize = 20, searchPageSize = 10) { - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("92.84.132.251", null) - - override val listUrl: String - get() = "/project" - override val tableMode: Boolean - get() = true - - override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH) - } - - @MangaSourceParser("KOMIKINDO", "KomikIndo", "id") - class KomikIndoParser(override val context: MangaLoaderContext) : MangaReaderParser(MangaSource.KOMIKINDO, pageSize = 20, searchPageSize = 10) { - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("komikindo.co", null) - - override val listUrl: String - get() = "/project" - override val tableMode: Boolean - get() = true - - override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", idLocale) - } - - @MangaSourceParser("KOMIKMANGA", "KomikManga", "id") - class KomikMangaParser(override val context: MangaLoaderContext) : MangaReaderParser(MangaSource.KOMIKMANGA, pageSize = 20, searchPageSize = 10) { - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("komikmanga.xyz", null) - - override val listUrl: String - get() = "/project" - override val tableMode: Boolean - get() = true - - override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH) - } + context: MangaLoaderContext, + source: MangaSource, + pageSize: Int, + searchPageSize: Int +) : PagedMangaParser(context, source, pageSize, searchPageSize) { + + override val sortOrders: Set + get() = EnumSet.of(SortOrder.UPDATED, SortOrder.POPULARITY, SortOrder.ALPHABETICAL, SortOrder.NEWEST) + + protected val idLocale + get() = Locale("in", "ID") + + abstract val listUrl: String + abstract val tableMode: Boolean + open val chapterDateFormat = SimpleDateFormat("MMM d, yyyy", idLocale) + + private var tagCache: ArrayMap? = null + private val mutex = Mutex() + private var lastSearchPage = 1 + + override suspend fun getDetails(manga: Manga): Manga { + val docs = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val chapters = docs.select("#chapterlist > ul > li").reversed().mapChapters { index, element -> + val url = element.selectFirst("a")?.attrAsRelativeUrl("href") ?: return@mapChapters null + MangaChapter( + id = generateUid(url), + name = element.selectFirst(".chapternum")?.text() ?: "Chapter ${index + 1}", + url = url, + number = index + 1, + scanlator = null, + uploadDate = chapterDateFormat.tryParse(element.selectFirst(".chapterdate")?.text()), + branch = null, + source = source + ) + } + return if (tableMode) parseInfoTable(docs, manga, chapters) else parseInfoList(docs, manga, chapters) + } + + open suspend fun parseInfoTable(docs: Document, manga: Manga, chapters: List): Manga { + val mangaInfo = docs.selectFirst("div.seriestucontent > div.seriestucontentr") + val mangaState = mangaInfo?.selectFirst(".infotable td:contains(Status)")?.lastElementSibling()?.let { + when (it.text()) { + "Ongoing" -> MangaState.ONGOING + "Completed" -> MangaState.FINISHED + else -> null + } + } + val tagMap = getOrCreateTagMap() + val tags = mangaInfo?.select(".seriestugenre > a")?.mapNotNullToSet { tagMap[it.text()] } + + return manga.copy( + description = mangaInfo?.selectFirst("div.entry-content")?.html(), + state = mangaState, + author = mangaInfo?.selectFirst(".infotable td:contains(Author)")?.lastElementSibling()?.text(), + isNsfw = docs.selectFirst(".restrictcontainer") != null, + tags = tags.orEmpty(), + chapters = chapters, + ) + } + + open suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { + val mangaState = docs.selectFirst(".info-left .tsinfo div:contains(Status)")?.lastElementChild()?.let { + when (it.text()) { + "Ongoing" -> MangaState.ONGOING + "Completed" -> MangaState.FINISHED + else -> null + } + } + val tagMap = getOrCreateTagMap() + val tags = docs.select(".info-right .mgen > a").mapNotNullToSet { tagMap[it.text()] } + + return manga.copy( + description = docs.selectFirst(".info-right div.entry-content > p")?.html(), + state = mangaState, + author = docs.selectFirst(".info-left .tsinfo div:contains(Author)")?.lastElementChild()?.text(), + isNsfw = docs.selectFirst(".info-right .alr") != null, + tags = tags, + chapters = chapters, + ) + } + + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder + ): List { + if (!query.isNullOrEmpty()) { + if (page > lastSearchPage) { + return emptyList() + } + + val url = buildString { + append("https://") + append(domain) + append("/page/") + append(page) + append("/?s=") + append(query.urlEncoded()) + } + + val docs = webClient.httpGet(url).parseHtml() + lastSearchPage = docs.selectFirst(".pagination .next") + ?.previousElementSibling() + ?.text()?.toIntOrNull() ?: 1 + return parseMangaList(docs) + } + + val sortQuery = when (sortOrder) { + SortOrder.ALPHABETICAL -> "title" + SortOrder.NEWEST -> "latest" + SortOrder.POPULARITY -> "popular" + SortOrder.UPDATED -> "update" + else -> "" + } + val tagKey = "genre[]".urlEncoded() + val tagQuery = + if (tags.isNullOrEmpty()) "" else tags.joinToString(separator = "&", prefix = "&") { "$tagKey=${it.key}" } + val url = buildString { + append("https://") + append(domain) + append(listUrl) + append("/?order=") + append(sortQuery) + append(tagQuery) + append("&page=") + append(page) + } + + return parseMangaList(webClient.httpGet(url).parseHtml()) + } + + private fun parseMangaList(docs: Document): List { + return docs.select(".postbody .listupd .bs .bsx").mapNotNull { + val a = it.selectFirst("a") ?: return@mapNotNull null + val relativeUrl = a.attrAsRelativeUrl("href") + val rating = it.selectFirst(".numscore")?.text() + ?.toFloatOrNull()?.div(10) ?: RATING_UNKNOWN + + Manga( + id = generateUid(relativeUrl), + url = relativeUrl, + title = a.attr("title"), + altTitle = null, + publicUrl = a.attrAsAbsoluteUrl("href"), + rating = rating, + isNsfw = false, + coverUrl = it.selectFirst("img.ts-post-image")?.imageUrl().orEmpty(), + tags = emptySet(), + state = null, + author = null, + source = source + ) + } + } + + override suspend fun getPages(chapter: MangaChapter): List { + val chapterUrl = chapter.url.toAbsoluteUrl(domain) + val docs = webClient.httpGet(chapterUrl).parseHtml() + val script = docs.selectFirstOrThrow("script:containsData(ts_reader)") + val images = JSONObject(script.data().substringAfter('(').substringBeforeLast(')')) + .getJSONArray("sources") + .getJSONObject(0) + .getJSONArray("images") + + val pages = ArrayList(images.length()) + for (i in 0 until images.length()) { + pages.add( + MangaPage( + id = generateUid(images.getString(i)), + url = images.getString(i), + referer = chapterUrl, + preview = null, + source = source + ) + ) + } + + return pages + } + + override suspend fun getTags(): Set { + return getOrCreateTagMap().values.toSet() + } + + protected suspend fun getOrCreateTagMap(): Map = mutex.withLock { + tagCache?.let { return@withLock it } + val tagMap = ArrayMap() + + val url = listUrl.toAbsoluteUrl(domain) + val tagElements = webClient.httpGet(url).parseHtml().select("ul.genrez > li") + for (el in tagElements) { + if (el.text().isEmpty()) continue + + tagMap[el.text()] = MangaTag( + title = el.text(), + key = el.selectFirst("input")?.attr("value") ?: continue, + source = source + ) + } + + tagCache = tagMap + return@withLock tagMap + } + + private fun Element.imageUrl(): String { + return attrAsAbsoluteUrlOrNull("src") + ?: attrAsAbsoluteUrlOrNull("data-cfsrc") + ?: "" + } + + @MangaSourceParser("MANHWALAND", "Manhwaland", "id") + class ManhwaLandParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.MANHWALAND, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("manhwaland.guru", null) + + override val listUrl: String + get() = "/series" + override val tableMode: Boolean + get() = false + } + + @MangaSourceParser("SEKAIKOMIK", "Sekaikomik", "id") + class SekaikomikParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.SEKAIKOMIK, pageSize = 20, searchPageSize = 100) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("sekaikomik.pro", null) + + override val listUrl: String + get() = "/manga" + override val tableMode: Boolean + get() = false + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMMM D, yyyy", idLocale) + } + + @MangaSourceParser("MANHWAINDO", "Manhwaindo", "id") + class ManhwaIndoParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.MANHWAINDO, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("manhwaindo.id", null) + + override val listUrl: String get() = "/series" + override val tableMode: Boolean get() = false + } + + @MangaSourceParser("MANHWALIST", "Manhwalist", "id") + class ManhwalistParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.MANHWALIST, pageSize = 24, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("manhwalist.in", null) + + override val listUrl: String = "/manga" + override val tableMode: Boolean get() = false + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMMM d, yyyy", Locale.ENGLISH) + } + + @MangaSourceParser("KIRYUU", "Kiryuu", "id") + class KiryuuParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.KIRYUU, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("kiryuu.id", null) + + override val listUrl: String + get() = "/manga" + override val tableMode: Boolean + get() = true + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMMM d, yyyy", idLocale) + } + + @MangaSourceParser("TURKTOON", "Turktoon", "tr") + class TurktoonParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.TURKTOON, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("turktoon.com", null) + + override val listUrl: String + get() = "/manga" + override val tableMode: Boolean + get() = false + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMMM d, yyyy", Locale("tr", "TR")) + + override suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { + val mangaState = docs.selectFirst(".info-left .tsinfo div:contains(Durum)")?.lastElementChild()?.let { + when (it.text()) { + "Devam Ediyor" -> MangaState.ONGOING + "Tamamlandı" -> MangaState.FINISHED + else -> null + } + } + + return super.parseInfoList(docs, manga, chapters).copy(state = mangaState) + } + } + + @MangaSourceParser("WESTMANGA", "Westmanga", "id") + class WestmangaParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.WESTMANGA, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("westmanga.info", null) + + override val listUrl: String + get() = "/manga" + override val tableMode: Boolean + get() = true + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMMM d, yyyy", Locale.ENGLISH) + } + + @MangaSourceParser("TEMPESTFANSUB", "Tempestfansub", "tr") + class TempestfansubParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.TEMPESTFANSUB, pageSize = 40, searchPageSize = 40) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("manga.tempestfansub.com", null) + + override val listUrl: String get() = "/manga" + override val tableMode: Boolean get() = true + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMMM d, yyyy", Locale("tr", "TR")) + + override suspend fun parseInfoTable(docs: Document, manga: Manga, chapters: List): Manga { + val infoElement = docs.selectFirst("div.infox") + return manga.copy( + chapters = chapters, + description = infoElement?.selectFirst("div.entry-content")?.html(), + author = infoElement?.selectFirst(".flex-wrap div:contains(Artist)")?.lastElementSibling()?.text(), + tags = infoElement?.select(".wd-full .mgen > a") + ?.mapNotNullToSet { getOrCreateTagMap()[it.text()] } + .orEmpty(), + isNsfw = docs.selectFirst(".postbody .alr") != null, + ) + } + } + + @MangaSourceParser("MANHWADESU", "ManhwaDesu", "id") + class ManhwadesuParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.MANHWADESU, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("manhwadesu.org", null) + + override val listUrl: String get() = "/komik" + override val tableMode: Boolean get() = false + } + + @MangaSourceParser("MANGATALE", "MangaTale", "id") + class MangaTaleParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.MANGATALE, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("mangatale.co", null) + + override val listUrl: String get() = "/manga" + override val tableMode: Boolean get() = false + + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMMM d, yyyy", Locale.ENGLISH) + + override suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { + val infoElement = docs.selectFirst("div.infox") + return manga.copy( + chapters = chapters, + description = infoElement?.selectFirst("div.entry-content")?.html(), + author = infoElement?.selectFirst(".flex-wrap div:contains(Author)")?.lastElementSibling()?.text(), + tags = infoElement?.select(".wd-full .mgen > a") + ?.mapNotNullToSet { getOrCreateTagMap()[it.text()] } + .orEmpty(), + isNsfw = docs.selectFirst(".postbody .alr") != null, + ) + } + } + + @MangaSourceParser("DRAGONTRANSLATION", "DragonTranslation", "es") + class DragonTranslationParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.DRAGONTRANSLATION, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("dragontranslation.com", null) + + override val listUrl: String get() = "/manga" + override val tableMode: Boolean get() = false + + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMMM d, yyyy", Locale.ENGLISH) + + override suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { + val infoElement = docs.selectFirst("div.infox") + return manga.copy( + chapters = chapters, + description = infoElement?.selectFirst("div.entry-content")?.html(), + author = infoElement?.selectFirst(".flex-wrap div:contains(Author)")?.lastElementSibling()?.text(), + tags = infoElement?.select(".wd-full .mgen > a") + ?.mapNotNullToSet { getOrCreateTagMap()[it.text()] } + .orEmpty(), + isNsfw = docs.selectFirst(".postbody .alr") != null, + ) + } + } + + @MangaSourceParser("ASURATR", "Asura Scans (tr)", "tr") + class AsuraTRParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.ASURATR, pageSize = 30, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("asurascanstr.com", null) + + override val listUrl: String get() = "/manga" + override val tableMode: Boolean get() = false + + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", Locale("tr")) + + override suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { + val infoElement = docs.selectFirst("div.infox") + return manga.copy( + chapters = chapters, + description = infoElement?.selectFirst("div.entry-content")?.html(), + author = infoElement?.selectFirst(".flex-wrap div:contains(Yazar)")?.lastElementSibling()?.text(), + tags = infoElement?.select(".wd-full .mgen > a") + ?.mapNotNullToSet { getOrCreateTagMap()[it.text()] } + .orEmpty(), + isNsfw = docs.selectFirst(".postbody .alr") != null, + ) + } + } + + @MangaSourceParser("KOMIKTAP", "KomikTap", "id") + class KomikTapParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.KOMIKTAP, pageSize = 15, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("92.87.6.124", null) + + override val listUrl: String + get() = "/manga" + override val tableMode: Boolean + get() = true + + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", idLocale) + } + + @MangaSourceParser("ASURASCANS", "Asura Scans", "en") + class AsuraScansParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.ASURASCANS, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("asurascans.com", null) + + override val listUrl: String + get() = "/manga" + override val tableMode: Boolean + get() = false + + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH) + + override suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { + val infoElement = docs.selectFirst("div.infox") + return manga.copy( + chapters = chapters, + description = infoElement?.selectFirst("div.entry-content")?.html(), + author = infoElement?.selectFirst(".flex-wrap div:contains(Author)")?.lastElementSibling()?.text(), + tags = infoElement?.select(".wd-full .mgen > a") + ?.mapNotNullToSet { getOrCreateTagMap()[it.text()] } + .orEmpty(), + isNsfw = docs.selectFirst(".postbody .alr") != null, + ) + } + } + + @MangaSourceParser("COSMICSCANS", "CosmicScans", "en") + class CosmicScansParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.COSMICSCANS, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("cosmicscans.com", null) + + override val listUrl: String + get() = "/manga" + override val tableMode: Boolean + get() = false + + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH) + + override suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { + val infoElement = docs.selectFirst("div.infox") + return manga.copy( + chapters = chapters, + description = infoElement?.selectFirst("div.entry-content")?.html(), + author = infoElement?.selectFirst(".flex-wrap div:contains(Author)")?.lastElementSibling()?.text(), + tags = infoElement?.select(".wd-full .mgen > a") + ?.mapNotNullToSet { getOrCreateTagMap()[it.text()] } + .orEmpty(), + ) + } + } + + @MangaSourceParser("MANGASUSU", "Mangasusu", "id") + class MangasusuParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.MANGASUSU, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("92.84.132.251", null) + + override val listUrl: String + get() = "/project" + override val tableMode: Boolean + get() = true + + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH) + } + + @MangaSourceParser("KOMIKINDO", "KomikIndo", "id") + class KomikIndoParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.KOMIKINDO, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("komikindo.co", null) + + override val listUrl: String + get() = "/project" + override val tableMode: Boolean + get() = true + + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", idLocale) + } + + @MangaSourceParser("KOMIKMANGA", "KomikManga", "id") + class KomikMangaParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.KOMIKMANGA, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("komikmanga.xyz", null) + + override val listUrl: String + get() = "/project" + override val tableMode: Boolean + get() = true + + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH) + } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/multichan/ChanParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/multichan/ChanParser.kt index 5792505d..2a886fca 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/multichan/ChanParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/multichan/ChanParser.kt @@ -1,5 +1,6 @@ package org.koitharu.kotatsu.parsers.site.multichan +import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaParser import org.koitharu.kotatsu.parsers.MangaParserAuthProvider import org.koitharu.kotatsu.parsers.exception.AuthRequiredException @@ -8,172 +9,175 @@ import org.koitharu.kotatsu.parsers.util.* import java.text.SimpleDateFormat import java.util.* -internal abstract class ChanParser(source: MangaSource) : MangaParser(source), MangaParserAuthProvider { +internal abstract class ChanParser( + context: MangaLoaderContext, + source: MangaSource, +) : MangaParser(context, source), MangaParserAuthProvider { - override val sortOrders: Set = EnumSet.of( - SortOrder.NEWEST, - SortOrder.POPULARITY, - SortOrder.ALPHABETICAL, - ) + override val sortOrders: Set = EnumSet.of( + SortOrder.NEWEST, + SortOrder.POPULARITY, + SortOrder.ALPHABETICAL, + ) - override val authUrl: String - get() = "https://${getDomain()}" + override val authUrl: String + get() = "https://${domain}" - override val isAuthorized: Boolean - get() = context.cookieJar.getCookies(getDomain()).any { it.name == "dle_user_id" } + override val isAuthorized: Boolean + get() = context.cookieJar.getCookies(domain).any { it.name == "dle_user_id" } - override suspend fun getList( - offset: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - val domain = getDomain() - val url = when { - !query.isNullOrEmpty() -> { - if (offset != 0) { - return emptyList() - } - "https://$domain/?do=search&subaction=search&story=${query.urlEncoded()}" - } + override suspend fun getList( + offset: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + val domain = domain + val url = when { + !query.isNullOrEmpty() -> { + if (offset != 0) { + return emptyList() + } + "https://$domain/?do=search&subaction=search&story=${query.urlEncoded()}" + } - !tags.isNullOrEmpty() -> tags.joinToString( - prefix = "https://$domain/tags/", - postfix = "&n=${getSortKey2(sortOrder)}?offset=$offset", - separator = "+", - ) { tag -> tag.key } + !tags.isNullOrEmpty() -> tags.joinToString( + prefix = "https://$domain/tags/", + postfix = "&n=${getSortKey2(sortOrder)}?offset=$offset", + separator = "+", + ) { tag -> tag.key } - else -> "https://$domain/${getSortKey(sortOrder)}?offset=$offset" - } - val doc = context.httpGet(url).parseHtml() - val root = doc.body().selectFirst("div.main_fon")?.getElementById("content") - ?: doc.parseFailed("Cannot find root") - return root.select("div.content_row").mapNotNull { row -> - val a = row.selectFirst("div.manga_row1")?.selectFirst("h2")?.selectFirst("a") - ?: return@mapNotNull null - val href = a.attrAsRelativeUrl("href") - Manga( - id = generateUid(href), - url = href, - publicUrl = href.toAbsoluteUrl(a.host ?: domain), - altTitle = a.attr("title"), - title = a.text().substringAfterLast('(').substringBeforeLast(')'), - author = row.getElementsByAttributeValueStarting( - "href", - "/mangaka", - ).firstOrNull()?.text(), - coverUrl = row.selectFirst("div.manga_images")?.selectFirst("img") - ?.absUrl("src").orEmpty(), - tags = runCatching { - row.selectFirst("div.genre")?.select("a")?.mapToSet { - MangaTag( - title = it.text().toTagName(), - key = it.attr("href").substringAfterLast('/').urlEncoded(), - source = source, - ) - } - }.getOrNull().orEmpty(), - rating = RATING_UNKNOWN, - state = null, - isNsfw = false, - source = source, - ) - } - } + else -> "https://$domain/${getSortKey(sortOrder)}?offset=$offset" + } + val doc = webClient.httpGet(url).parseHtml() + val root = doc.body().selectFirst("div.main_fon")?.getElementById("content") + ?: doc.parseFailed("Cannot find root") + return root.select("div.content_row").mapNotNull { row -> + val a = row.selectFirst("div.manga_row1")?.selectFirst("h2")?.selectFirst("a") + ?: return@mapNotNull null + val href = a.attrAsRelativeUrl("href") + Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(a.host ?: domain), + altTitle = a.attr("title"), + title = a.text().substringAfterLast('(').substringBeforeLast(')'), + author = row.getElementsByAttributeValueStarting( + "href", + "/mangaka", + ).firstOrNull()?.text(), + coverUrl = row.selectFirst("div.manga_images")?.selectFirst("img") + ?.absUrl("src").orEmpty(), + tags = runCatching { + row.selectFirst("div.genre")?.select("a")?.mapToSet { + MangaTag( + title = it.text().toTagName(), + key = it.attr("href").substringAfterLast('/').urlEncoded(), + source = source, + ) + } + }.getOrNull().orEmpty(), + rating = RATING_UNKNOWN, + state = null, + isNsfw = false, + source = source, + ) + } + } - override suspend fun getDetails(manga: Manga): Manga { - val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml() - val root = doc.body().getElementById("dle-content") ?: doc.parseFailed("Cannot find root") - val dateFormat = SimpleDateFormat("yyyy-MM-dd", Locale.US) - return manga.copy( - description = root.getElementById("description")?.html()?.substringBeforeLast(" - val href = tr?.selectFirst("a")?.attrAsRelativeUrlOrNull("href") - ?: return@mapChapters null - MangaChapter( - id = generateUid(href), - name = tr.selectFirst("a")?.text().orEmpty(), - number = i + 1, - url = href, - scanlator = null, - branch = null, - uploadDate = dateFormat.tryParse(tr.selectFirst("div.date")?.text()), - source = source, - ) - }, - ) - } + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val root = doc.body().getElementById("dle-content") ?: doc.parseFailed("Cannot find root") + val dateFormat = SimpleDateFormat("yyyy-MM-dd", Locale.US) + return manga.copy( + description = root.getElementById("description")?.html()?.substringBeforeLast(" + val href = tr?.selectFirst("a")?.attrAsRelativeUrlOrNull("href") + ?: return@mapChapters null + MangaChapter( + id = generateUid(href), + name = tr.selectFirst("a")?.text().orEmpty(), + number = i + 1, + url = href, + scanlator = null, + branch = null, + uploadDate = dateFormat.tryParse(tr.selectFirst("div.date")?.text()), + source = source, + ) + }, + ) + } - override suspend fun getPages(chapter: MangaChapter): List { - val fullUrl = chapter.url.toAbsoluteUrl(getDomain()) - val doc = context.httpGet(fullUrl).parseHtml() - val scripts = doc.select("script") - for (script in scripts) { - val data = script.html() - val pos = data.indexOf("\"fullimg") - if (pos == -1) { - continue - } - val json = data.substring(pos).substringAfter('[').substringBefore(';') - .substringBeforeLast(']') - val domain = getDomain() - return json.split(",").mapNotNull { - it.trim() - .removeSurrounding('"', '\'') - .toRelativeUrl(domain) - .takeUnless(String::isBlank) - }.map { url -> - MangaPage( - id = generateUid(url), - url = url, - preview = null, - referer = fullUrl, - source = source, - ) - } - } - doc.parseFailed("Pages list not found at ${chapter.url}") - } + override suspend fun getPages(chapter: MangaChapter): List { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + val scripts = doc.select("script") + for (script in scripts) { + val data = script.html() + val pos = data.indexOf("\"fullimg") + if (pos == -1) { + continue + } + val json = data.substring(pos).substringAfter('[').substringBefore(';') + .substringBeforeLast(']') + val domain = domain + return json.split(",").mapNotNull { + it.trim() + .removeSurrounding('"', '\'') + .toRelativeUrl(domain) + .takeUnless(String::isBlank) + }.map { url -> + MangaPage( + id = generateUid(url), + url = url, + preview = null, + referer = fullUrl, + source = source, + ) + } + } + doc.parseFailed("Pages list not found at ${chapter.url}") + } - override suspend fun getTags(): Set { - val domain = getDomain() - val doc = context.httpGet("https://$domain/mostfavorites&sort=manga").parseHtml() - val root = doc.body().selectFirst("div.main_fon")?.getElementById("side") - ?.select("ul")?.last() ?: doc.parseFailed("Cannot find root") - return root.select("li.sidetag").mapToSet { li -> - val a = li.children().lastOrNull() ?: li.parseFailed("a is null") - MangaTag( - title = a.text().toTagName(), - key = a.attr("href").substringAfterLast('/'), - source = source, - ) - } - } + override suspend fun getTags(): Set { + val domain = domain + val doc = webClient.httpGet("https://$domain/mostfavorites&sort=manga").parseHtml() + val root = doc.body().selectFirst("div.main_fon")?.getElementById("side") + ?.select("ul")?.last() ?: doc.parseFailed("Cannot find root") + return root.select("li.sidetag").mapToSet { li -> + val a = li.children().lastOrNull() ?: li.parseFailed("a is null") + MangaTag( + title = a.text().toTagName(), + key = a.attr("href").substringAfterLast('/'), + source = source, + ) + } + } - override suspend fun getUsername(): String { - val doc = context.httpGet("https://${getDomain()}").parseHtml().body() - val root = doc.requireElementById("top_user") - val a = root.getElementsByAttributeValueContaining("href", "/user/").firstOrNull() - ?: throw AuthRequiredException(source) - return a.attr("href").removeSuffix('/').substringAfterLast('/') - } + override suspend fun getUsername(): String { + val doc = webClient.httpGet("https://${domain}").parseHtml().body() + val root = doc.requireElementById("top_user") + val a = root.getElementsByAttributeValueContaining("href", "/user/").firstOrNull() + ?: throw AuthRequiredException(source) + return a.attr("href").removeSuffix('/').substringAfterLast('/') + } - private fun getSortKey(sortOrder: SortOrder) = - when (sortOrder) { - SortOrder.ALPHABETICAL -> "catalog" - SortOrder.POPULARITY -> "mostfavorites" - SortOrder.NEWEST -> "manga/new" - else -> "mostfavorites" - } + private fun getSortKey(sortOrder: SortOrder) = + when (sortOrder) { + SortOrder.ALPHABETICAL -> "catalog" + SortOrder.POPULARITY -> "mostfavorites" + SortOrder.NEWEST -> "manga/new" + else -> "mostfavorites" + } - private fun getSortKey2(sortOrder: SortOrder) = - when (sortOrder) { - SortOrder.ALPHABETICAL -> "abcasc" - SortOrder.POPULARITY -> "favdesc" - SortOrder.NEWEST -> "datedesc" - else -> "favdesc" - } + private fun getSortKey2(sortOrder: SortOrder) = + when (sortOrder) { + SortOrder.ALPHABETICAL -> "abcasc" + SortOrder.POPULARITY -> "favdesc" + SortOrder.NEWEST -> "datedesc" + else -> "favdesc" + } - private fun String.toTagName() = replace('_', ' ').toTitleCase() -} \ No newline at end of file + private fun String.toTagName() = replace('_', ' ').toTitleCase() +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/multichan/HenChanParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/multichan/HenChanParser.kt index 97a5f032..eec7465c 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/multichan/HenChanParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/multichan/HenChanParser.kt @@ -7,54 +7,54 @@ import org.koitharu.kotatsu.parsers.model.* import org.koitharu.kotatsu.parsers.util.* @MangaSourceParser("HENCHAN", "Хентай-тян", "ru") -internal class HenChanParser(override val context: MangaLoaderContext) : ChanParser(MangaSource.HENCHAN) { +internal class HenChanParser(context: MangaLoaderContext) : ChanParser(context, MangaSource.HENCHAN) { - override val configKeyDomain = ConfigKey.Domain( - "y.hentaichan.live", - arrayOf("y.hentaichan.live", "xxx.hentaichan.live", "xx.hentaichan.live", "hentaichan.live", "hentaichan.pro"), - ) + override val configKeyDomain = ConfigKey.Domain( + "y.hentaichan.live", + arrayOf("y.hentaichan.live", "xxx.hentaichan.live", "xx.hentaichan.live", "hentaichan.live", "hentaichan.pro"), + ) - override suspend fun getList( - offset: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - return super.getList(offset, query, tags, sortOrder).map { - it.copy( - coverUrl = it.coverUrl.replace("_blur", ""), - isNsfw = true, - ) - } - } + override suspend fun getList( + offset: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + return super.getList(offset, query, tags, sortOrder).map { + it.copy( + coverUrl = it.coverUrl.replace("_blur", ""), + isNsfw = true, + ) + } + } - override suspend fun getDetails(manga: Manga): Manga { - val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml() - val root = doc.body().requireElementById("dle-content") - val readLink = manga.url.replace("manga", "online") - return manga.copy( - description = root.getElementById("description")?.html()?.substringBeforeLast(" - table.select("div.manga") - }.mapNotNull { it.selectFirst("a") }.reversed().mapChapters { i, a -> - val href = a.attrAsRelativeUrl("href") - MangaChapter( - id = generateUid(href), - name = a.text().trim(), - number = i + 1, - url = href, - uploadDate = 0L, - source = source, - scanlator = null, - branch = null, - ) - }, - ) - } -} \ No newline at end of file + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val root = doc.body().requireElementById("dle-content") + return manga.copy( + description = root.getElementById("description")?.html()?.substringBeforeLast(" + table.select("div.manga") + }.mapNotNull { it.selectFirst("a") }.reversed().mapChapters { i, a -> + val href = a.attrAsRelativeUrl("href") + MangaChapter( + id = generateUid(href), + name = a.text().trim(), + number = i + 1, + url = href, + uploadDate = 0L, + source = source, + scanlator = null, + branch = null, + ) + }, + ) + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/rulib/MangaLibParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/rulib/MangaLibParser.kt index 27872a76..d00dd2dd 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/rulib/MangaLibParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/rulib/MangaLibParser.kt @@ -20,284 +20,284 @@ import java.text.SimpleDateFormat import java.util.* internal open class MangaLibParser( - override val context: MangaLoaderContext, - source: MangaSource, -) : PagedMangaParser(source, pageSize = 60), MangaParserAuthProvider { + context: MangaLoaderContext, + source: MangaSource, +) : PagedMangaParser(context, source, pageSize = 60), MangaParserAuthProvider { - override val configKeyDomain = ConfigKey.Domain("mangalib.me", null) + override val configKeyDomain = ConfigKey.Domain("mangalib.me", null) - override val authUrl: String - get() = "https://${getDomain()}/login" + override val authUrl: String + get() = "https://${domain}/login" - override val sortOrders: Set = EnumSet.of( - SortOrder.RATING, - SortOrder.ALPHABETICAL, - SortOrder.POPULARITY, - SortOrder.UPDATED, - SortOrder.NEWEST, - ) + override val sortOrders: Set = EnumSet.of( + SortOrder.RATING, + SortOrder.ALPHABETICAL, + SortOrder.POPULARITY, + SortOrder.UPDATED, + SortOrder.NEWEST, + ) - override suspend fun getListPage( - page: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - if (!query.isNullOrEmpty()) { - return if (page == searchPaginator.firstPage) search(query) else emptyList() - } - val url = buildString { - append("https://") - append(getDomain()) - append("/manga-list?dir=") - append(getSortKey(sortOrder)) - append("&page=") - append(page) - tags?.forEach { tag -> - append("&genres[include][]=") - append(tag.key) - } - } - val doc = context.httpGet(url).parseHtml() - val root = doc.body().getElementById("manga-list") ?: doc.parseFailed("Root not found") - val items = root.selectFirst("div.media-cards-grid")?.select("div.media-card-wrap") - ?: return emptyList() - return items.mapNotNull { card -> - val a = card.selectFirst("a.media-card") ?: return@mapNotNull null - val href = a.attrAsRelativeUrl("href") - Manga( - id = generateUid(href), - title = card.selectFirst("h3")?.text().orEmpty(), - coverUrl = a.absUrl("data-src"), - altTitle = null, - author = null, - rating = RATING_UNKNOWN, - url = href, - publicUrl = href.toAbsoluteUrl(a.host ?: getDomain()), - tags = emptySet(), - state = null, - isNsfw = false, - source = source, - ) - } - } + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + if (!query.isNullOrEmpty()) { + return if (page == searchPaginator.firstPage) search(query) else emptyList() + } + val url = buildString { + append("https://") + append(domain) + append("/manga-list?dir=") + append(getSortKey(sortOrder)) + append("&page=") + append(page) + tags?.forEach { tag -> + append("&genres[include][]=") + append(tag.key) + } + } + val doc = webClient.httpGet(url).parseHtml() + val root = doc.body().getElementById("manga-list") ?: doc.parseFailed("Root not found") + val items = root.selectFirst("div.media-cards-grid")?.select("div.media-card-wrap") + ?: return emptyList() + return items.mapNotNull { card -> + val a = card.selectFirst("a.media-card") ?: return@mapNotNull null + val href = a.attrAsRelativeUrl("href") + Manga( + id = generateUid(href), + title = card.selectFirst("h3")?.text().orEmpty(), + coverUrl = a.absUrl("data-src"), + altTitle = null, + author = null, + rating = RATING_UNKNOWN, + url = href, + publicUrl = href.toAbsoluteUrl(a.host ?: domain), + tags = emptySet(), + state = null, + isNsfw = false, + source = source, + ) + } + } - override suspend fun getDetails(manga: Manga): Manga { - val fullUrl = manga.url.toAbsoluteUrl(getDomain()) - val doc = context.httpGet("$fullUrl?section=info").parseHtml() - val root = doc.body().getElementById("main-page") ?: throw ParseException("Root not found", fullUrl) - val title = root.selectFirst("div.media-header__wrap")?.children() - val info = root.selectFirst("div.media-content") - val chaptersDoc = context.httpGet("$fullUrl?section=chapters").parseHtml() - val scripts = chaptersDoc.select("script") - val dateFormat = SimpleDateFormat("yyy-MM-dd", Locale.US) - var chapters: ChaptersListBuilder? = null - scripts@ for (script in scripts) { - val raw = script.html().lines() - for (line in raw) { - if (line.startsWith("window.__DATA__")) { - val json = JSONObject(line.substringAfter('=').substringBeforeLast(';')) - val list = json.getJSONObject("chapters").getJSONArray("list") - val id = json.optJSONObject("user")?.getLong("id")?.toString() ?: "not" - val total = list.length() - chapters = ChaptersListBuilder(total) - for (i in 0 until total) { - val item = list.getJSONObject(i) - val chapterId = item.getLong("chapter_id") - val scanlator = item.getStringOrNull("username") - val url = buildString { - if (isAuthorized) { - append(manga.url) - append("/v") - append(item.getInt("chapter_volume")) - append("/c") - append(item.getString("chapter_number")) - append("?ui=") - append(id) - } else { - append(manga.url) - append("/v") - append(item.getInt("chapter_volume")) - append("/c") - append(item.getString("chapter_number")) - } - } - val nameChapter = item.getStringOrNull("chapter_name") - val volume = item.getInt("chapter_volume") - val number = item.getString("chapter_number") - val fullNameChapter = "Том $volume. Глава $number" - chapters.add( - MangaChapter( - id = generateUid(chapterId), - url = url, - source = source, - number = total - i, - uploadDate = dateFormat.tryParse( - item.getString("chapter_created_at").substringBefore(" "), - ), - scanlator = scanlator, - branch = null, - name = if (nameChapter.isNullOrBlank()) fullNameChapter else "$fullNameChapter - $nameChapter", - ), - ) - } - chapters.reverse() - break@scripts - } - } - } - return manga.copy( - title = title?.getOrNull(0)?.text()?.takeUnless(String::isBlank) ?: manga.title, - altTitle = title?.getOrNull(1)?.text()?.substringBefore('/')?.trim(), - rating = root.selectFirst("div.media-stats-item__score") - ?.selectFirst("span") - ?.text()?.toFloatOrNull()?.div(5f) ?: manga.rating, - author = info?.getElementsMatchingOwnText("Автор")?.firstOrNull() - ?.nextElementSibling()?.text() ?: manga.author, - tags = info?.selectFirst("div.media-tags") - ?.select("a.media-tag-item")?.mapNotNullToSet { a -> - val href = a.attr("href") - if (href.contains("genres")) { - MangaTag( - title = a.text().toTitleCase(), - key = href.substringAfterLast('='), - source = source, - ) - } else null - } ?: manga.tags, - isNsfw = isNsfw(doc), - description = info?.selectFirst("div.media-description__text")?.html(), - chapters = chapters?.toList(), - ) - } + override suspend fun getDetails(manga: Manga): Manga { + val fullUrl = manga.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet("$fullUrl?section=info").parseHtml() + val root = doc.body().getElementById("main-page") ?: throw ParseException("Root not found", fullUrl) + val title = root.selectFirst("div.media-header__wrap")?.children() + val info = root.selectFirst("div.media-content") + val chaptersDoc = webClient.httpGet("$fullUrl?section=chapters").parseHtml() + val scripts = chaptersDoc.select("script") + val dateFormat = SimpleDateFormat("yyy-MM-dd", Locale.US) + var chapters: ChaptersListBuilder? = null + scripts@ for (script in scripts) { + val raw = script.html().lines() + for (line in raw) { + if (line.startsWith("window.__DATA__")) { + val json = JSONObject(line.substringAfter('=').substringBeforeLast(';')) + val list = json.getJSONObject("chapters").getJSONArray("list") + val id = json.optJSONObject("user")?.getLong("id")?.toString() ?: "not" + val total = list.length() + chapters = ChaptersListBuilder(total) + for (i in 0 until total) { + val item = list.getJSONObject(i) + val chapterId = item.getLong("chapter_id") + val scanlator = item.getStringOrNull("username") + val url = buildString { + if (isAuthorized) { + append(manga.url) + append("/v") + append(item.getInt("chapter_volume")) + append("/c") + append(item.getString("chapter_number")) + append("?ui=") + append(id) + } else { + append(manga.url) + append("/v") + append(item.getInt("chapter_volume")) + append("/c") + append(item.getString("chapter_number")) + } + } + val nameChapter = item.getStringOrNull("chapter_name") + val volume = item.getInt("chapter_volume") + val number = item.getString("chapter_number") + val fullNameChapter = "Том $volume. Глава $number" + chapters.add( + MangaChapter( + id = generateUid(chapterId), + url = url, + source = source, + number = total - i, + uploadDate = dateFormat.tryParse( + item.getString("chapter_created_at").substringBefore(" "), + ), + scanlator = scanlator, + branch = null, + name = if (nameChapter.isNullOrBlank()) fullNameChapter else "$fullNameChapter - $nameChapter", + ), + ) + } + chapters.reverse() + break@scripts + } + } + } + return manga.copy( + title = title?.getOrNull(0)?.text()?.takeUnless(String::isBlank) ?: manga.title, + altTitle = title?.getOrNull(1)?.text()?.substringBefore('/')?.trim(), + rating = root.selectFirst("div.media-stats-item__score") + ?.selectFirst("span") + ?.text()?.toFloatOrNull()?.div(5f) ?: manga.rating, + author = info?.getElementsMatchingOwnText("Автор")?.firstOrNull() + ?.nextElementSibling()?.text() ?: manga.author, + tags = info?.selectFirst("div.media-tags") + ?.select("a.media-tag-item")?.mapNotNullToSet { a -> + val href = a.attr("href") + if (href.contains("genres")) { + MangaTag( + title = a.text().toTitleCase(), + key = href.substringAfterLast('='), + source = source, + ) + } else null + } ?: manga.tags, + isNsfw = isNsfw(doc), + description = info?.selectFirst("div.media-description__text")?.html(), + chapters = chapters?.toList(), + ) + } - override suspend fun getPages(chapter: MangaChapter): List { - val fullUrl = chapter.url.toAbsoluteUrl(getDomain()) - val doc = context.httpGet(fullUrl).parseHtml() - if (doc.location().substringBefore('?').endsWith("/register")) { - throw AuthRequiredException(source) - } - val scripts = doc.head().select("script") - val pg = (doc.body().getElementById("pg")?.html() ?: doc.parseFailed("Element #pg not found")) - .substringAfter('=') - .substringBeforeLast(';') - val pages = JSONArray(pg) - for (script in scripts) { - val raw = script.html().trim() - if (raw.contains("window.__info")) { - val json = JSONObject( - raw.substringAfter("window.__info") - .substringAfter('=') - .substringBeforeLast(';'), - ) - val domain = json.getJSONObject("servers").run { - getStringOrNull("main") ?: getString( - json.getJSONObject("img").getString("server"), - ) - } - val url = json.getJSONObject("img").getString("url") - return pages.mapJSON { x -> - val pageUrl = "$domain/$url${x.getString("u")}" - MangaPage( - id = generateUid(pageUrl), - url = pageUrl, - preview = null, - referer = fullUrl, - source = source, - ) - } - } - } - throw ParseException("Script with info not found", fullUrl) - } + override suspend fun getPages(chapter: MangaChapter): List { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + if (doc.location().substringBefore('?').endsWith("/register")) { + throw AuthRequiredException(source) + } + val scripts = doc.head().select("script") + val pg = (doc.body().getElementById("pg")?.html() ?: doc.parseFailed("Element #pg not found")) + .substringAfter('=') + .substringBeforeLast(';') + val pages = JSONArray(pg) + for (script in scripts) { + val raw = script.html().trim() + if (raw.contains("window.__info")) { + val json = JSONObject( + raw.substringAfter("window.__info") + .substringAfter('=') + .substringBeforeLast(';'), + ) + val domain = json.getJSONObject("servers").run { + getStringOrNull("main") ?: getString( + json.getJSONObject("img").getString("server"), + ) + } + val url = json.getJSONObject("img").getString("url") + return pages.mapJSON { x -> + val pageUrl = "$domain/$url${x.getString("u")}" + MangaPage( + id = generateUid(pageUrl), + url = pageUrl, + preview = null, + referer = fullUrl, + source = source, + ) + } + } + } + throw ParseException("Script with info not found", fullUrl) + } - override suspend fun getTags(): Set { - val url = "https://${getDomain()}/manga-list" - val doc = context.httpGet(url).parseHtml() - val scripts = doc.body().select("script") - for (script in scripts) { - val raw = script.html().trim() - if (raw.startsWith("window.__DATA")) { - val json = JSONObject(raw.substringAfter('=').substringBeforeLast(';')) - val genres = json.getJSONObject("filters").getJSONArray("genres") - val result = ArraySet(genres.length()) - for (x in genres.JSONIterator()) { - result += MangaTag( - source = source, - key = x.getInt("id").toString(), - title = x.getString("name").toTitleCase(), - ) - } - return result - } - } - throw ParseException("Script with genres not found", url) - } + override suspend fun getTags(): Set { + val url = "https://${domain}/manga-list" + val doc = webClient.httpGet(url).parseHtml() + val scripts = doc.body().select("script") + for (script in scripts) { + val raw = script.html().trim() + if (raw.startsWith("window.__DATA")) { + val json = JSONObject(raw.substringAfter('=').substringBeforeLast(';')) + val genres = json.getJSONObject("filters").getJSONArray("genres") + val result = ArraySet(genres.length()) + for (x in genres.JSONIterator()) { + result += MangaTag( + source = source, + key = x.getInt("id").toString(), + title = x.getString("name").toTitleCase(), + ) + } + return result + } + } + throw ParseException("Script with genres not found", url) + } - override val isAuthorized: Boolean - get() { - return context.cookieJar.getCookies(getDomain()).any { - it.name.startsWith("remember_web_") - } - } + override val isAuthorized: Boolean + get() { + return context.cookieJar.getCookies(domain).any { + it.name.startsWith("remember_web_") + } + } - override suspend fun getUsername(): String { - val body = context.httpGet("https://${LibConst.LIB_SOCIAL_LINK}/messages").parseHtml().body() - if (body.baseUri().endsWith("/login")) { - throw AuthRequiredException(source) - } - return body.selectFirst(".profile-user__username")?.text() ?: body.parseFailed("Cannot find username") - } + override suspend fun getUsername(): String { + val body = webClient.httpGet("https://${LibConst.LIB_SOCIAL_LINK}/messages").parseHtml().body() + if (body.baseUri().endsWith("/login")) { + throw AuthRequiredException(source) + } + return body.selectFirst(".profile-user__username")?.text() ?: body.parseFailed("Cannot find username") + } - protected open fun isNsfw(doc: Document): Boolean { - val sidebar = doc.body().run { - selectFirst(".media-sidebar") ?: selectFirst(".media-info") - } ?: doc.parseFailed("Sidebar not found") - return sidebar.getElementsContainingOwnText("18+").isNotEmpty() - } + protected open fun isNsfw(doc: Document): Boolean { + val sidebar = doc.body().run { + selectFirst(".media-sidebar") ?: selectFirst(".media-info") + } ?: doc.parseFailed("Sidebar not found") + return sidebar.getElementsContainingOwnText("18+").isNotEmpty() + } - private fun getSortKey(sortOrder: SortOrder?) = when (sortOrder) { - SortOrder.RATING -> "desc&sort=rate" - SortOrder.ALPHABETICAL -> "asc&sort=name" - SortOrder.POPULARITY -> "desc&sort=views" - SortOrder.UPDATED -> "desc&sort=last_chapter_at" - SortOrder.NEWEST -> "desc&sort=created_at" - else -> "desc&sort=last_chapter_at" - } + private fun getSortKey(sortOrder: SortOrder?) = when (sortOrder) { + SortOrder.RATING -> "desc&sort=rate" + SortOrder.ALPHABETICAL -> "asc&sort=name" + SortOrder.POPULARITY -> "desc&sort=views" + SortOrder.UPDATED -> "desc&sort=last_chapter_at" + SortOrder.NEWEST -> "desc&sort=created_at" + else -> "desc&sort=last_chapter_at" + } - private suspend fun search(query: String): List { - val domain = getDomain() - val json = context.httpGet("https://$domain/search?type=manga&q=$query") - .parseJsonArray() - return json.mapJSON { jo -> - val slug = jo.getString("slug") - val url = "/$slug" - val covers = jo.getJSONObject("covers") - val title = jo.getString("rus_name").ifEmpty { jo.getString("name") } - Manga( - id = generateUid(url), - url = url, - publicUrl = "https://$domain/$slug", - title = title, - altTitle = jo.getString("name").takeUnless { it == title }, - author = null, - tags = emptySet(), - rating = jo.getString("rate_avg").toFloatOrNull()?.div(5f) ?: RATING_UNKNOWN, - state = null, - isNsfw = false, - source = source, - coverUrl = covers.getString("thumbnail").toAbsoluteUrl(domain), - largeCoverUrl = covers.getString("default").toAbsoluteUrl(domain), - ) - } - } + private suspend fun search(query: String): List { + val domain = domain + val json = webClient.httpGet("https://$domain/search?type=manga&q=$query") + .parseJsonArray() + return json.mapJSON { jo -> + val slug = jo.getString("slug") + val url = "/$slug" + val covers = jo.getJSONObject("covers") + val title = jo.getString("rus_name").ifEmpty { jo.getString("name") } + Manga( + id = generateUid(url), + url = url, + publicUrl = "https://$domain/$slug", + title = title, + altTitle = jo.getString("name").takeUnless { it == title }, + author = null, + tags = emptySet(), + rating = jo.getString("rate_avg").toFloatOrNull()?.div(5f) ?: RATING_UNKNOWN, + state = null, + isNsfw = false, + source = source, + coverUrl = covers.getString("thumbnail").toAbsoluteUrl(domain), + largeCoverUrl = covers.getString("default").toAbsoluteUrl(domain), + ) + } + } - @MangaSourceParser("MANGALIB", "MangaLib", "ru") - class Impl(context: MangaLoaderContext) : MangaLibParser(context, MangaSource.MANGALIB) + @MangaSourceParser("MANGALIB", "MangaLib", "ru") + class Impl(context: MangaLoaderContext) : MangaLibParser(context, MangaSource.MANGALIB) - object LibConst { + object LibConst { - val LIB_SOCIAL_LINK = "lib.social" + val LIB_SOCIAL_LINK = "lib.social" - } -} \ No newline at end of file + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/FaviconParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/FaviconParser.kt index fec7f706..cb8a6a3d 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/FaviconParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/FaviconParser.kt @@ -1,94 +1,92 @@ package org.koitharu.kotatsu.parsers.util -import okhttp3.Headers import org.jsoup.nodes.Element -import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.model.Favicon import org.koitharu.kotatsu.parsers.model.Favicons +import org.koitharu.kotatsu.parsers.network.WebClient import org.koitharu.kotatsu.parsers.util.json.mapJSON class FaviconParser( - private val context: MangaLoaderContext, - private val domain: String, - private val headers: Headers?, + private val webClient: WebClient, + private val domain: String, ) { - suspend fun parseFavicons(): Favicons { - val url = "https://$domain" - val doc = context.httpGet(url, headers).parseHtml() - val result = HashSet() - val manifestLink = doc.getElementsByAttributeValue("rel", "manifest").firstOrNull() - ?.attrAsAbsoluteUrlOrNull("href") - if (manifestLink != null) { - result += parseManifest(manifestLink) - } - val links = doc.getElementsByAttributeValueContaining("rel", "icon") - links.mapNotNullTo(result) { link -> - parseLink(link) - } - if (result.isEmpty()) { - result.add(createFallback()) - } - return Favicons(result, url) - } + suspend fun parseFavicons(): Favicons { + val url = "https://$domain" + val doc = webClient.httpGet(url).parseHtml() + val result = HashSet() + val manifestLink = doc.getElementsByAttributeValue("rel", "manifest").firstOrNull() + ?.attrAsAbsoluteUrlOrNull("href") + if (manifestLink != null) { + result += parseManifest(manifestLink) + } + val links = doc.getElementsByAttributeValueContaining("rel", "icon") + links.mapNotNullTo(result) { link -> + parseLink(link) + } + if (result.isEmpty()) { + result.add(createFallback()) + } + return Favicons(result, url) + } - private fun parseLink(link: Element): Favicon? { - val href = link.attrAsAbsoluteUrlOrNull("href") - if (href == null || href.endsWith('/')) { - return null - } - val sizes = link.attr("sizes") - return Favicon( - url = href, - size = parseSize(sizes), - rel = link.attrOrNull("rel"), - ) - } + private fun parseLink(link: Element): Favicon? { + val href = link.attrAsAbsoluteUrlOrNull("href") + if (href == null || href.endsWith('/')) { + return null + } + val sizes = link.attr("sizes") + return Favicon( + url = href, + size = parseSize(sizes), + rel = link.attrOrNull("rel"), + ) + } - private fun parseSize(sizes: String): Int { - if (sizes.isEmpty() || sizes == "any") { - return 0 - } - return sizes.substringBefore(' ') - .split('x', 'X', '*') - .firstNotNullOfOrNull { it.toIntOrNull() } - ?: 0 - } + private fun parseSize(sizes: String): Int { + if (sizes.isEmpty() || sizes == "any") { + return 0 + } + return sizes.substringBefore(' ') + .split('x', 'X', '*') + .firstNotNullOfOrNull { it.toIntOrNull() } + ?: 0 + } - private suspend fun parseManifest(url: String): List { - val json = context.httpGet(url, headers).parseJson() - val icons = json.getJSONArray("icons") - return icons.mapJSON { jo -> - Favicon( - url = jo.getString("src").resolveLink(), - size = parseSize(jo.getString("sizes")), - rel = null, - ) - } - } + private suspend fun parseManifest(url: String): List { + val json = webClient.httpGet(url).parseJson() + val icons = json.getJSONArray("icons") + return icons.mapJSON { jo -> + Favicon( + url = jo.getString("src").resolveLink(), + size = parseSize(jo.getString("sizes")), + rel = null, + ) + } + } - private fun createFallback(): Favicon { - val href = "https://$domain/favicon.ico" - return Favicon( - url = href, - size = 0, - rel = null, - ) - } + private fun createFallback(): Favicon { + val href = "https://$domain/favicon.ico" + return Favicon( + url = href, + size = 0, + rel = null, + ) + } - private fun String.resolveLink(): String { - return when { - startsWith("http:") || startsWith("https:") -> { - this - } + private fun String.resolveLink(): String { + return when { + startsWith("http:") || startsWith("https:") -> { + this + } - startsWith('/') -> { - "https://$domain$this" - } + startsWith('/') -> { + "https://$domain$this" + } - else -> { - "https://$domain/$this" - } - } - } -} \ No newline at end of file + else -> { + "https://$domain/$this" + } + } + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/MangaParserEnv.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/MangaParserEnv.kt new file mode 100644 index 00000000..1e20c9e0 --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/MangaParserEnv.kt @@ -0,0 +1,78 @@ +package org.koitharu.kotatsu.parsers.util + +import okhttp3.HttpUrl +import org.jsoup.nodes.Element +import org.koitharu.kotatsu.parsers.InternalParsersApi +import org.koitharu.kotatsu.parsers.MangaParser +import org.koitharu.kotatsu.parsers.exception.ParseException +import org.koitharu.kotatsu.parsers.model.Manga +import org.koitharu.kotatsu.parsers.model.MangaChapter +import org.koitharu.kotatsu.parsers.model.MangaPage +import org.koitharu.kotatsu.parsers.model.MangaTag + + +/** + * Create a unique id for [Manga]/[MangaChapter]/[MangaPage]. + * @param url must be relative url, without a domain + * @see [Manga.id] + * @see [MangaChapter.id] + * @see [MangaPage.id] + */ +@InternalParsersApi +fun MangaParser.generateUid(url: String): Long { + var h = 1125899906842597L + source.name.forEach { c -> + h = 31 * h + c.code + } + url.forEach { c -> + h = 31 * h + c.code + } + return h +} + +/** + * Create a unique id for [Manga]/[MangaChapter]/[MangaPage]. + * @param id an internal identifier + * @see [Manga.id] + * @see [MangaChapter.id] + * @see [MangaPage.id] + */ +@InternalParsersApi +fun MangaParser.generateUid(id: Long): Long { + var h = 1125899906842597L + source.name.forEach { c -> + h = 31 * h + c.code + } + h = 31 * h + id + return h +} + +@InternalParsersApi +fun Element.parseFailed(message: String? = null): Nothing { + throw ParseException(message, ownerDocument()?.location() ?: baseUri(), null) +} + +@InternalParsersApi +fun Set?.oneOrThrowIfMany(): MangaTag? { + return when { + isNullOrEmpty() -> null + size == 1 -> first() + else -> throw IllegalArgumentException("Multiple genres are not supported by this source") + } +} + +val MangaParser.domain: String + get() { + return config[configKeyDomain] + } + +fun MangaParser.getDomain(subdomain: String): String { + val domain = domain + return subdomain + "." + domain.removePrefix("www.") +} + +fun MangaParser.urlBuilder(): HttpUrl.Builder { + return HttpUrl.Builder() + .scheme("https") + .host(domain) +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/OkHttp.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/OkHttp.kt index 08c5d3c0..df363e70 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/OkHttp.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/OkHttp.kt @@ -4,16 +4,26 @@ package org.koitharu.kotatsu.parsers.util import kotlinx.coroutines.suspendCancellableCoroutine import okhttp3.Call +import okhttp3.Headers import okhttp3.Response suspend fun Call.await(): Response = suspendCancellableCoroutine { continuation -> - val callback = ContinuationCallCallback(this, continuation) - enqueue(callback) - continuation.invokeOnCancellation(callback) + val callback = ContinuationCallCallback(this, continuation) + enqueue(callback) + continuation.invokeOnCancellation(callback) } val Response.mimeType: String? - get() = body?.contentType()?.run { "$type/$subtype" } + get() = body?.contentType()?.run { "$type/$subtype" } val Response.contentDisposition: String? - get() = header("Content-Disposition") \ No newline at end of file + get() = header("Content-Disposition") + +fun Headers.Builder.mergeWith(other: Headers, replaceExisting: Boolean): Headers.Builder { + for ((name, value) in other) { + if (replaceExisting || this[name] == null) { + this[name] = value + } + } + return this +} diff --git a/src/test/kotlin/org/koitharu/kotatsu/parsers/AuthCheckExtension.kt b/src/test/kotlin/org/koitharu/kotatsu/parsers/AuthCheckExtension.kt index 0dfbe83f..05ab3c6f 100644 --- a/src/test/kotlin/org/koitharu/kotatsu/parsers/AuthCheckExtension.kt +++ b/src/test/kotlin/org/koitharu/kotatsu/parsers/AuthCheckExtension.kt @@ -8,27 +8,27 @@ import org.koitharu.kotatsu.parsers.util.runCatchingCancellable class AuthCheckExtension : BeforeAllCallback { - private val loaderContext: MangaLoaderContext = MangaLoaderContextMock() + private val loaderContext: MangaLoaderContext = MangaLoaderContextMock - override fun beforeAll(context: ExtensionContext) { - for (source in MangaSource.values()) { - if (source == MangaSource.LOCAL || source == MangaSource.DUMMY) { - continue - } - val parser = source.newParser(loaderContext) - if (parser is MangaParserAuthProvider) { - checkAuthorization(source, parser) - } - } - } + override fun beforeAll(context: ExtensionContext) { + for (source in MangaSource.values()) { + if (source == MangaSource.LOCAL || source == MangaSource.DUMMY) { + continue + } + val parser = source.newParser(loaderContext) + if (parser is MangaParserAuthProvider) { + checkAuthorization(source, parser) + } + } + } - private fun checkAuthorization(source: MangaSource, parser: MangaParserAuthProvider) = runTest { - runCatchingCancellable { - parser.getUsername() - }.onSuccess { username -> - println("Signed in to ${source.name} as $username") - }.onFailure { error -> - System.err.println("Auth failed for ${source.name}: ${error.javaClass.name}(${error.message})") - } - } -} \ No newline at end of file + private fun checkAuthorization(source: MangaSource, parser: MangaParserAuthProvider) = runTest { + runCatchingCancellable { + parser.getUsername() + }.onSuccess { username -> + println("Signed in to ${source.name} as $username") + }.onFailure { error -> + System.err.println("Auth failed for ${source.name}: ${error.javaClass.name}(${error.message})") + } + } +} diff --git a/src/test/kotlin/org/koitharu/kotatsu/parsers/CloudFlareInterceptor.kt b/src/test/kotlin/org/koitharu/kotatsu/parsers/CloudFlareInterceptor.kt index 957eb889..8878779d 100644 --- a/src/test/kotlin/org/koitharu/kotatsu/parsers/CloudFlareInterceptor.kt +++ b/src/test/kotlin/org/koitharu/kotatsu/parsers/CloudFlareInterceptor.kt @@ -4,26 +4,25 @@ import okhttp3.Interceptor import okhttp3.Response import okhttp3.internal.closeQuietly import org.koitharu.kotatsu.parsers.exception.CloudFlareProtectedException -import java.net.HttpURLConnection.HTTP_FORBIDDEN -import java.net.HttpURLConnection.HTTP_UNAVAILABLE +import java.net.HttpURLConnection private const val HEADER_SERVER = "Server" private const val SERVER_CLOUDFLARE = "cloudflare" -class CloudFlareInterceptor : Interceptor { +internal class CloudFlareInterceptor : Interceptor { - override fun intercept(chain: Interceptor.Chain): Response { - val request = chain.request() - val response = chain.proceed(request) - if (response.code == HTTP_FORBIDDEN || response.code == HTTP_UNAVAILABLE) { - if (response.header(HEADER_SERVER)?.startsWith(SERVER_CLOUDFLARE) == true) { - response.closeQuietly() - throw CloudFlareProtectedException( - url = response.request.url.toString(), - headers = request.headers, - ) - } - } - return response - } -} \ No newline at end of file + override fun intercept(chain: Interceptor.Chain): Response { + val request = chain.request() + val response = chain.proceed(request) + if (response.code == HttpURLConnection.HTTP_FORBIDDEN || response.code == HttpURLConnection.HTTP_UNAVAILABLE) { + if (response.header(HEADER_SERVER)?.startsWith(SERVER_CLOUDFLARE) == true) { + response.closeQuietly() + throw CloudFlareProtectedException( + url = response.request.url.toString(), + headers = request.headers, + ) + } + } + return response + } +} diff --git a/src/test/kotlin/org/koitharu/kotatsu/parsers/CommonHeadersInterceptor.kt b/src/test/kotlin/org/koitharu/kotatsu/parsers/CommonHeadersInterceptor.kt new file mode 100644 index 00000000..73236481 --- /dev/null +++ b/src/test/kotlin/org/koitharu/kotatsu/parsers/CommonHeadersInterceptor.kt @@ -0,0 +1,47 @@ +package org.koitharu.kotatsu.parsers + +import okhttp3.Interceptor +import okhttp3.Request +import okhttp3.Response +import org.koitharu.kotatsu.parsers.model.MangaSource +import org.koitharu.kotatsu.parsers.util.domain +import org.koitharu.kotatsu.parsers.util.mergeWith + +private const val HEADER_USER_AGENT = "User-Agent" +private const val HEADER_REFERER = "Referer" + +internal class CommonHeadersInterceptor( + private val userAgent: String, +) : Interceptor { + + override fun intercept(chain: Interceptor.Chain): Response { + val request = chain.request() + val source = request.tag(MangaSource::class.java) + val parser = source?.newParser(MangaLoaderContextMock) + val sourceHeaders = parser?.headers + val headersBuilder = request.headers.newBuilder() + if (sourceHeaders != null) { + headersBuilder.mergeWith(sourceHeaders, replaceExisting = false) + } + if (headersBuilder[HEADER_USER_AGENT] == null) { + headersBuilder[HEADER_USER_AGENT] = userAgent + } + if (headersBuilder[HEADER_REFERER] == null && parser != null) { + headersBuilder[HEADER_REFERER] = "https://${parser.domain}/" + } + val newRequest = request.newBuilder().headers(headersBuilder.build()).build() + return if (parser is Interceptor) { + parser.intercept(ProxyChain(chain, newRequest)) + } else { + return chain.proceed(newRequest) + } + } + + private class ProxyChain( + private val delegate: Interceptor.Chain, + private val request: Request, + ) : Interceptor.Chain by delegate { + + override fun request(): Request = request + } +} diff --git a/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaLoaderContextMock.kt b/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaLoaderContextMock.kt index 95ebb59c..2c80cb0f 100644 --- a/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaLoaderContextMock.kt +++ b/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaLoaderContextMock.kt @@ -10,59 +10,59 @@ import org.koitharu.kotatsu.parsers.model.MangaSource import org.koitharu.kotatsu.parsers.util.await import java.util.concurrent.TimeUnit -internal class MangaLoaderContextMock : MangaLoaderContext() { +internal object MangaLoaderContextMock : MangaLoaderContext() { - private val userAgent = "Kotatsu/%s (Android %s; %s; %s %s; %s)".format( - /*BuildConfig.VERSION_NAME*/ "3.0", - /*Build.VERSION.RELEASE*/ "r", - /*Build.MODEL*/ "", - /*Build.BRAND*/ "", - /*Build.DEVICE*/ "", - /*Locale.getDefault().language*/ "en", - ) + private val userAgent = "Kotatsu/%s (Android %s; %s; %s %s; %s)".format( + /*BuildConfig.VERSION_NAME*/ "4.3", + /*Build.VERSION.RELEASE*/ "r", + /*Build.MODEL*/ "", + /*Build.BRAND*/ "", + /*Build.DEVICE*/ "", + /*Locale.getDefault().language*/ "en", + ) - override val cookieJar = InMemoryCookieJar() + override val cookieJar = InMemoryCookieJar() - override val httpClient: OkHttpClient = OkHttpClient.Builder() - .cookieJar(cookieJar) - .addInterceptor(UserAgentInterceptor(userAgent)) - .addInterceptor(CloudFlareInterceptor()) - .connectTimeout(20, TimeUnit.SECONDS) - .readTimeout(60, TimeUnit.SECONDS) - .writeTimeout(20, TimeUnit.SECONDS) - .build() + override val httpClient: OkHttpClient = OkHttpClient.Builder() + .cookieJar(cookieJar) + .addInterceptor(CommonHeadersInterceptor(userAgent)) + .addInterceptor(CloudFlareInterceptor()) + .connectTimeout(20, TimeUnit.SECONDS) + .readTimeout(60, TimeUnit.SECONDS) + .writeTimeout(20, TimeUnit.SECONDS) + .build() - init { - loadTestCookies() - } + init { + loadTestCookies() + } - override suspend fun evaluateJs(script: String): String? { - return QuackContext.create().use { - it.evaluate(script)?.toString() - } - } + override suspend fun evaluateJs(script: String): String? { + return QuackContext.create().use { + it.evaluate(script)?.toString() + } + } - override fun getConfig(source: MangaSource): MangaSourceConfig { - return SourceConfigMock() - } + override fun getConfig(source: MangaSource): MangaSourceConfig { + return SourceConfigMock() + } - suspend fun doRequest(url: String, referer: String? = null, extraHeaders: Headers? = null): Response { - val request = Request.Builder() - .get() - .url(url) - if (extraHeaders != null) { - request.headers(extraHeaders) - } - if (referer != null) { - request.header("Referer", referer) - } - return httpClient.newCall(request.build()).await() - } + suspend fun doRequest(url: String, referer: String? = null, extraHeaders: Headers? = null): Response { + val request = Request.Builder() + .get() + .url(url) + if (extraHeaders != null) { + request.headers(extraHeaders) + } + if (referer != null) { + request.header("Referer", referer) + } + return httpClient.newCall(request.build()).await() + } - private fun loadTestCookies() { - // https://addons.mozilla.org/ru/firefox/addon/cookies-txt/ - javaClass.getResourceAsStream("/cookies.txt")?.use { - cookieJar.loadFromStream(it) - } ?: println("No cookies loaded!") - } -} \ No newline at end of file + private fun loadTestCookies() { + // https://addons.mozilla.org/ru/firefox/addon/cookies-txt/ + javaClass.getResourceAsStream("/cookies.txt")?.use { + cookieJar.loadFromStream(it) + } ?: println("No cookies loaded!") + } +} diff --git a/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaParserTest.kt b/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaParserTest.kt index 4abd1506..ab450369 100644 --- a/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaParserTest.kt +++ b/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaParserTest.kt @@ -8,6 +8,7 @@ import org.junit.jupiter.params.ParameterizedTest import org.koitharu.kotatsu.parsers.model.Manga import org.koitharu.kotatsu.parsers.model.MangaSource import org.koitharu.kotatsu.parsers.model.SortOrder +import org.koitharu.kotatsu.parsers.util.domain import org.koitharu.kotatsu.parsers.util.medianOrNull import org.koitharu.kotatsu.parsers.util.mimeType import org.koitharu.kotatsu.test_util.isDistinct @@ -19,184 +20,184 @@ import org.koitharu.kotatsu.test_util.maxDuplicates @ExtendWith(AuthCheckExtension::class) internal class MangaParserTest { - private val context = MangaLoaderContextMock() - - @ParameterizedTest(name = "{index}|list|{0}") - @MangaSources - fun list(source: MangaSource) = runTest { - val parser = source.newParser(context) - val list = parser.getList(20, sortOrder = SortOrder.POPULARITY, tags = null) - checkMangaList(list, "list") - assert(list.all { it.source == source }) - } - - @ParameterizedTest(name = "{index}|pagination|{0}") - @MangaSources - fun pagination(source: MangaSource) = runTest { - val parser = source.newParser(context) - val page1 = parser.getList(0, sortOrder = null, tags = null) - val page2 = parser.getList(page1.size, sortOrder = null, tags = null) - assert(page1.isNotEmpty()) { "Page 1 is empty" } - assert(page2.isNotEmpty()) { "Page 2 is empty" } - val intersection = page1.intersect(page2.toSet()) - assert(intersection.isEmpty()) { - "Pages are intersected by " + intersection.size - } - } - - @ParameterizedTest(name = "{index}|search|{0}") - @MangaSources - fun search(source: MangaSource) = runTest { - val parser = source.newParser(context) - val subject = parser.getList(20, sortOrder = SortOrder.POPULARITY, tags = null).minByOrNull { - it.title.length - } ?: error("No manga found") - val query = subject.title - check(query.isNotBlank()) { "Manga title '$query' is blank" } - val list = parser.getList(0, query) - assert(list.isNotEmpty()) { "Empty search results by \"$query\"" } - assert(list.singleOrNull { it.url == subject.url && it.id == subject.id } != null) { - "Single subject '${subject.title} (${subject.publicUrl})' not found in search results" - } - checkMangaList(list, "search('$query')") - assert(list.all { it.source == source }) - } - - @ParameterizedTest(name = "{index}|tags|{0}") - @MangaSources - fun tags(source: MangaSource) = runTest { - val parser = source.newParser(context) - val tags = parser.getTags() - assert(tags.isNotEmpty()) { "No tags found" } - val keys = tags.map { it.key } - assert(keys.isDistinct()) - assert("" !in keys) - val titles = tags.map { it.title } + private val context = MangaLoaderContextMock + + @ParameterizedTest(name = "{index}|list|{0}") + @MangaSources + fun list(source: MangaSource) = runTest { + val parser = source.newParser(context) + val list = parser.getList(20, sortOrder = SortOrder.POPULARITY, tags = null) + checkMangaList(list, "list") + assert(list.all { it.source == source }) + } + + @ParameterizedTest(name = "{index}|pagination|{0}") + @MangaSources + fun pagination(source: MangaSource) = runTest { + val parser = source.newParser(context) + val page1 = parser.getList(0, sortOrder = null, tags = null) + val page2 = parser.getList(page1.size, sortOrder = null, tags = null) + assert(page1.isNotEmpty()) { "Page 1 is empty" } + assert(page2.isNotEmpty()) { "Page 2 is empty" } + val intersection = page1.intersect(page2.toSet()) + assert(intersection.isEmpty()) { + "Pages are intersected by " + intersection.size + } + } + + @ParameterizedTest(name = "{index}|search|{0}") + @MangaSources + fun search(source: MangaSource) = runTest { + val parser = source.newParser(context) + val subject = parser.getList(20, sortOrder = SortOrder.POPULARITY, tags = null).minByOrNull { + it.title.length + } ?: error("No manga found") + val query = subject.title + check(query.isNotBlank()) { "Manga title '$query' is blank" } + val list = parser.getList(0, query) + assert(list.isNotEmpty()) { "Empty search results by \"$query\"" } + assert(list.singleOrNull { it.url == subject.url && it.id == subject.id } != null) { + "Single subject '${subject.title} (${subject.publicUrl})' not found in search results" + } + checkMangaList(list, "search('$query')") + assert(list.all { it.source == source }) + } + + @ParameterizedTest(name = "{index}|tags|{0}") + @MangaSources + fun tags(source: MangaSource) = runTest { + val parser = source.newParser(context) + val tags = parser.getTags() + assert(tags.isNotEmpty()) { "No tags found" } + val keys = tags.map { it.key } + assert(keys.isDistinct()) + assert("" !in keys) + val titles = tags.map { it.title } // assert(titles.isDistinct()) - assert("" !in titles) - assert(tags.all { it.source == source }) - - val tag = tags.last() - val list = parser.getList(offset = 0, tags = setOf(tag), sortOrder = null) - checkMangaList(list, "${tag.title} (${tag.key})") - assert(list.all { it.source == source }) - } - - @ParameterizedTest(name = "{index}|details|{0}") - @MangaSources - fun details(source: MangaSource) = runTest { - val parser = source.newParser(context) - val list = parser.getList(20, sortOrder = SortOrder.POPULARITY, tags = null) - val manga = list[3] - parser.getDetails(manga).apply { - assert(!chapters.isNullOrEmpty()) { "Chapters are null or empty" } - assert(publicUrl.isUrlAbsolute()) { "Manga public url is not absolute: '$publicUrl'" } - assert(description != null) { "Detailed description is null: '$publicUrl'" } - assert(title.startsWith(manga.title)) { - "Titles are mismatch: '$title' and '${manga.title}' for $publicUrl" - } - assert(this.source == source) - val c = checkNotNull(chapters) - assert(c.isDistinctBy { it.id }) { - "Chapters are not distinct by id: ${c.maxDuplicates { it.id }} for $publicUrl" - } - assert(c.isDistinctBy { it.number to it.branch }) { - "Chapters are not distinct by number: ${c.maxDuplicates { it.number to it.branch }} for $publicUrl" - } - assert(c.all { it.source == source }) - checkImageRequest(coverUrl, publicUrl) - largeCoverUrl?.let { - checkImageRequest(it, publicUrl) - } - } - } - - @ParameterizedTest(name = "{index}|pages|{0}") - @MangaSources - fun pages(source: MangaSource) = runTest { - val parser = source.newParser(context) - val list = parser.getList(20, sortOrder = SortOrder.POPULARITY, tags = null) - val manga = list.first() - val chapter = parser.getDetails(manga).chapters?.firstOrNull() ?: error("Chapter is null") - val pages = parser.getPages(chapter) - - assert(pages.isNotEmpty()) - assert(pages.isDistinctBy { it.id }) - assert(pages.all { it.source == source }) - - val page = pages.medianOrNull() ?: error("No page") - val pageUrl = parser.getPageUrl(page) - assert(pageUrl.isNotEmpty()) - assert(pageUrl.isUrlAbsolute()) - checkImageRequest(pageUrl, page.referer) - } - - @ParameterizedTest(name = "{index}|favicon|{0}") - @MangaSources - fun favicon(source: MangaSource) = runTest { - val parser = source.newParser(context) - val favicons = parser.getFavicons() - val types = setOf("png", "svg", "ico", "gif", "jpg", "jpeg") - assert(favicons.isNotEmpty()) - favicons.forEach { - assert(it.url.isUrlAbsolute()) { "Favicon url is not absolute: ${it.url}" } - assert(it.type in types) { "Unknown icon type: ${it.type}" } - } - val favicon = favicons.find(24) - checkNotNull(favicon) - checkImageRequest(favicon.url, favicons.referer) - } - - @ParameterizedTest(name = "{index}|domain|{0}") - @MangaSources - fun domain(source: MangaSource) = runTest { - val parser = source.newParser(context) - val defaultDomain = parser.getDomain() - val url = HttpUrl.Builder() - .host(defaultDomain) - .scheme("https") - .toString() - val response = context.doRequest(url, extraHeaders = parser.headers) - val realUrl = response.request.url - val realDomain = realUrl.topPrivateDomain() - val realHost = realUrl.host - assert(defaultDomain == realHost || defaultDomain == realDomain) { - "Domain mismatch:\nRequired:\t\t\t$defaultDomain\nActual:\t\t\t$realDomain\nHost:\t\t\t$realHost" - } - } - - @ParameterizedTest(name = "{index}|authorization|{0}") - @MangaSources - @Disabled - fun authorization(source: MangaSource) = runTest { - val parser = source.newParser(context) - if (parser is MangaParserAuthProvider) { - val username = parser.getUsername() - assert(username.isNotBlank()) { "Username is blank" } - println("Signed in to ${source.name} as $username") - } - } - - private suspend fun checkMangaList(list: List, cause: String) { - assert(list.isNotEmpty()) { "Manga list for '$cause' is empty" } - assert(list.isDistinctBy { it.id }) { "Manga list for '$cause' contains duplicated ids" } - for (item in list) { - assert(item.url.isNotEmpty()) { "Url is empty" } - assert(!item.url.isUrlAbsolute()) { "Url looks like absolute: ${item.url}" } - assert(item.coverUrl.isUrlAbsolute()) { "Cover url is not absolute: ${item.coverUrl}" } - assert(item.title.isNotEmpty()) { "Title for ${item.publicUrl} is empty" } - assert(item.publicUrl.isUrlAbsolute()) - } - val testItem = list.random() - checkImageRequest(testItem.coverUrl, testItem.publicUrl) - } - - private suspend fun checkImageRequest(url: String, referer: String?) { - context.doRequest(url, referer).use { - assert(it.isSuccessful) { "Request failed: ${it.code}(${it.message}): $url" } - assert(it.mimeType?.startsWith("image/") == true) { - "Wrong response mime type: ${it.mimeType}" - } - } - } -} \ No newline at end of file + assert("" !in titles) + assert(tags.all { it.source == source }) + + val tag = tags.last() + val list = parser.getList(offset = 0, tags = setOf(tag), sortOrder = null) + checkMangaList(list, "${tag.title} (${tag.key})") + assert(list.all { it.source == source }) + } + + @ParameterizedTest(name = "{index}|details|{0}") + @MangaSources + fun details(source: MangaSource) = runTest { + val parser = source.newParser(context) + val list = parser.getList(20, sortOrder = SortOrder.POPULARITY, tags = null) + val manga = list[3] + parser.getDetails(manga).apply { + assert(!chapters.isNullOrEmpty()) { "Chapters are null or empty" } + assert(publicUrl.isUrlAbsolute()) { "Manga public url is not absolute: '$publicUrl'" } + assert(description != null) { "Detailed description is null: '$publicUrl'" } + assert(title.startsWith(manga.title)) { + "Titles are mismatch: '$title' and '${manga.title}' for $publicUrl" + } + assert(this.source == source) + val c = checkNotNull(chapters) + assert(c.isDistinctBy { it.id }) { + "Chapters are not distinct by id: ${c.maxDuplicates { it.id }} for $publicUrl" + } + assert(c.isDistinctBy { it.number to it.branch }) { + "Chapters are not distinct by number: ${c.maxDuplicates { it.number to it.branch }} for $publicUrl" + } + assert(c.all { it.source == source }) + checkImageRequest(coverUrl, publicUrl) + largeCoverUrl?.let { + checkImageRequest(it, publicUrl) + } + } + } + + @ParameterizedTest(name = "{index}|pages|{0}") + @MangaSources + fun pages(source: MangaSource) = runTest { + val parser = source.newParser(context) + val list = parser.getList(20, sortOrder = SortOrder.POPULARITY, tags = null) + val manga = list.first() + val chapter = parser.getDetails(manga).chapters?.firstOrNull() ?: error("Chapter is null") + val pages = parser.getPages(chapter) + + assert(pages.isNotEmpty()) + assert(pages.isDistinctBy { it.id }) + assert(pages.all { it.source == source }) + + val page = pages.medianOrNull() ?: error("No page") + val pageUrl = parser.getPageUrl(page) + assert(pageUrl.isNotEmpty()) + assert(pageUrl.isUrlAbsolute()) + checkImageRequest(pageUrl, page.referer) + } + + @ParameterizedTest(name = "{index}|favicon|{0}") + @MangaSources + fun favicon(source: MangaSource) = runTest { + val parser = source.newParser(context) + val favicons = parser.getFavicons() + val types = setOf("png", "svg", "ico", "gif", "jpg", "jpeg") + assert(favicons.isNotEmpty()) + favicons.forEach { + assert(it.url.isUrlAbsolute()) { "Favicon url is not absolute: ${it.url}" } + assert(it.type in types) { "Unknown icon type: ${it.type}" } + } + val favicon = favicons.find(24) + checkNotNull(favicon) + checkImageRequest(favicon.url, favicons.referer) + } + + @ParameterizedTest(name = "{index}|domain|{0}") + @MangaSources + fun domain(source: MangaSource) = runTest { + val parser = source.newParser(context) + val defaultDomain = parser.domain + val url = HttpUrl.Builder() + .host(defaultDomain) + .scheme("https") + .toString() + val response = context.doRequest(url, extraHeaders = parser.headers) + val realUrl = response.request.url + val realDomain = realUrl.topPrivateDomain() + val realHost = realUrl.host + assert(defaultDomain == realHost || defaultDomain == realDomain) { + "Domain mismatch:\nRequired:\t\t\t$defaultDomain\nActual:\t\t\t$realDomain\nHost:\t\t\t$realHost" + } + } + + @ParameterizedTest(name = "{index}|authorization|{0}") + @MangaSources + @Disabled + fun authorization(source: MangaSource) = runTest { + val parser = source.newParser(context) + if (parser is MangaParserAuthProvider) { + val username = parser.getUsername() + assert(username.isNotBlank()) { "Username is blank" } + println("Signed in to ${source.name} as $username") + } + } + + private suspend fun checkMangaList(list: List, cause: String) { + assert(list.isNotEmpty()) { "Manga list for '$cause' is empty" } + assert(list.isDistinctBy { it.id }) { "Manga list for '$cause' contains duplicated ids" } + for (item in list) { + assert(item.url.isNotEmpty()) { "Url is empty" } + assert(!item.url.isUrlAbsolute()) { "Url looks like absolute: ${item.url}" } + assert(item.coverUrl.isUrlAbsolute()) { "Cover url is not absolute: ${item.coverUrl}" } + assert(item.title.isNotEmpty()) { "Title for ${item.publicUrl} is empty" } + assert(item.publicUrl.isUrlAbsolute()) + } + val testItem = list.random() + checkImageRequest(testItem.coverUrl, testItem.publicUrl) + } + + private suspend fun checkImageRequest(url: String, referer: String?) { + context.doRequest(url, referer).use { + assert(it.isSuccessful) { "Request failed: ${it.code}(${it.message}): $url" } + assert(it.mimeType?.startsWith("image/") == true) { + "Wrong response mime type: ${it.mimeType}" + } + } + } +} diff --git a/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaSources.kt b/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaSources.kt index 5795da96..ef660faa 100644 --- a/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaSources.kt +++ b/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaSources.kt @@ -4,4 +4,4 @@ import org.junit.jupiter.params.provider.EnumSource import org.koitharu.kotatsu.parsers.model.MangaSource @EnumSource(MangaSource::class, names = ["LOCAL", "DUMMY"], mode = EnumSource.Mode.EXCLUDE) -internal annotation class MangaSources \ No newline at end of file +internal annotation class MangaSources diff --git a/src/test/kotlin/org/koitharu/kotatsu/parsers/UserAgentInterceptor.kt b/src/test/kotlin/org/koitharu/kotatsu/parsers/UserAgentInterceptor.kt deleted file mode 100644 index 10320160..00000000 --- a/src/test/kotlin/org/koitharu/kotatsu/parsers/UserAgentInterceptor.kt +++ /dev/null @@ -1,20 +0,0 @@ -package org.koitharu.kotatsu.parsers - -import okhttp3.Interceptor -import okhttp3.Response - -private const val HEADER_USER_AGENT = "User-Agent" -internal class UserAgentInterceptor( - private val userAgent: String, -) : Interceptor { - - override fun intercept(chain: Interceptor.Chain): Response { - val request = chain.request() - val newRequest = if (request.header(HEADER_USER_AGENT) == null) { - request.newBuilder().header(HEADER_USER_AGENT, userAgent).build() - } else { - request - } - return chain.proceed(newRequest) - } -} \ No newline at end of file From 05d705ac0384a3fb08579e4bc91c728469a05b58 Mon Sep 17 00:00:00 2001 From: Koitharu Date: Wed, 8 Feb 2023 11:08:57 +0200 Subject: [PATCH 064/121] Remove deprecated "getFavicon" method --- CONTRIBUTING.md | 6 ++--- .../koitharu/kotatsu/parsers/MangaParser.kt | 9 -------- .../kotatsu/parsers/site/AnibelParser.kt | 4 ---- .../kotatsu/parsers/site/BatoToParser.kt | 2 -- .../kotatsu/parsers/site/CloneMangaParser.kt | 4 ---- .../kotatsu/parsers/site/NudeMoonParser.kt | 4 ---- .../parsers/site/grouple/MintMangaParser.kt | 3 --- .../parsers/site/grouple/ReadmangaParser.kt | 3 --- .../parsers/site/grouple/SelfMangaParser.kt | 3 --- .../parsers/site/madara/Madara5Parser.kt | 1 - .../parsers/site/madara/MadaraParser.kt | 23 +------------------ .../parsers/site/madara/MangalinkParser.kt | 2 -- .../site/madara/MangasOriginesParser.kt | 5 ---- .../parsers/site/madara/PrismaScansParser.kt | 5 ---- .../parsers/site/madara/ReaperScansParser.kt | 2 -- .../parsers/site/madara/TatakaeScansParser.kt | 5 ---- 16 files changed, 4 insertions(+), 77 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3b38769b..ab56094d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -59,13 +59,13 @@ All functions in `MangaParser` class are documented. Pay attention to some pecul - `sortOrders` set should not be empty. If your source is not support sorting, specify one most relevance value. - If you cannot obtain direct links to pages images inside `getPages` method, it is ok to use an intermediate url as `Page.url` and fetch a direct link at `getPageUrl` function. -- `getFaviconUrl` function is deprecated, but Kotatsu application before v4 still use it, so it is recommended to - override it too. - You can use _asserts_ to check some optional fields. For example. `Manga.author` field is not required, but if your source provide such information, add `assert(it != null)`. This will not have any effect on production but help to find issues during unit testing. - If your source website (or it's api) uses pages for pagination instead of offset you should extend `PagedMangaParser` instead of `MangaParser`. +- Your parser may also implement the `Interceptor` interface for additional manipulation of all network requests and/or + responses, including image loading. ## Development process @@ -87,4 +87,4 @@ It is recommended to run unit tests before submitting a PR. ## Help -If you need a help or have some questions, ask a community in our [Discord server](https://discord.gg/NNJ5RgVBC5). \ No newline at end of file +If you need a help or have some questions, ask a community in our [Discord server](https://discord.gg/NNJ5RgVBC5). diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaParser.kt index 435976f1..a99d6d64 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaParser.kt @@ -112,15 +112,6 @@ abstract class MangaParser @InternalParsersApi constructor( */ abstract suspend fun getTags(): Set - /** - * Returns direct link to the website favicon - */ - @Deprecated( - message = "Use parseFavicons() to get multiple favicons with different size", - replaceWith = ReplaceWith("parseFavicons()"), - ) - open fun getFaviconUrl() = "https://${domain}/favicon.ico" - /** * Parse favicons from the main page of the source`s website */ diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/AnibelParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/AnibelParser.kt index a01f96be..dddfc47d 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/AnibelParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/AnibelParser.kt @@ -26,10 +26,6 @@ internal class AnibelParser(context: MangaLoaderContext) : MangaParser(context, SortOrder.NEWEST, ) - override fun getFaviconUrl(): String { - return "https://cdn.${domain}/favicons/favicon.png" - } - override suspend fun getList( offset: Int, query: String?, diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BatoToParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BatoToParser.kt index f162a680..afbcfbb2 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BatoToParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BatoToParser.kt @@ -158,8 +158,6 @@ internal class BatoToParser(context: MangaLoaderContext) : PagedMangaParser( throw ParseException("Cannot find gernes list", scripts[0].baseUri()) } - override fun getFaviconUrl(): String = "https://styles.amarkcdn.com/img/batoto/favicon.ico?v0" - private suspend fun search(page: Int, query: String): List { val url = buildString { append("https://") diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/CloneMangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/CloneMangaParser.kt index 9bdea84f..889aa81c 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/CloneMangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/CloneMangaParser.kt @@ -21,10 +21,6 @@ internal class CloneMangaParser(context: MangaLoaderContext) : PagedMangaParser( override val configKeyDomain = ConfigKey.Domain("manga.clone-army.org", null) - override fun getFaviconUrl(): String { - return "https://pbs.twimg.com/profile_images/458758466346029056/Ys93EANp_400x400.png" - } - override suspend fun getListPage( page: Int, query: String?, diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NudeMoonParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NudeMoonParser.kt index 6341dcd7..823cd37b 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NudeMoonParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NudeMoonParser.kt @@ -205,10 +205,6 @@ internal class NudeMoonParser( } } - override fun getFaviconUrl(): String { - return "https://${domain}/favicon.jpg" - } - private fun getSortKey(sortOrder: SortOrder) = when (sortOrder) { SortOrder.POPULARITY -> "views" diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/MintMangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/MintMangaParser.kt index ddbbdd3b..3cc040af 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/MintMangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/MintMangaParser.kt @@ -15,7 +15,4 @@ internal class MintMangaParser( arrayOf("mintmanga.live", "mintmanga.com"), ) - override fun getFaviconUrl(): String { - return "https://resmm.rmr.rocks/static/apple-touch-icon-8fff291039c140493adb0c7ba81065ad.png" - } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/ReadmangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/ReadmangaParser.kt index f4fe90d4..9ac27e9a 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/ReadmangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/ReadmangaParser.kt @@ -15,7 +15,4 @@ internal class ReadmangaParser( arrayOf("readmanga.io", "readmanga.live", "readmanga.me"), ) - override fun getFaviconUrl(): String { - return "https://resrm.rmr.rocks/static/apple-touch-icon-3162037c9df9f28dca0f9a4092cb0f65.png" - } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/SelfMangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/SelfMangaParser.kt index 91295808..44fc703b 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/SelfMangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/SelfMangaParser.kt @@ -12,7 +12,4 @@ internal class SelfMangaParser( override val configKeyDomain = ConfigKey.Domain("selfmanga.live", null) - override fun getFaviconUrl(): String { - return "https://ressm.rmr.rocks/static/apple-touch-icon-a769ea533d811b73ac3eedde658bb1d3.png" - } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Madara5Parser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Madara5Parser.kt index ae2e4110..3cc26196 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Madara5Parser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Madara5Parser.kt @@ -182,6 +182,5 @@ abstract class Madara5Parser @InternalParsersApi constructor( @MangaSourceParser("MANGAOWLS", "BeautyManga", "en") class BeautyManga(context: MangaLoaderContext) : Madara5Parser(context, MangaSource.MANGAOWLS, "beautymanga.com") { - override fun getFaviconUrl() = "http://${domain}/frontend/images/favico.png" } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt index ad7a6b59..5e8b95e9 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt @@ -316,7 +316,7 @@ internal abstract class MadaraParser( override val datePattern = "dd MMMM HH:mm" } - @MangaSourceParser("HachiManga", "HACHIMANGA", "ja") + @MangaSourceParser("HACHIMANGA", "HachiManga", "ja") class HachiManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.HACHIMANGA, "hachimanga.com") { @@ -340,17 +340,11 @@ internal abstract class MadaraParser( @MangaSourceParser("AQUAMANGA", "AquaManga", "en") class AquaManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.AQUAMANGA, "aquamanga.com") { - override fun getFaviconUrl(): String { - return "https://$domain/wp-content/uploads/2021/03/cropped-cropped-favicon-1-32x32.png" - } } @MangaSourceParser("MANGALEK", "MangaLek", "ar") class MangaLek(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGALEK, "mangalek.com") { - override fun getFaviconUrl(): String { - return "https://$domain/wp-content/logo192.png" - } } @MangaSourceParser("HARIMANGA", "HariManga", "en") @@ -384,9 +378,6 @@ internal abstract class MadaraParser( @MangaSourceParser("MANHWAKOOL", "Manhwa Kool", "en") class ManhwaKool(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANHWAKOOL, "manhwakool.com") { override val datePattern: String = "MM/dd" - override fun getFaviconUrl(): String { - return "https://$domain/wp-content/uploads/2021/10/cropped-logo-kool-32x32.jpeg" - } } @MangaSourceParser("TOPMANHUA", "Top Manhua", "en") @@ -403,9 +394,6 @@ internal abstract class MadaraParser( override val isNsfwSource = true - override fun getFaviconUrl(): String { - return "https://$domain/wp-content/uploads/cropped-sky-tv-1-32x32.png" - } } @MangaSourceParser("BAKAMAN", "BakaMan", "th") @@ -461,9 +449,6 @@ internal abstract class MadaraParser( override val isNsfwSource = true - override fun getFaviconUrl(): String { - return "https://cdn.$domain/wp-content/uploads/2019/01/cropped-cropped-pcround-32x32.png" - } } @MangaSourceParser("CAT_300", "Cat300", "th") @@ -483,17 +468,11 @@ internal abstract class MadaraParser( override val datePattern = "MM/dd/yyyy" - override fun getFaviconUrl(): String { - return "https://$domain/wp-content/uploads/2017/10/lgoo-treemanga-2-1.jpg" - } } @MangaSourceParser("MANGACV", "Manga Cv", "en") class MangaCv(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGACV, "mangacv.com") { - override fun getFaviconUrl(): String { - return "https://$domain/wp-content/uploads/2020/10/cropped-mangaCV-32x32.png" - } } @MangaSourceParser("MANGA_MANHUA", "Manga Manhua", "en") diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangalinkParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangalinkParser.kt index ff78aab4..25404f35 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangalinkParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangalinkParser.kt @@ -43,6 +43,4 @@ internal class MangalinkParser(context: MangaLoaderContext) : ) } - override fun getFaviconUrl(): String = - "https://cdn.${domain}/wp-content/uploads/2020/05/cropped-mangalink-180x180.jpg" } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangasOriginesParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangasOriginesParser.kt index af1c3801..f2cf3cab 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangasOriginesParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangasOriginesParser.kt @@ -8,7 +8,6 @@ import org.koitharu.kotatsu.parsers.model.MangaChapter import org.koitharu.kotatsu.parsers.model.MangaSource import org.koitharu.kotatsu.parsers.model.MangaState import org.koitharu.kotatsu.parsers.util.attrAsAbsoluteUrlOrNull -import org.koitharu.kotatsu.parsers.util.domain import org.koitharu.kotatsu.parsers.util.mapToSet import org.koitharu.kotatsu.parsers.util.selectFirstOrThrow import java.util.* @@ -19,10 +18,6 @@ internal class MangasOriginesParser(context: MangaLoaderContext) : override val tagPrefix = "catalogues-genre/" - override fun getFaviconUrl(): String { - return "https://${domain}/wp-content/uploads/2020/11/Mangas-150x150.png" - } - override fun parseDetails(manga: Manga, body: Element, chapters: List): Manga { val root = body.selectFirstOrThrow(".site-content") val postContent = root.selectFirstOrThrow(".post-content") diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/PrismaScansParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/PrismaScansParser.kt index b48770b0..cea3be89 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/PrismaScansParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/PrismaScansParser.kt @@ -8,7 +8,6 @@ import org.koitharu.kotatsu.parsers.model.MangaChapter import org.koitharu.kotatsu.parsers.model.MangaSource import org.koitharu.kotatsu.parsers.model.MangaState import org.koitharu.kotatsu.parsers.util.attrAsAbsoluteUrlOrNull -import org.koitharu.kotatsu.parsers.util.domain import org.koitharu.kotatsu.parsers.util.mapToSet import org.koitharu.kotatsu.parsers.util.selectFirstOrThrow import java.util.* @@ -20,10 +19,6 @@ internal class PrismaScansParser(context: MangaLoaderContext) : override val tagPrefix = "manga-genre/" override val datePattern = "MMM dd, yyyy" - override fun getFaviconUrl(): String { - return "https://${domain}/wp-content/uploads/2022/07/cropped-branca-1-192x192.png" - } - override fun parseDetails(manga: Manga, body: Element, chapters: List): Manga { val root = body.selectFirstOrThrow(".site-content") val postContent = root.selectFirstOrThrow(".post-content") diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/ReaperScansParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/ReaperScansParser.kt index b161995f..d613607d 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/ReaperScansParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/ReaperScansParser.kt @@ -83,6 +83,4 @@ internal class ReaperScansParser(context: MangaLoaderContext) : } } - override fun getFaviconUrl(): String = - "https://wuz.${domain}/wp-content/uploads/2022/03/20063242/cropped-ms-icon-310x310-2-192x192.png" } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/TatakaeScansParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/TatakaeScansParser.kt index 4a9a65cb..661f33c6 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/TatakaeScansParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/TatakaeScansParser.kt @@ -8,7 +8,6 @@ import org.koitharu.kotatsu.parsers.model.MangaChapter import org.koitharu.kotatsu.parsers.model.MangaSource import org.koitharu.kotatsu.parsers.model.MangaState import org.koitharu.kotatsu.parsers.util.attrAsAbsoluteUrlOrNull -import org.koitharu.kotatsu.parsers.util.domain import org.koitharu.kotatsu.parsers.util.mapToSet import org.koitharu.kotatsu.parsers.util.selectFirstOrThrow import java.util.* @@ -48,8 +47,4 @@ internal class TatakaeScansParser(context: MangaLoaderContext) : else -> null } - override fun getFaviconUrl(): String { - return "https://${domain}/wp-content/uploads/2022/07/cropped-favicon-180x180.png" - } - } From e51b33c74a912768ad2a502986f809e3ccd4f748 Mon Sep 17 00:00:00 2001 From: Saksham Shekher <95137948+OshekharO@users.noreply.github.com> Date: Thu, 9 Feb 2023 23:26:04 +0530 Subject: [PATCH 065/121] Add: Toonily, Komiklab, KomikDewasa, KomiKav (#148) * Add: Toonily * Add: KOMIKLAB * Update MangaReaderParser.kt * Add: KomikDewasa * Add: KomiKav --- .../parsers/site/madara/MadaraParser.kt | 11 +++- .../site/mangareader/MangaReaderParser.kt | 64 ++++++++++++++++++- 2 files changed, 72 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt index 5e8b95e9..3e8b0318 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt @@ -317,8 +317,7 @@ internal abstract class MadaraParser( } @MangaSourceParser("HACHIMANGA", "HachiManga", "ja") - class HachiManga(context: MangaLoaderContext) : - MadaraParser(context, MangaSource.HACHIMANGA, "hachimanga.com") { + class HachiManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.HACHIMANGA, "hachimanga.com") { override val sourceLocale: Locale = Locale.ENGLISH } @@ -475,6 +474,14 @@ internal abstract class MadaraParser( } + @MangaSourceParser("TOONILY", "Toonily", "en") + class Toonily(context: MangaLoaderContext) : MadaraParser(context, MangaSource.TOONILY, "toonily.com") { + + override val tagPrefix = "webtoon-genre/" + + override val isNsfwSource = true + } + @MangaSourceParser("MANGA_MANHUA", "Manga Manhua", "en") class MangaManhua(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGA_MANHUA, "mangamanhua.online") diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt index 8482aa43..904f65ab 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt @@ -500,11 +500,73 @@ internal abstract class MangaReaderParser( } } + @MangaSourceParser("KOMIKAV", "KomiKav", "id") + class KomiKavParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.KOMIKAV, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("komikav.com", null) + + override val listUrl: String + get() = "/manga" + override val tableMode: Boolean + get() = false + + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH) + + override suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { + val infoElement = docs.selectFirst("div.infox") + return manga.copy( + chapters = chapters, + description = infoElement?.selectFirst("div.entry-content")?.html(), + author = infoElement?.selectFirst(".flex-wrap div:contains(Author)")?.lastElementSibling()?.text(), + tags = infoElement?.select(".wd-full .mgen > a") + ?.mapNotNullToSet { getOrCreateTagMap()[it.text()] } + .orEmpty(), + ) + } + } + + @MangaSourceParser("KOMIKDEWASA", "KomikDewasa", "id") + class KomikDewasaParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.KOMIKDEWASA, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("komikdewasa.club", null) + + override val listUrl: String + get() = "/manga" + override val tableMode: Boolean + get() = false + + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH) + + override suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { + val infoElement = docs.selectFirst("div.infox") + return manga.copy( + chapters = chapters, + description = infoElement?.selectFirst("div.entry-content")?.html(), + ) + } + } + @MangaSourceParser("MANGASUSU", "Mangasusu", "id") class MangasusuParser(context: MangaLoaderContext) : MangaReaderParser(context, MangaSource.MANGASUSU, pageSize = 20, searchPageSize = 10) { override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("92.84.132.251", null) + get() = ConfigKey.Domain("mangasusu.co.in", null) + + override val listUrl: String + get() = "/project" + override val tableMode: Boolean + get() = true + + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH) + } + + @MangaSourceParser("KOMIKLAB", "KomikLab", "id") + class KomikLabParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.KOMIKLAB, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("komiklab.com", null) override val listUrl: String get() = "/project" From 29cd1f1fdf6ba8e0d7263ac168a0da97c9c3ee9c Mon Sep 17 00:00:00 2001 From: Saksham Shekher <95137948+OshekharO@users.noreply.github.com> Date: Tue, 14 Feb 2023 11:54:42 +0530 Subject: [PATCH 066/121] Add: KumaPoi --- .../parsers/site/mangareader/MangaReaderParser.kt | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt index 904f65ab..769e3b24 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt @@ -447,6 +447,20 @@ internal abstract class MangaReaderParser( override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", idLocale) } + @MangaSourceParser("KUMAPOI", "KumaPoi", "id") + class KumaPoiParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.KUMAPOI, pageSize = 15, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("kumapoi.me", null) + + override val listUrl: String + get() = "/manga" + override val tableMode: Boolean + get() = true + + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH) + } + @MangaSourceParser("ASURASCANS", "Asura Scans", "en") class AsuraScansParser(context: MangaLoaderContext) : MangaReaderParser(context, MangaSource.ASURASCANS, pageSize = 20, searchPageSize = 10) { From 503573f3880784e9445cf057f933335048ea8ca4 Mon Sep 17 00:00:00 2001 From: Saksham Shekher <95137948+OshekharO@users.noreply.github.com> Date: Tue, 14 Feb 2023 12:03:18 +0530 Subject: [PATCH 067/121] Add: KomikLokal --- .../site/mangareader/MangaReaderParser.kt | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt index 769e3b24..f61a9779 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt @@ -514,6 +514,30 @@ internal abstract class MangaReaderParser( } } + @MangaSourceParser("KOMIKLOKAL", "KomikLokal", "id") + class CosmicScansParser(context: MangaLoaderContext) : + KomikLokalParser(context, MangaSource.KOMIKLOKAL, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("komiklokal.pics", null) + + override val listUrl: String + get() = "/manga" + override val tableMode: Boolean + get() = false + + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH) + + override suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { + val infoElement = docs.selectFirst("div.infox") + return manga.copy( + chapters = chapters, + tags = infoElement?.select(".wd-full .mgen > a") + ?.mapNotNullToSet { getOrCreateTagMap()[it.text()] } + .orEmpty(), + ) + } + } + @MangaSourceParser("KOMIKAV", "KomiKav", "id") class KomiKavParser(context: MangaLoaderContext) : MangaReaderParser(context, MangaSource.KOMIKAV, pageSize = 20, searchPageSize = 10) { From 0da57c041cd7ffa15a3a13b2eaaa4725de25c9e3 Mon Sep 17 00:00:00 2001 From: Saksham Shekher <95137948+OshekharO@users.noreply.github.com> Date: Tue, 14 Feb 2023 12:09:40 +0530 Subject: [PATCH 068/121] Fix --- .../kotatsu/parsers/site/mangareader/MangaReaderParser.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt index f61a9779..8dcbbc54 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt @@ -515,8 +515,8 @@ internal abstract class MangaReaderParser( } @MangaSourceParser("KOMIKLOKAL", "KomikLokal", "id") - class CosmicScansParser(context: MangaLoaderContext) : - KomikLokalParser(context, MangaSource.KOMIKLOKAL, pageSize = 20, searchPageSize = 10) { + class KomikLokalParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.KOMIKLOKAL, pageSize = 20, searchPageSize = 10) { override val configKeyDomain: ConfigKey.Domain get() = ConfigKey.Domain("komiklokal.pics", null) From a51d6d64855b4d4c97dcb1b663eb49842e22e06b Mon Sep 17 00:00:00 2001 From: Saksham Shekher <95137948+OshekharO@users.noreply.github.com> Date: Tue, 14 Feb 2023 12:37:01 +0530 Subject: [PATCH 069/121] Add: Hentai20 --- .../kotatsu/parsers/site/madara/MadaraParser.kt | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt index 3e8b0318..f6c6dac7 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt @@ -405,7 +405,16 @@ internal abstract class MadaraParser( class MangaDistrict(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGA_DISTRICT, "mangadistrict.com") { - override val tagPrefix = "publication-genre/" + override val tagPrefix = "read-scan/" + + override val isNsfwSource = true + } + + @MangaSourceParser("HENTAI20", "Hentai20", "en") + class Hentai20(context: MangaLoaderContext) : + MadaraParser(context, MangaSource.HENTAI20, "hentai20.io") { + + override val tagPrefix = "manga-genre/" override val isNsfwSource = true } @@ -414,7 +423,7 @@ internal abstract class MadaraParser( class Hentai4Free(context: MangaLoaderContext) : MadaraParser(context, MangaSource.HENTAI_4FREE, "hentai4free.net") { - override val tagPrefix = "hentai-tag/" + override val tagPrefix = "hentai-genre/" override val isNsfwSource = true From 2c6c64fbe994e1e5f0535dd4bf55e8704a5a5d46 Mon Sep 17 00:00:00 2001 From: Saksham Shekher <95137948+OshekharO@users.noreply.github.com> Date: Tue, 14 Feb 2023 12:46:10 +0530 Subject: [PATCH 070/121] Add: ToonHunter --- .../site/mangareader/MangaReaderParser.kt | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt index 8dcbbc54..5a1b8499 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt @@ -488,6 +488,30 @@ internal abstract class MangaReaderParser( } } + @MangaSourceParser("TOONHUNTER", "Toon Hunter", "th") + class ToonHunterParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.TOONHUNTER, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("toonhunter.com", null) + + override val listUrl: String + get() = "/manga" + override val tableMode: Boolean + get() = false + + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH) + + override suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { + val infoElement = docs.selectFirst("div.infox") + return manga.copy( + chapters = chapters, + description = infoElement?.selectFirst("div.entry-content")?.html(), + author = infoElement?.selectFirst(".flex-wrap div:contains(Author)")?.lastElementSibling()?.text(), + isNsfw = docs.selectFirst(".postbody .alr") != null, + ) + } + } + @MangaSourceParser("COSMICSCANS", "CosmicScans", "en") class CosmicScansParser(context: MangaLoaderContext) : MangaReaderParser(context, MangaSource.COSMICSCANS, pageSize = 20, searchPageSize = 10) { From 454b24ec881e2a5d25cb37aed037e67098438c1c Mon Sep 17 00:00:00 2001 From: Koitharu Date: Thu, 16 Feb 2023 11:36:11 +0200 Subject: [PATCH 071/121] [Grouple] Selecting the fastest pages server --- .../parsers/site/grouple/GroupleParser.kt | 560 +++++++++--------- .../kotatsu/parsers/util/Coroutines.kt | 40 ++ 2 files changed, 325 insertions(+), 275 deletions(-) create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/util/Coroutines.kt diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt index e8f3a94b..bcf87638 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt @@ -1,5 +1,7 @@ package org.koitharu.kotatsu.parsers.site.grouple +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope import okhttp3.Headers import okhttp3.HttpUrl.Companion.toHttpUrl import okhttp3.Response @@ -21,298 +23,306 @@ private const val NSFW_ALERT = "сексуальные сцены" private const val NOTHING_FOUND = "Ничего не найдено" internal abstract class GroupleParser( - context: MangaLoaderContext, - source: MangaSource, - userAgent: String, - private val siteId: Int, + context: MangaLoaderContext, + source: MangaSource, + userAgent: String, + private val siteId: Int, ) : MangaParser(context, source), MangaParserAuthProvider { - override val headers = Headers.Builder() - .add("User-Agent", userAgent) - .build() + @Volatile + private var cachedPagesServer: String? = null - override val sortOrders: Set = EnumSet.of( - SortOrder.UPDATED, - SortOrder.POPULARITY, - SortOrder.NEWEST, - SortOrder.RATING, - ) + override val headers = Headers.Builder() + .add("User-Agent", userAgent) + .build() - override val authUrl: String - get() { - val targetUri = "https://${domain}/".urlEncoded() - return "https://grouple.co/internal/auth/sso?siteId=$siteId&=targetUri=$targetUri" - } + override val sortOrders: Set = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.NEWEST, + SortOrder.RATING, + ) - override val isAuthorized: Boolean - get() = context.cookieJar.getCookies(domain).any { it.name == "gwt" } + override val authUrl: String + get() { + val targetUri = "https://${domain}/".urlEncoded() + return "https://grouple.co/internal/auth/sso?siteId=$siteId&=targetUri=$targetUri" + } - override suspend fun getList( - offset: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - val domain = domain - val doc = when { - !query.isNullOrEmpty() -> webClient.httpPost( - "https://$domain/search", - mapOf( - "q" to query.urlEncoded(), - "offset" to (offset upBy PAGE_SIZE_SEARCH).toString(), - ), - ) + override val isAuthorized: Boolean + get() = context.cookieJar.getCookies(domain).any { it.name == "gwt" } - tags.isNullOrEmpty() -> webClient.httpGet( - "https://$domain/list?sortType=${ - getSortKey(sortOrder) - }&offset=${offset upBy PAGE_SIZE}", - ) + override suspend fun getList( + offset: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + val domain = domain + val doc = when { + !query.isNullOrEmpty() -> webClient.httpPost( + "https://$domain/search", + mapOf( + "q" to query.urlEncoded(), + "offset" to (offset upBy PAGE_SIZE_SEARCH).toString(), + ), + ) - tags.size == 1 -> webClient.httpGet( - "https://$domain/list/genre/${tags.first().key}?sortType=${ - getSortKey(sortOrder) - }&offset=${offset upBy PAGE_SIZE}", - ) + tags.isNullOrEmpty() -> webClient.httpGet( + "https://$domain/list?sortType=${ + getSortKey(sortOrder) + }&offset=${offset upBy PAGE_SIZE}", + ) - offset > 0 -> return emptyList() - else -> advancedSearch(domain, tags) - }.parseHtml().body() - val root = (doc.getElementById("mangaBox") ?: doc.getElementById("mangaResults")) - ?: doc.parseFailed("Cannot find root") - val tiles = root.selectFirst("div.tiles.row") ?: if ( - root.select(".alert").any { it.ownText() == NOTHING_FOUND } - ) { - return emptyList() - } else { - doc.parseFailed("No tiles found") - } - val baseHost = root.baseUri().toHttpUrl().host - return tiles.select("div.tile").mapNotNull { node -> - val imgDiv = node.selectFirst("div.img") ?: return@mapNotNull null - val descDiv = node.selectFirst("div.desc") ?: return@mapNotNull null - if (descDiv.selectFirst("i.fa-user") != null) { - return@mapNotNull null // skip author - } - val href = imgDiv.selectFirst("a")?.attrAsAbsoluteUrlOrNull("href") - if (href == null || href.toHttpUrl().host != baseHost) { - return@mapNotNull null // skip external links - } - val title = descDiv.selectFirst("h3")?.selectFirst("a")?.text() - ?: return@mapNotNull null - val tileInfo = descDiv.selectFirst("div.tile-info") - val relUrl = href.toRelativeUrl(baseHost) - Manga( - id = generateUid(relUrl), - url = relUrl, - publicUrl = href, - title = title, - altTitle = descDiv.selectFirst("h4")?.text(), - coverUrl = imgDiv.selectFirst("img.lazy")?.attr("data-original")?.replace("_p.", ".").orEmpty(), - rating = runCatching { - node.selectFirst(".compact-rate") - ?.attr("title") - ?.toFloatOrNull() - ?.div(5f) - }.getOrNull() ?: RATING_UNKNOWN, - author = tileInfo?.selectFirst("a.person-link")?.text(), - isNsfw = false, - tags = runCatching { - tileInfo?.select("a.element-link") - ?.mapToSet { - MangaTag( - title = it.text().toTitleCase(), - key = it.attr("href").substringAfterLast('/'), - source = source, - ) - } - }.getOrNull().orEmpty(), - state = when { - node.selectFirst("div.tags") - ?.selectFirst("span.mangaCompleted") != null -> MangaState.FINISHED + tags.size == 1 -> webClient.httpGet( + "https://$domain/list/genre/${tags.first().key}?sortType=${ + getSortKey(sortOrder) + }&offset=${offset upBy PAGE_SIZE}", + ) - else -> null - }, - source = source, - ) - } - } + offset > 0 -> return emptyList() + else -> advancedSearch(domain, tags) + }.parseHtml().body() + val root = (doc.getElementById("mangaBox") ?: doc.getElementById("mangaResults")) + ?: doc.parseFailed("Cannot find root") + val tiles = root.selectFirst("div.tiles.row") ?: if ( + root.select(".alert").any { it.ownText() == NOTHING_FOUND } + ) { + return emptyList() + } else { + doc.parseFailed("No tiles found") + } + val baseHost = root.baseUri().toHttpUrl().host + return tiles.select("div.tile").mapNotNull { node -> + val imgDiv = node.selectFirst("div.img") ?: return@mapNotNull null + val descDiv = node.selectFirst("div.desc") ?: return@mapNotNull null + if (descDiv.selectFirst("i.fa-user") != null) { + return@mapNotNull null // skip author + } + val href = imgDiv.selectFirst("a")?.attrAsAbsoluteUrlOrNull("href") + if (href == null || href.toHttpUrl().host != baseHost) { + return@mapNotNull null // skip external links + } + val title = descDiv.selectFirst("h3")?.selectFirst("a")?.text() + ?: return@mapNotNull null + val tileInfo = descDiv.selectFirst("div.tile-info") + val relUrl = href.toRelativeUrl(baseHost) + Manga( + id = generateUid(relUrl), + url = relUrl, + publicUrl = href, + title = title, + altTitle = descDiv.selectFirst("h4")?.text(), + coverUrl = imgDiv.selectFirst("img.lazy")?.attr("data-original")?.replace("_p.", ".").orEmpty(), + rating = runCatching { + node.selectFirst(".compact-rate") + ?.attr("title") + ?.toFloatOrNull() + ?.div(5f) + }.getOrNull() ?: RATING_UNKNOWN, + author = tileInfo?.selectFirst("a.person-link")?.text(), + isNsfw = false, + tags = runCatching { + tileInfo?.select("a.element-link") + ?.mapToSet { + MangaTag( + title = it.text().toTitleCase(), + key = it.attr("href").substringAfterLast('/'), + source = source, + ) + } + }.getOrNull().orEmpty(), + state = when { + node.selectFirst("div.tags") + ?.selectFirst("span.mangaCompleted") != null -> MangaState.FINISHED - override suspend fun getDetails(manga: Manga): Manga { - val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).checkAuthRequired().parseHtml() - val root = doc.body().getElementById("mangaBox")?.selectFirst("div.leftContent") - ?: doc.parseFailed("Cannot find root") - val dateFormat = SimpleDateFormat("dd.MM.yy", Locale.US) - val coverImg = root.selectFirst("div.subject-cover")?.selectFirst("img") - return manga.copy( - description = root.selectFirst("div.manga-description")?.html(), - largeCoverUrl = coverImg?.attr("data-full"), - coverUrl = coverImg?.attr("data-thumb") ?: manga.coverUrl, - tags = manga.tags + root.select("div.subject-meta").select("span.elem_genre ") - .mapNotNull { - val a = it.selectFirst("a.element-link") ?: return@mapNotNull null - MangaTag( - title = a.text().toTitleCase(), - key = a.attr("href").substringAfterLast('/'), - source = source, - ) - }, - author = root.selectFirst("a.person-link")?.text() ?: manga.author, - isNsfw = root.select(".alert-warning").any { it.ownText().contains(NSFW_ALERT) }, - chapters = root.selectFirst("div.chapters-link")?.selectFirst("table") - ?.select("tr:has(td > a)")?.asReversed()?.mapChapters { i, tr -> - val a = tr.selectFirst("a.chapter-link") ?: return@mapChapters null - val href = a.attrAsRelativeUrl("href") - var translators = "" - val translatorElement = a.attr("title") - if (!translatorElement.isNullOrBlank()) { - translators = translatorElement - .replace("(Переводчик),", "&") - .removeSuffix(" (Переводчик)") - } - MangaChapter( - id = generateUid(href), - name = tr.selectFirst("a")?.text().orEmpty().removePrefix(manga.title).trim(), - number = i + 1, - url = href, - uploadDate = dateFormat.tryParse(tr.selectFirst("td.date")?.text()), - scanlator = translators, - source = source, - branch = null, - ) - }, - ) - } + else -> null + }, + source = source, + ) + } + } - override suspend fun getPages(chapter: MangaChapter): List { - val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain) + "?mtr=1") - .checkAuthRequired() - .parseHtml() - val scripts = doc.select("script") - for (script in scripts) { - val data = script.html() - val pos = data.indexOf("rm_h.readerInit( 0,") - if (pos == -1) { - continue - } - val json = data.substring(pos) - .substringAfter('(') - .substringBefore('\n') - .substringBeforeLast(')') - if (json.isEmpty()) { - continue - } - val ja = JSONArray("[$json]") - val pages = ja.getJSONArray(1) - val servers = ja.getJSONArray(3).mapJSON { it.getString("path") } - val serversStr = servers.joinToString("|") - return (0 until pages.length()).map { i -> - val page = pages.getJSONArray(i) - val primaryServer = page.getString(0) - val url = page.getString(2) - MangaPage( - id = generateUid(url), - url = "$primaryServer|$serversStr|$url", - preview = null, - referer = chapter.url, - source = source, - ) - } - } - doc.parseFailed("Pages list not found at ${chapter.url}") - } + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).checkAuthRequired().parseHtml() + val root = doc.body().getElementById("mangaBox")?.selectFirst("div.leftContent") + ?: doc.parseFailed("Cannot find root") + val dateFormat = SimpleDateFormat("dd.MM.yy", Locale.US) + val coverImg = root.selectFirst("div.subject-cover")?.selectFirst("img") + return manga.copy( + description = root.selectFirst("div.manga-description")?.html(), + largeCoverUrl = coverImg?.attr("data-full"), + coverUrl = coverImg?.attr("data-thumb") ?: manga.coverUrl, + tags = manga.tags + root.select("div.subject-meta").select("span.elem_genre ") + .mapNotNull { + val a = it.selectFirst("a.element-link") ?: return@mapNotNull null + MangaTag( + title = a.text().toTitleCase(), + key = a.attr("href").substringAfterLast('/'), + source = source, + ) + }, + author = root.selectFirst("a.person-link")?.text() ?: manga.author, + isNsfw = root.select(".alert-warning").any { it.ownText().contains(NSFW_ALERT) }, + chapters = root.selectFirst("div.chapters-link")?.selectFirst("table") + ?.select("tr:has(td > a)")?.asReversed()?.mapChapters { i, tr -> + val a = tr.selectFirst("a.chapter-link") ?: return@mapChapters null + val href = a.attrAsRelativeUrl("href") + var translators = "" + val translatorElement = a.attr("title") + if (!translatorElement.isNullOrBlank()) { + translators = translatorElement + .replace("(Переводчик),", "&") + .removeSuffix(" (Переводчик)") + } + MangaChapter( + id = generateUid(href), + name = tr.selectFirst("a")?.text().orEmpty().removePrefix(manga.title).trim(), + number = i + 1, + url = href, + uploadDate = dateFormat.tryParse(tr.selectFirst("td.date")?.text()), + scanlator = translators, + source = source, + branch = null, + ) + }, + ) + } - override suspend fun getPageUrl(page: MangaPage): String { - val parts = page.url.split('|') - val path = parts.last() - val servers = parts.dropLast(1).toSet() - for (server in servers) { - val url = server + path - if (tryHead(url)) { - return url - } - } - val fallbackServer = servers.firstOrNull() ?: throw ParseException("Cannot find any page url", page.url) - return fallbackServer + path - } + override suspend fun getPages(chapter: MangaChapter): List { + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain) + "?mtr=1") + .checkAuthRequired() + .parseHtml() + val scripts = doc.select("script") + for (script in scripts) { + val data = script.html() + val pos = data.indexOf("rm_h.readerInit( 0,") + if (pos == -1) { + continue + } + val json = data.substring(pos) + .substringAfter('(') + .substringBefore('\n') + .substringBeforeLast(')') + if (json.isEmpty()) { + continue + } + val ja = JSONArray("[$json]") + val pages = ja.getJSONArray(1) + val servers = ja.getJSONArray(3).mapJSON { it.getString("path") } + val serversStr = servers.joinToString("|") + return (0 until pages.length()).map { i -> + val page = pages.getJSONArray(i) + val primaryServer = page.getString(0) + val url = page.getString(2) + MangaPage( + id = generateUid(url), + url = "$primaryServer|$serversStr|$url", + preview = null, + referer = chapter.url, + source = source, + ) + } + } + doc.parseFailed("Pages list not found at ${chapter.url}") + } - override suspend fun getTags(): Set { - val doc = webClient.httpGet("https://${domain}/list/genres/sort_name").parseHtml() - val root = doc.body().getElementById("mangaBox")?.selectFirst("div.leftContent") - ?.selectFirst("table.table") ?: doc.parseFailed("Cannot find root") - return root.select("a.element-link").mapToSet { a -> - MangaTag( - title = a.text().toTitleCase(), - key = a.attr("href").substringAfterLast('/'), - source = source, - ) - } - } + override suspend fun getPageUrl(page: MangaPage): String { + val parts = page.url.split('|') + val path = parts.last() + val servers = parts.dropLast(1).toSet() + val cachedServer = cachedPagesServer + if (cachedServer != null && cachedServer in servers && tryHead(cachedServer + path)) { + return cachedServer + path + } + val server = coroutineScope { + servers.map { server -> + async { + if (tryHead(server + path)) server else null + } + }.awaitFirst { it != null } + } + cachedPagesServer = server + return checkNotNull(server + path) + } - override suspend fun getUsername(): String { - val root = webClient.httpGet("https://grouple.co/").parseHtml().body() - val element = root.selectFirst("img.user-avatar") ?: throw AuthRequiredException(source) - val res = element.parent()?.text() - return if (res.isNullOrEmpty()) { - root.parseFailed("Cannot find username") - } else res - } + override suspend fun getTags(): Set { + val doc = webClient.httpGet("https://${domain}/list/genres/sort_name").parseHtml() + val root = doc.body().getElementById("mangaBox")?.selectFirst("div.leftContent") + ?.selectFirst("table.table") ?: doc.parseFailed("Cannot find root") + return root.select("a.element-link").mapToSet { a -> + MangaTag( + title = a.text().toTitleCase(), + key = a.attr("href").substringAfterLast('/'), + source = source, + ) + } + } - private fun getSortKey(sortOrder: SortOrder) = - when (sortOrder) { - SortOrder.ALPHABETICAL -> "name" - SortOrder.POPULARITY -> "rate" - SortOrder.UPDATED -> "updated" - SortOrder.NEWEST -> "created" - SortOrder.RATING -> "votes" - } + override suspend fun getUsername(): String { + val root = webClient.httpGet("https://grouple.co/").parseHtml().body() + val element = root.selectFirst("img.user-avatar") ?: throw AuthRequiredException(source) + val res = element.parent()?.text() + return if (res.isNullOrEmpty()) { + root.parseFailed("Cannot find username") + } else res + } - private suspend fun advancedSearch(domain: String, tags: Set): Response { - val url = "https://$domain/search/advanced" - // Step 1: map catalog genres names to advanced-search genres ids - val tagsIndex = webClient.httpGet(url).parseHtml() - .body().selectFirst("form.search-form") - ?.select("div.form-group") - ?.get(1) ?: throw ParseException("Genres filter element not found", url) - val tagNames = tags.map { it.title.lowercase() } - val payload = HashMap() - var foundGenres = 0 - tagsIndex.select("li.property").forEach { li -> - val name = li.text().trim().lowercase() - val id = li.selectFirst("input")?.id() - ?: li.parseFailed("Id for tag $name not found") - payload[id] = if (name in tagNames) { - foundGenres++ - "in" - } else "" - } - if (foundGenres != tags.size) { - tagsIndex.parseFailed("Some genres are not found") - } - // Step 2: advanced search - payload["q"] = "" - payload["s_high_rate"] = "" - payload["s_single"] = "" - payload["s_mature"] = "" - payload["s_completed"] = "" - payload["s_translated"] = "" - payload["s_many_chapters"] = "" - payload["s_wait_upload"] = "" - payload["s_sale"] = "" - payload["years"] = "1900,2099" - payload["+"] = "Искать".urlEncoded() - return webClient.httpPost(url, payload) - } + private fun getSortKey(sortOrder: SortOrder) = + when (sortOrder) { + SortOrder.ALPHABETICAL -> "name" + SortOrder.POPULARITY -> "rate" + SortOrder.UPDATED -> "updated" + SortOrder.NEWEST -> "created" + SortOrder.RATING -> "votes" + } - private suspend fun tryHead(url: String): Boolean = runCatchingCancellable { - webClient.httpHead(url).isSuccessful - }.getOrDefault(false) + private suspend fun advancedSearch(domain: String, tags: Set): Response { + val url = "https://$domain/search/advanced" + // Step 1: map catalog genres names to advanced-search genres ids + val tagsIndex = webClient.httpGet(url).parseHtml() + .body().selectFirst("form.search-form") + ?.select("div.form-group") + ?.get(1) ?: throw ParseException("Genres filter element not found", url) + val tagNames = tags.map { it.title.lowercase() } + val payload = HashMap() + var foundGenres = 0 + tagsIndex.select("li.property").forEach { li -> + val name = li.text().trim().lowercase() + val id = li.selectFirst("input")?.id() + ?: li.parseFailed("Id for tag $name not found") + payload[id] = if (name in tagNames) { + foundGenres++ + "in" + } else "" + } + if (foundGenres != tags.size) { + tagsIndex.parseFailed("Some genres are not found") + } + // Step 2: advanced search + payload["q"] = "" + payload["s_high_rate"] = "" + payload["s_single"] = "" + payload["s_mature"] = "" + payload["s_completed"] = "" + payload["s_translated"] = "" + payload["s_many_chapters"] = "" + payload["s_wait_upload"] = "" + payload["s_sale"] = "" + payload["years"] = "1900,2099" + payload["+"] = "Искать".urlEncoded() + return webClient.httpPost(url, payload) + } - private fun Response.checkAuthRequired(): Response { - val lastPathSegment = request.url.pathSegments.lastOrNull() ?: return this - if (lastPathSegment == "login") { - throw AuthRequiredException(source) - } - return this - } + private suspend fun tryHead(url: String): Boolean = runCatchingCancellable { + webClient.httpHead(url).isSuccessful + }.getOrDefault(false) + + private fun Response.checkAuthRequired(): Response { + val lastPathSegment = request.url.pathSegments.lastOrNull() ?: return this + if (lastPathSegment == "login") { + throw AuthRequiredException(source) + } + return this + } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Coroutines.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Coroutines.kt new file mode 100644 index 00000000..52fb8b17 --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Coroutines.kt @@ -0,0 +1,40 @@ +package org.koitharu.kotatsu.parsers.util + +import kotlinx.coroutines.Deferred +import kotlinx.coroutines.Job +import kotlinx.coroutines.selects.select +import kotlin.coroutines.cancellation.CancellationException + +fun Iterable.cancelAll(cause: CancellationException? = null) { + forEach { it.cancel(cause) } +} + +suspend fun Iterable>.awaitFirst(): T = select { + for (async in this@awaitFirst) { + async.onAwait { it } + } +}.also { this@awaitFirst.cancelAll() } + +suspend fun Collection>.awaitFirst(condition: (T) -> Boolean): T { + var result: Any? = NULL + var counter = size + while (result === NULL && counter > 0) { + val candidate = select { + for (async in this@awaitFirst) { + async.onAwait { it } + } + } + if (condition(candidate)) { + result = candidate + } + counter-- + } + cancelAll() + if (result === NULL) { + throw NoSuchElementException() + } + @Suppress("UNCHECKED_CAST") + return result as T +} + +private val NULL = Any() From f3b731114e7d01a6968e95d22b299abdebb2eb9d Mon Sep 17 00:00:00 2001 From: Koitharu Date: Thu, 16 Feb 2023 17:50:12 +0200 Subject: [PATCH 072/121] [DesuMe] Genres in list --- .../kotatsu/parsers/site/DesuMeParser.kt | 293 ++++++++++-------- .../kotatsu/parsers/util/SuspendLazy.kt | 36 +++ 2 files changed, 192 insertions(+), 137 deletions(-) create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/util/SuspendLazy.kt diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/DesuMeParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/DesuMeParser.kt index 3b76baed..3004dcc4 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/DesuMeParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/DesuMeParser.kt @@ -1,5 +1,6 @@ package org.koitharu.kotatsu.parsers.site +import androidx.collection.ArrayMap import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaSourceParser import org.koitharu.kotatsu.parsers.PagedMangaParser @@ -15,147 +16,165 @@ import java.util.* @MangaSourceParser("DESUME", "Desu.me", "ru") internal class DesuMeParser(context: MangaLoaderContext) : PagedMangaParser(context, MangaSource.DESUME, 20) { - override val configKeyDomain = ConfigKey.Domain("desu.me", null) + override val configKeyDomain = ConfigKey.Domain("desu.me", null) - override val sortOrders: Set = EnumSet.of( - SortOrder.UPDATED, - SortOrder.POPULARITY, - SortOrder.NEWEST, - SortOrder.ALPHABETICAL, - ) + override val sortOrders: Set = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.NEWEST, + SortOrder.ALPHABETICAL, + ) - override suspend fun getListPage( - page: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - if (query != null && page != searchPaginator.firstPage) { - return emptyList() - } - val domain = domain - val url = buildString { - append("https://") - append(domain) - append("/manga/api/?limit=20&order=") - append(getSortKey(sortOrder)) - append("&page=") - append(page) - if (!tags.isNullOrEmpty()) { - append("&genres=") - appendAll(tags, ",") { it.key } - } - if (query != null) { - append("&search=") - append(query) - } - } - val json = webClient.httpGet(url).parseJson().getJSONArray("response") - ?: throw ParseException("Invalid response", url) - val total = json.length() - val list = ArrayList(total) - for (i in 0 until total) { - val jo = json.getJSONObject(i) - val cover = jo.getJSONObject("image") - val id = jo.getLong("id") - list += Manga( - url = "/manga/api/$id", - publicUrl = jo.getString("url"), - source = MangaSource.DESUME, - title = jo.getString("russian"), - altTitle = jo.getString("name"), - coverUrl = cover.getString("preview"), - largeCoverUrl = cover.getString("original"), - state = when { - jo.getInt("ongoing") == 1 -> MangaState.ONGOING - else -> null - }, - rating = jo.getDouble("score").toFloat().coerceIn(0f, 1f), - id = generateUid(id), - isNsfw = false, - tags = emptySet(), - author = null, - description = jo.getString("description"), - ) - } - return list - } + private val tagsCache = SuspendLazy(::fetchTags) - override suspend fun getDetails(manga: Manga): Manga { - val url = manga.url.toAbsoluteUrl(domain) - val json = webClient.httpGet(url).parseJson().getJSONObject("response") - ?: throw ParseException("Invalid response", url) - val baseChapterUrl = manga.url + "/chapter/" - val chaptersList = json.getJSONObject("chapters").getJSONArray("list") - val totalChapters = chaptersList.length() - return manga.copy( - tags = json.getJSONArray("genres").mapJSONToSet { - MangaTag( - key = it.getString("text"), - title = it.getString("russian").toTitleCase(), - source = manga.source, - ) - }, - publicUrl = json.getString("url"), - description = json.getString("description"), - chapters = chaptersList.mapJSONIndexed { i, it -> - val chid = it.getLong("id") - val volChap = "Том " + it.optString("vol", "0") + ". " + "Глава " + it.optString("ch", "0") - val title = it.optString("title", "null").takeUnless { it == "null" } - MangaChapter( - id = generateUid(chid), - source = manga.source, - url = "$baseChapterUrl$chid", - uploadDate = it.getLong("date") * 1000, - name = if (title.isNullOrEmpty()) volChap else "$volChap: $title", - number = totalChapters - i, - scanlator = null, - branch = null, - ) - }.reversed(), - ) - } + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + if (query != null && page != searchPaginator.firstPage) { + return emptyList() + } + val domain = domain + val url = buildString { + append("https://") + append(domain) + append("/manga/api/?limit=20&order=") + append(getSortKey(sortOrder)) + append("&page=") + append(page) + if (!tags.isNullOrEmpty()) { + append("&genres=") + appendAll(tags, ",") { it.key } + } + if (query != null) { + append("&search=") + append(query) + } + } + val json = webClient.httpGet(url).parseJson().getJSONArray("response") + ?: throw ParseException("Invalid response", url) + val total = json.length() + val list = ArrayList(total) + val tagsMap = tagsCache.tryGet().getOrNull() + for (i in 0 until total) { + val jo = json.getJSONObject(i) + val cover = jo.getJSONObject("image") + val id = jo.getLong("id") + val genres = jo.getString("genres").split(',') + list += Manga( + url = "/manga/api/$id", + publicUrl = jo.getString("url"), + source = MangaSource.DESUME, + title = jo.getString("russian"), + altTitle = jo.getString("name"), + coverUrl = cover.getString("preview"), + largeCoverUrl = cover.getString("original"), + state = when { + jo.getInt("ongoing") == 1 -> MangaState.ONGOING + else -> null + }, + rating = jo.getDouble("score").toFloat().coerceIn(0f, 1f), + id = generateUid(id), + isNsfw = false, + tags = if (!tagsMap.isNullOrEmpty()) { + genres.mapNotNullToSet { g -> + tagsMap[g.trim().toTitleCase()] + } + } else { + emptySet() + }, + author = null, + description = jo.getString("description"), + ) + } + return list + } - override suspend fun getPages(chapter: MangaChapter): List { - val fullUrl = chapter.url.toAbsoluteUrl(domain) - val json = webClient.httpGet(fullUrl) - .parseJson() - .getJSONObject("response") ?: throw ParseException("Invalid response", fullUrl) - return json.getJSONObject("pages").getJSONArray("list").mapJSON { jo -> - MangaPage( - id = generateUid(jo.getLong("id")), - referer = fullUrl, - preview = null, - source = chapter.source, - url = jo.getString("img"), - ) - } - } + override suspend fun getDetails(manga: Manga): Manga { + val url = manga.url.toAbsoluteUrl(domain) + val json = webClient.httpGet(url).parseJson().getJSONObject("response") + ?: throw ParseException("Invalid response", url) + val baseChapterUrl = manga.url + "/chapter/" + val chaptersList = json.getJSONObject("chapters").getJSONArray("list") + val totalChapters = chaptersList.length() + return manga.copy( + tags = json.getJSONArray("genres").mapJSONToSet { + MangaTag( + key = it.getString("text"), + title = it.getString("russian").toTitleCase(), + source = manga.source, + ) + }, + publicUrl = json.getString("url"), + description = json.getString("description"), + chapters = chaptersList.mapJSONIndexed { i, it -> + val chid = it.getLong("id") + val volChap = "Том " + it.optString("vol", "0") + ". " + "Глава " + it.optString("ch", "0") + val title = it.optString("title", "null").takeUnless { it == "null" } + MangaChapter( + id = generateUid(chid), + source = manga.source, + url = "$baseChapterUrl$chid", + uploadDate = it.getLong("date") * 1000, + name = if (title.isNullOrEmpty()) volChap else "$volChap: $title", + number = totalChapters - i, + scanlator = null, + branch = null, + ) + }.reversed(), + ) + } - override suspend fun getTags(): Set { - val doc = webClient.httpGet("https://${domain}/manga/").parseHtml() - val root = doc.body().requireElementById("animeFilter") - .selectFirstOrThrow(".catalog-genres") - return root.select("li").mapToSet { - val input = it.selectFirstOrThrow("input") - MangaTag( - source = source, - key = input.attr("data-genre-slug").ifEmpty { - it.parseFailed("data-genre-slug is empty") - }, - title = input.attr("data-genre-name").toTitleCase().ifEmpty { - it.parseFailed("data-genre-name is empty") - }, - ) - } - } + override suspend fun getPages(chapter: MangaChapter): List { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val json = webClient.httpGet(fullUrl) + .parseJson() + .getJSONObject("response") ?: throw ParseException("Invalid response", fullUrl) + return json.getJSONObject("pages").getJSONArray("list").mapJSON { jo -> + MangaPage( + id = generateUid(jo.getLong("id")), + referer = fullUrl, + preview = null, + source = chapter.source, + url = jo.getString("img"), + ) + } + } - private fun getSortKey(sortOrder: SortOrder) = - when (sortOrder) { - SortOrder.ALPHABETICAL -> "name" - SortOrder.POPULARITY -> "popular" - SortOrder.UPDATED -> "updated" - SortOrder.NEWEST -> "id" - else -> "updated" - } + override suspend fun getTags(): Set { + return tagsCache.get().values.toSet() + } + + private fun getSortKey(sortOrder: SortOrder) = + when (sortOrder) { + SortOrder.ALPHABETICAL -> "name" + SortOrder.POPULARITY -> "popular" + SortOrder.UPDATED -> "updated" + SortOrder.NEWEST -> "id" + else -> "updated" + } + + private suspend fun fetchTags(): Map { + val doc = webClient.httpGet("https://${domain}/manga/").parseHtml() + val root = doc.body().requireElementById("animeFilter") + .selectFirstOrThrow(".catalog-genres") + val li = root.select("li") + val result = ArrayMap(li.size) + li.forEach { + val input = it.selectFirstOrThrow("input") + val tag = MangaTag( + source = source, + key = input.attr("data-genre-slug").ifEmpty { + it.parseFailed("data-genre-slug is empty") + }, + title = input.attr("data-genre-name").toTitleCase().ifEmpty { + it.parseFailed("data-genre-name is empty") + }, + ) + result[tag.title] = tag + } + return result + } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/SuspendLazy.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/SuspendLazy.kt new file mode 100644 index 00000000..cb9a911a --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/SuspendLazy.kt @@ -0,0 +1,36 @@ +package org.koitharu.kotatsu.parsers.util + +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock + +class SuspendLazy( + private val initializer: suspend () -> T, +) { + + private val mutex = Mutex() + private var cachedValue: Any? = Uninitialized + + @Suppress("UNCHECKED_CAST") + suspend fun get(): T { + // fast way + cachedValue.let { + if (it !== Uninitialized) { + return it as T + } + } + return mutex.withLock { + cachedValue.let { + if (it !== Uninitialized) { + return it as T + } + } + val result = initializer() + cachedValue = result + result + } + } + + suspend fun tryGet() = runCatchingCancellable { get() } + + private object Uninitialized +} From cf345d2d0c831e991be9f117be789aba4865ad14 Mon Sep 17 00:00:00 2001 From: Koitharu Date: Thu, 16 Feb 2023 18:52:20 +0200 Subject: [PATCH 073/121] [Grouple] update UserAgent --- .../kotatsu/parsers/site/grouple/GroupleParser.kt | 6 ++++-- .../kotatsu/parsers/site/grouple/MintMangaParser.kt | 12 ++++++------ .../kotatsu/parsers/site/grouple/ReadmangaParser.kt | 12 ++++++------ .../kotatsu/parsers/site/grouple/SelfMangaParser.kt | 6 +++--- 4 files changed, 19 insertions(+), 17 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt index bcf87638..b98fd807 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt @@ -25,7 +25,6 @@ private const val NOTHING_FOUND = "Ничего не найдено" internal abstract class GroupleParser( context: MangaLoaderContext, source: MangaSource, - userAgent: String, private val siteId: Int, ) : MangaParser(context, source), MangaParserAuthProvider { @@ -33,7 +32,10 @@ internal abstract class GroupleParser( private var cachedPagesServer: String? = null override val headers = Headers.Builder() - .add("User-Agent", userAgent) + .add( + "User-Agent", + "Mozilla/5.0 (X11; U; UNICOS lcLinux; en-US) Gecko/20140730 (KHTML, like Gecko, Safari/419.3) Arora/0.8.0", + ) .build() override val sortOrders: Set = EnumSet.of( diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/MintMangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/MintMangaParser.kt index 3cc040af..fa3c5835 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/MintMangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/MintMangaParser.kt @@ -7,12 +7,12 @@ import org.koitharu.kotatsu.parsers.model.MangaSource @MangaSourceParser("MINTMANGA", "MintManga", "ru") internal class MintMangaParser( - context: MangaLoaderContext, -) : GroupleParser(context, MangaSource.MINTMANGA, "mintmangafun", 2) { + context: MangaLoaderContext, +) : GroupleParser(context, MangaSource.MINTMANGA, 2) { - override val configKeyDomain = ConfigKey.Domain( - "mintmanga.live", - arrayOf("mintmanga.live", "mintmanga.com"), - ) + override val configKeyDomain = ConfigKey.Domain( + "mintmanga.live", + arrayOf("mintmanga.live", "mintmanga.com"), + ) } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/ReadmangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/ReadmangaParser.kt index 9ac27e9a..a68bfe63 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/ReadmangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/ReadmangaParser.kt @@ -7,12 +7,12 @@ import org.koitharu.kotatsu.parsers.model.MangaSource @MangaSourceParser("READMANGA_RU", "ReadManga", "ru") internal class ReadmangaParser( - context: MangaLoaderContext, -) : GroupleParser(context, MangaSource.READMANGA_RU, "readmangafun", 1) { + context: MangaLoaderContext, +) : GroupleParser(context, MangaSource.READMANGA_RU, 1) { - override val configKeyDomain = ConfigKey.Domain( - "readmanga.live", - arrayOf("readmanga.io", "readmanga.live", "readmanga.me"), - ) + override val configKeyDomain = ConfigKey.Domain( + "readmanga.live", + arrayOf("readmanga.io", "readmanga.live", "readmanga.me"), + ) } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/SelfMangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/SelfMangaParser.kt index 44fc703b..00006135 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/SelfMangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/SelfMangaParser.kt @@ -7,9 +7,9 @@ import org.koitharu.kotatsu.parsers.model.MangaSource @MangaSourceParser("SELFMANGA", "SelfManga", "ru") internal class SelfMangaParser( - context: MangaLoaderContext, -) : GroupleParser(context, MangaSource.SELFMANGA, "selfmangafun", 3) { + context: MangaLoaderContext, +) : GroupleParser(context, MangaSource.SELFMANGA, 3) { - override val configKeyDomain = ConfigKey.Domain("selfmanga.live", null) + override val configKeyDomain = ConfigKey.Domain("selfmanga.live", null) } From 03f021377df896496720c4a4aa67cc5afd6ba3e4 Mon Sep 17 00:00:00 2001 From: Saksham Shekher <95137948+OshekharO@users.noreply.github.com> Date: Thu, 16 Feb 2023 23:17:31 +0530 Subject: [PATCH 074/121] Update README.md some minor grammatical corrections --- README.md | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 2dad2486..35186f09 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,12 @@ # Kotatsu parsers -Library that provides manga sources. +This library provides manga sources. [![](https://jitpack.io/v/KotatsuApp/kotatsu-parsers.svg)](https://jitpack.io/#KotatsuApp/kotatsu-parsers) ![Kotlin](https://img.shields.io/github/languages/top/KotatsuApp/kotatsu-parsers) ![License](https://img.shields.io/github/license/KotatsuApp/Kotatsu) [![Discord](https://img.shields.io/discord/898363402467045416?color=5865f2&label=discord)](https://discord.gg/NNJ5RgVBC5) ## Usage -1. Add it in your root build.gradle at the end of repositories: +1. Add it to your root build.gradle at the end of repositories: ```groovy allprojects { @@ -35,7 +35,7 @@ Library that provides manga sources. } ``` - See for versions at [JitPack](https://jitpack.io/#KotatsuApp/kotatsu-parsers) + Versions are available on [JitPack](https://jitpack.io/#KotatsuApp/kotatsu-parsers) 3. Usage in code @@ -43,10 +43,10 @@ Library that provides manga sources. val parser = MangaSource.MANGADEX.newParser(mangaLoaderContext) ``` - `mangaLoaderContext` is an implementation of the `MangaLoaderContext` class. - See [Android](https://github.com/KotatsuApp/Kotatsu/blob/devel/app/src/main/java/org/koitharu/kotatsu/core/parser/MangaLoaderContextImpl.kt) - and [Non-Android](https://github.com/KotatsuApp/kotatsu-dl/blob/master/src/main/kotlin/org/koitharu/kotatsu_dl/env/MangaLoaderContextImpl.kt) - implementation examples. + `mangaLoaderContext` is an implementation of the MangaLoaderContext class. + See examples of [Android](https://github.com/KotatsuApp/Kotatsu/blob/devel/app/src/main/java/org/koitharu/kotatsu/core/parser/MangaLoaderContextImpl.kt) + and [Non-Android](https://github.com/KotatsuApp/kotatsu-dl/blob/master/src/main/kotlin/org/koitharu/kotatsu_dl/env/MangaLoaderContextImpl.kt) + implementation. Note that the `MangaSource.LOCAL` and `MangaSource.DUMMY` parsers cannot be instantiated. @@ -56,5 +56,4 @@ See [CONTRIBUTING.md](./CONTRIBUTING.md) for the guidelines. ## DMCA disclaimer -The developers of this application does not have any affiliation with the content available in the app. It is collecting -from the sources freely available through any web browser. \ No newline at end of file +The developers of this application have no affiliation with the content available in the app. It is collected from sources freely available through any web browser. From 38efd7935a2c9b1dec559894029e17c100773515 Mon Sep 17 00:00:00 2001 From: Saksham Shekher <95137948+OshekharO@users.noreply.github.com> Date: Sat, 18 Feb 2023 07:35:24 +0530 Subject: [PATCH 075/121] BatoTo: New Subdomain --- .../kotlin/org/koitharu/kotatsu/parsers/site/BatoToParser.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BatoToParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BatoToParser.kt index afbcfbb2..0f5e3603 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BatoToParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BatoToParser.kt @@ -35,7 +35,7 @@ internal class BatoToParser(context: MangaLoaderContext) : PagedMangaParser( override val configKeyDomain = ConfigKey.Domain( "bato.to", - arrayOf("bato.to", "mto.to", "mangatoto.com", "battwo.com", "batotwo.com", "comiko.net", "batotoo.com"), + arrayOf("bato.to", "mto.to", "hto.to", "mangatoto.com", "battwo.com", "batotwo.com", "comiko.net", "batotoo.com"), ) override suspend fun getListPage( From fa78d5dd6faa2087ce2dd974b1c105b4e9ce9142 Mon Sep 17 00:00:00 2001 From: vianh Date: Tue, 21 Feb 2023 00:47:53 +0700 Subject: [PATCH 076/121] [TruyentranhLH]: New source --- .../parsers/site/TruyentranhLHParser.kt | 154 ++++++++++++++++++ 1 file changed, 154 insertions(+) create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/TruyentranhLHParser.kt diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/TruyentranhLHParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/TruyentranhLHParser.kt new file mode 100644 index 00000000..715c3a1a --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/TruyentranhLHParser.kt @@ -0,0 +1,154 @@ +package org.koitharu.kotatsu.parsers.site + +import androidx.collection.ArrayMap +import androidx.collection.ArraySet +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.PagedMangaParser +import org.koitharu.kotatsu.parsers.config.ConfigKey +import org.koitharu.kotatsu.parsers.model.* +import org.koitharu.kotatsu.parsers.util.* +import java.text.SimpleDateFormat +import java.util.* + +@MangaSourceParser("TRUYENTRANHLH", "TruyentranhLH", "vi") +class TruyentranhLHParser(context: MangaLoaderContext) : + PagedMangaParser(context, source = MangaSource.TRUYENTRANHLH, pageSize = 18) { + + override val configKeyDomain: ConfigKey.Domain = ConfigKey.Domain("truyentranhlh.net", null) + override val sortOrders: Set = EnumSet.allOf(SortOrder::class.java) + + private val mutex = Mutex() + private var tagCache: Map? = null + + override suspend fun getDetails(manga: Manga): Manga { + val docs = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val infoHeaderEl = docs.selectFirst("main.section-body") + val infoEl = docs.selectFirst("main.section-body .series-information") + val tags = infoEl?.select(".info-item:contains(Thể loại) > .info-value > a")?.mapNotNullToSet { + getOrCreateTagMap()[it.text().trim()] + } + val state = when (infoEl?.selectFirst(".info-item:contains(Tình trạng) > .info-value")?.text()) { + "Đang tiến hành" -> MangaState.ONGOING + "Đã hoàn thành" -> MangaState.FINISHED + else -> null + } + val rating = infoHeaderEl?.let { + val like = it.selectFirst("#like .block.feature-name")?.text()?.toIntOrNull() + val disLike = it.selectFirst("#dislike .block.feature-name")?.text()?.toIntOrNull() + when { + like == null || disLike == null -> RATING_UNKNOWN + like == 0 && disLike == 0 -> RATING_UNKNOWN + else -> like.toFloat() / (like + disLike) + } + } + val chapterDateFormat = SimpleDateFormat("dd/MM/yyyy", Locale.US) + + return manga.copy( + altTitle = infoEl?.selectFirst(".info-item:contains(Tên khác) > .info-value")?.text(), + author = infoEl?.select(".info-item:contains(Tác giả) > .info-value")?.joinToString { it.text() }, + tags = tags ?: emptySet(), + state = state, + rating = rating ?: RATING_UNKNOWN, + description = infoHeaderEl?.selectFirst(".series-summary .summary-content")?.html(), + chapters = docs.select("ul.list-chapters.at-series > a").reversed().mapChapters { index, element -> + MangaChapter( + id = generateUid(element.attrAsRelativeUrl("href")), + name = element.selectFirst(".chapter-name")?.text()?.trim().orEmpty(), + number = index + 1, + url = element.attrAsRelativeUrl("href"), + scanlator = null, + uploadDate = chapterDateFormat.tryParse(element.selectFirst(".chapter-time")?.text()), + branch = null, + source = source, + ) + }, + ) + } + + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + val sortQuery = when (sortOrder) { + SortOrder.UPDATED -> "update" + SortOrder.NEWEST -> "new" + SortOrder.RATING -> "like" + SortOrder.POPULARITY -> "top" + SortOrder.ALPHABETICAL -> "az" + } + val url = urlBuilder().apply { + addPathSegment("tim-kiem") + addQueryParameter("sort", sortQuery) + addQueryParameter("page", page.toString()) + if (!query.isNullOrEmpty()) { + addQueryParameter("q", query) + } + if (!tags.isNullOrEmpty()) { + val tagsQuery = tags.joinToString(separator = ",") { it.key } + addEncodedQueryParameter("accept_genres", tagsQuery) + } + }.build() + + return webClient.httpGet(url).parseHtml() + .select(".container .card.card-dark .row > .thumb-item-flow") + .mapNotNull { + val a = it.selectFirstOrThrow(".thumb-wrapper a") + Manga( + id = generateUid(a.attrAsRelativeUrl("href")), + url = a.attrAsRelativeUrl("href"), + publicUrl = a.attrAsAbsoluteUrl("href"), + title = it.select(".thumb_attr.series-title").text(), + altTitle = null, + rating = RATING_UNKNOWN, + isNsfw = false, + coverUrl = a.selectFirst("div[data-bg]")?.attrAsAbsoluteUrl("data-bg").orEmpty(), + tags = emptySet(), + state = null, + author = null, + source = source, + ) + } + } + + override suspend fun getPages(chapter: MangaChapter): List { + val url = chapter.url.toAbsoluteUrl(domain) + return webClient.httpGet(url).parseHtml().select("#chapter-content > img").mapNotNull { + val imageUrl = it.attrAsRelativeUrlOrNull("data-src") + ?: it.attrAsRelativeUrlOrNull("src") + ?: return@mapNotNull null + MangaPage( + id = generateUid(imageUrl), + url = imageUrl, + referer = url, + preview = null, + source = source, + ) + } + } + + override suspend fun getTags(): Set { + return ArraySet(getOrCreateTagMap().values) + } + + private suspend fun getOrCreateTagMap(): Map = mutex.withLock { + tagCache?.let { return it } + val docs = webClient.httpGet("/tim-kiem".toAbsoluteUrl(domain)).parseHtml() + val tags = docs.select(".search-border-left .row > .search-gerne_item").mapNotNull { + MangaTag( + title = it.text().trim(), + key = it.selectFirst("label[data-genre-id]") + ?.attr("data-genre-id") + ?.trim() ?: return@mapNotNull null, + source = source, + ) + } + val tagMap = tags.associateByTo(ArrayMap(tags.size)) { it.title } + tagCache = tagMap + return tagMap + } +} From eea87d86074984226d5f9e144ab51dd5281da7c1 Mon Sep 17 00:00:00 2001 From: Koitharu Date: Sat, 25 Feb 2023 16:43:06 +0200 Subject: [PATCH 077/121] Fix grouple pages parsing #156 --- .../kotatsu/parsers/config/ConfigKey.kt | 6 +- .../parsers/site/grouple/GroupleParser.kt | 46 ++- .../kotatsu/parsers/MangaLoaderContextMock.kt | 90 +++-- .../kotatsu/parsers/MangaParserTest.kt | 363 +++++++++--------- 4 files changed, 258 insertions(+), 247 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/config/ConfigKey.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/config/ConfigKey.kt index 3ff6e276..8603ac62 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/config/ConfigKey.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/config/ConfigKey.kt @@ -14,4 +14,8 @@ sealed class ConfigKey( class ShowSuspiciousContent( override val defaultValue: Boolean, ) : ConfigKey("show_suspicious") -} \ No newline at end of file + + class UserAgent( + override val defaultValue: String, + ) : ConfigKey("user_agent") +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt index b98fd807..e99784af 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt @@ -5,10 +5,12 @@ import kotlinx.coroutines.coroutineScope import okhttp3.Headers import okhttp3.HttpUrl.Companion.toHttpUrl import okhttp3.Response +import okhttp3.internal.headersContentLength import org.json.JSONArray import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaParser import org.koitharu.kotatsu.parsers.MangaParserAuthProvider +import org.koitharu.kotatsu.parsers.config.ConfigKey import org.koitharu.kotatsu.parsers.exception.AuthRequiredException import org.koitharu.kotatsu.parsers.exception.ParseException import org.koitharu.kotatsu.parsers.model.* @@ -21,6 +23,7 @@ private const val PAGE_SIZE = 70 private const val PAGE_SIZE_SEARCH = 50 private const val NSFW_ALERT = "сексуальные сцены" private const val NOTHING_FOUND = "Ничего не найдено" +private const val MIN_IMAGE_SIZE = 1024L internal abstract class GroupleParser( context: MangaLoaderContext, @@ -31,12 +34,14 @@ internal abstract class GroupleParser( @Volatile private var cachedPagesServer: String? = null - override val headers = Headers.Builder() - .add( - "User-Agent", - "Mozilla/5.0 (X11; U; UNICOS lcLinux; en-US) Gecko/20140730 (KHTML, like Gecko, Safari/419.3) Arora/0.8.0", - ) - .build() + private val userAgentKey = ConfigKey.UserAgent( + "Mozilla/5.0 (X11; U; UNICOS lcLinux; en-US) Gecko/20140730 (KHTML, like Gecko, Safari/419.3) Arora/0.8.0", + ) + + override val headers: Headers + get() = Headers.Builder() + .add("User-Agent", config[userAgentKey]) + .build() override val sortOrders: Set = EnumSet.of( SortOrder.UPDATED, @@ -237,15 +242,23 @@ internal abstract class GroupleParser( if (cachedServer != null && cachedServer in servers && tryHead(cachedServer + path)) { return cachedServer + path } - val server = coroutineScope { - servers.map { server -> - async { - if (tryHead(server + path)) server else null - } - }.awaitFirst { it != null } + if (servers.isEmpty()) { + throw ParseException("No servers found for page", page.url) + } + val server = try { + coroutineScope { + servers.map { server -> + async { + if (tryHead(server + path)) server else null + } + }.awaitFirst { it != null } + }.also { + cachedPagesServer = it + } + } catch (e: NoSuchElementException) { + servers.random() } - cachedPagesServer = server - return checkNotNull(server + path) + return checkNotNull(server) + path } override suspend fun getTags(): Set { @@ -316,8 +329,9 @@ internal abstract class GroupleParser( return webClient.httpPost(url, payload) } - private suspend fun tryHead(url: String): Boolean = runCatchingCancellable { - webClient.httpHead(url).isSuccessful + suspend fun tryHead(url: String): Boolean = runCatchingCancellable { + val response = webClient.httpHead(url) + response.isSuccessful && response.headersContentLength() >= MIN_IMAGE_SIZE }.getOrDefault(false) private fun Response.checkAuthRequired(): Response { diff --git a/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaLoaderContextMock.kt b/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaLoaderContextMock.kt index 2c80cb0f..6a01b4cc 100644 --- a/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaLoaderContextMock.kt +++ b/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaLoaderContextMock.kt @@ -1,7 +1,6 @@ package org.koitharu.kotatsu.parsers import com.koushikdutta.quack.QuackContext -import okhttp3.Headers import okhttp3.OkHttpClient import okhttp3.Request import okhttp3.Response @@ -12,57 +11,54 @@ import java.util.concurrent.TimeUnit internal object MangaLoaderContextMock : MangaLoaderContext() { - private val userAgent = "Kotatsu/%s (Android %s; %s; %s %s; %s)".format( - /*BuildConfig.VERSION_NAME*/ "4.3", - /*Build.VERSION.RELEASE*/ "r", - /*Build.MODEL*/ "", - /*Build.BRAND*/ "", - /*Build.DEVICE*/ "", - /*Locale.getDefault().language*/ "en", - ) + private val userAgent = "Kotatsu/%s (Android %s; %s; %s %s; %s)".format( + /*BuildConfig.VERSION_NAME*/ "4.4", + /*Build.VERSION.RELEASE*/ "r", + /*Build.MODEL*/ "", + /*Build.BRAND*/ "", + /*Build.DEVICE*/ "", + /*Locale.getDefault().language*/ "en", + ) - override val cookieJar = InMemoryCookieJar() + override val cookieJar = InMemoryCookieJar() - override val httpClient: OkHttpClient = OkHttpClient.Builder() - .cookieJar(cookieJar) - .addInterceptor(CommonHeadersInterceptor(userAgent)) - .addInterceptor(CloudFlareInterceptor()) - .connectTimeout(20, TimeUnit.SECONDS) - .readTimeout(60, TimeUnit.SECONDS) - .writeTimeout(20, TimeUnit.SECONDS) - .build() + override val httpClient: OkHttpClient = OkHttpClient.Builder() + .cookieJar(cookieJar) + .addInterceptor(CommonHeadersInterceptor(userAgent)) + .addInterceptor(CloudFlareInterceptor()) + .connectTimeout(20, TimeUnit.SECONDS) + .readTimeout(60, TimeUnit.SECONDS) + .writeTimeout(20, TimeUnit.SECONDS) + .build() - init { - loadTestCookies() - } + init { + loadTestCookies() + } - override suspend fun evaluateJs(script: String): String? { - return QuackContext.create().use { - it.evaluate(script)?.toString() - } - } + override suspend fun evaluateJs(script: String): String? { + return QuackContext.create().use { + it.evaluate(script)?.toString() + } + } - override fun getConfig(source: MangaSource): MangaSourceConfig { - return SourceConfigMock() - } + override fun getConfig(source: MangaSource): MangaSourceConfig { + return SourceConfigMock() + } - suspend fun doRequest(url: String, referer: String? = null, extraHeaders: Headers? = null): Response { - val request = Request.Builder() - .get() - .url(url) - if (extraHeaders != null) { - request.headers(extraHeaders) - } - if (referer != null) { - request.header("Referer", referer) - } - return httpClient.newCall(request.build()).await() - } + suspend fun doRequest(url: String, source: MangaSource?): Response { + val request = Request.Builder() + .get() + .url(url) + if (source != null) { + request.tag(MangaSource::class.java, source) + } + return httpClient.newCall(request.build()).await() + } - private fun loadTestCookies() { - // https://addons.mozilla.org/ru/firefox/addon/cookies-txt/ - javaClass.getResourceAsStream("/cookies.txt")?.use { - cookieJar.loadFromStream(it) - } ?: println("No cookies loaded!") - } + private fun loadTestCookies() { + // https://addons.mozilla.org/ru/firefox/addon/cookies-txt/ + javaClass.getResourceAsStream("/cookies.txt")?.use { + cookieJar.loadFromStream(it) + } ?: println("No cookies loaded!") + } } diff --git a/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaParserTest.kt b/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaParserTest.kt index ab450369..337b484e 100644 --- a/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaParserTest.kt +++ b/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaParserTest.kt @@ -11,193 +11,190 @@ import org.koitharu.kotatsu.parsers.model.SortOrder import org.koitharu.kotatsu.parsers.util.domain import org.koitharu.kotatsu.parsers.util.medianOrNull import org.koitharu.kotatsu.parsers.util.mimeType -import org.koitharu.kotatsu.test_util.isDistinct -import org.koitharu.kotatsu.test_util.isDistinctBy -import org.koitharu.kotatsu.test_util.isUrlAbsolute -import org.koitharu.kotatsu.test_util.maxDuplicates +import org.koitharu.kotatsu.test_util.* @ExtendWith(AuthCheckExtension::class) internal class MangaParserTest { - private val context = MangaLoaderContextMock - - @ParameterizedTest(name = "{index}|list|{0}") - @MangaSources - fun list(source: MangaSource) = runTest { - val parser = source.newParser(context) - val list = parser.getList(20, sortOrder = SortOrder.POPULARITY, tags = null) - checkMangaList(list, "list") - assert(list.all { it.source == source }) - } - - @ParameterizedTest(name = "{index}|pagination|{0}") - @MangaSources - fun pagination(source: MangaSource) = runTest { - val parser = source.newParser(context) - val page1 = parser.getList(0, sortOrder = null, tags = null) - val page2 = parser.getList(page1.size, sortOrder = null, tags = null) - assert(page1.isNotEmpty()) { "Page 1 is empty" } - assert(page2.isNotEmpty()) { "Page 2 is empty" } - val intersection = page1.intersect(page2.toSet()) - assert(intersection.isEmpty()) { - "Pages are intersected by " + intersection.size - } - } - - @ParameterizedTest(name = "{index}|search|{0}") - @MangaSources - fun search(source: MangaSource) = runTest { - val parser = source.newParser(context) - val subject = parser.getList(20, sortOrder = SortOrder.POPULARITY, tags = null).minByOrNull { - it.title.length - } ?: error("No manga found") - val query = subject.title - check(query.isNotBlank()) { "Manga title '$query' is blank" } - val list = parser.getList(0, query) - assert(list.isNotEmpty()) { "Empty search results by \"$query\"" } - assert(list.singleOrNull { it.url == subject.url && it.id == subject.id } != null) { - "Single subject '${subject.title} (${subject.publicUrl})' not found in search results" - } - checkMangaList(list, "search('$query')") - assert(list.all { it.source == source }) - } - - @ParameterizedTest(name = "{index}|tags|{0}") - @MangaSources - fun tags(source: MangaSource) = runTest { - val parser = source.newParser(context) - val tags = parser.getTags() - assert(tags.isNotEmpty()) { "No tags found" } - val keys = tags.map { it.key } - assert(keys.isDistinct()) - assert("" !in keys) - val titles = tags.map { it.title } + private val context = MangaLoaderContextMock + + @ParameterizedTest(name = "{index}|list|{0}") + @MangaSources + fun list(source: MangaSource) = runTest { + val parser = source.newParser(context) + val list = parser.getList(20, sortOrder = SortOrder.POPULARITY, tags = null) + checkMangaList(list, "list") + assert(list.all { it.source == source }) + } + + @ParameterizedTest(name = "{index}|pagination|{0}") + @MangaSources + fun pagination(source: MangaSource) = runTest { + val parser = source.newParser(context) + val page1 = parser.getList(0, sortOrder = null, tags = null) + val page2 = parser.getList(page1.size, sortOrder = null, tags = null) + assert(page1.isNotEmpty()) { "Page 1 is empty" } + assert(page2.isNotEmpty()) { "Page 2 is empty" } + val intersection = page1.intersect(page2.toSet()) + assert(intersection.isEmpty()) { + "Pages are intersected by " + intersection.size + } + } + + @ParameterizedTest(name = "{index}|search|{0}") + @MangaSources + fun search(source: MangaSource) = runTest { + val parser = source.newParser(context) + val subject = parser.getList(20, sortOrder = SortOrder.POPULARITY, tags = null).minByOrNull { + it.title.length + } ?: error("No manga found") + val query = subject.title + check(query.isNotBlank()) { "Manga title '$query' is blank" } + val list = parser.getList(0, query) + assert(list.isNotEmpty()) { "Empty search results by \"$query\"" } + assert(list.singleOrNull { it.url == subject.url && it.id == subject.id } != null) { + "Single subject '${subject.title} (${subject.publicUrl})' not found in search results" + } + checkMangaList(list, "search('$query')") + assert(list.all { it.source == source }) + } + + @ParameterizedTest(name = "{index}|tags|{0}") + @MangaSources + fun tags(source: MangaSource) = runTest { + val parser = source.newParser(context) + val tags = parser.getTags() + assert(tags.isNotEmpty()) { "No tags found" } + val keys = tags.map { it.key } + assert(keys.isDistinct()) + assert("" !in keys) + val titles = tags.map { it.title } // assert(titles.isDistinct()) - assert("" !in titles) - assert(tags.all { it.source == source }) - - val tag = tags.last() - val list = parser.getList(offset = 0, tags = setOf(tag), sortOrder = null) - checkMangaList(list, "${tag.title} (${tag.key})") - assert(list.all { it.source == source }) - } - - @ParameterizedTest(name = "{index}|details|{0}") - @MangaSources - fun details(source: MangaSource) = runTest { - val parser = source.newParser(context) - val list = parser.getList(20, sortOrder = SortOrder.POPULARITY, tags = null) - val manga = list[3] - parser.getDetails(manga).apply { - assert(!chapters.isNullOrEmpty()) { "Chapters are null or empty" } - assert(publicUrl.isUrlAbsolute()) { "Manga public url is not absolute: '$publicUrl'" } - assert(description != null) { "Detailed description is null: '$publicUrl'" } - assert(title.startsWith(manga.title)) { - "Titles are mismatch: '$title' and '${manga.title}' for $publicUrl" - } - assert(this.source == source) - val c = checkNotNull(chapters) - assert(c.isDistinctBy { it.id }) { - "Chapters are not distinct by id: ${c.maxDuplicates { it.id }} for $publicUrl" - } - assert(c.isDistinctBy { it.number to it.branch }) { - "Chapters are not distinct by number: ${c.maxDuplicates { it.number to it.branch }} for $publicUrl" - } - assert(c.all { it.source == source }) - checkImageRequest(coverUrl, publicUrl) - largeCoverUrl?.let { - checkImageRequest(it, publicUrl) - } - } - } - - @ParameterizedTest(name = "{index}|pages|{0}") - @MangaSources - fun pages(source: MangaSource) = runTest { - val parser = source.newParser(context) - val list = parser.getList(20, sortOrder = SortOrder.POPULARITY, tags = null) - val manga = list.first() - val chapter = parser.getDetails(manga).chapters?.firstOrNull() ?: error("Chapter is null") - val pages = parser.getPages(chapter) - - assert(pages.isNotEmpty()) - assert(pages.isDistinctBy { it.id }) - assert(pages.all { it.source == source }) - - val page = pages.medianOrNull() ?: error("No page") - val pageUrl = parser.getPageUrl(page) - assert(pageUrl.isNotEmpty()) - assert(pageUrl.isUrlAbsolute()) - checkImageRequest(pageUrl, page.referer) - } - - @ParameterizedTest(name = "{index}|favicon|{0}") - @MangaSources - fun favicon(source: MangaSource) = runTest { - val parser = source.newParser(context) - val favicons = parser.getFavicons() - val types = setOf("png", "svg", "ico", "gif", "jpg", "jpeg") - assert(favicons.isNotEmpty()) - favicons.forEach { - assert(it.url.isUrlAbsolute()) { "Favicon url is not absolute: ${it.url}" } - assert(it.type in types) { "Unknown icon type: ${it.type}" } - } - val favicon = favicons.find(24) - checkNotNull(favicon) - checkImageRequest(favicon.url, favicons.referer) - } - - @ParameterizedTest(name = "{index}|domain|{0}") - @MangaSources - fun domain(source: MangaSource) = runTest { - val parser = source.newParser(context) - val defaultDomain = parser.domain - val url = HttpUrl.Builder() - .host(defaultDomain) - .scheme("https") - .toString() - val response = context.doRequest(url, extraHeaders = parser.headers) - val realUrl = response.request.url - val realDomain = realUrl.topPrivateDomain() - val realHost = realUrl.host - assert(defaultDomain == realHost || defaultDomain == realDomain) { - "Domain mismatch:\nRequired:\t\t\t$defaultDomain\nActual:\t\t\t$realDomain\nHost:\t\t\t$realHost" - } - } - - @ParameterizedTest(name = "{index}|authorization|{0}") - @MangaSources - @Disabled - fun authorization(source: MangaSource) = runTest { - val parser = source.newParser(context) - if (parser is MangaParserAuthProvider) { - val username = parser.getUsername() - assert(username.isNotBlank()) { "Username is blank" } - println("Signed in to ${source.name} as $username") - } - } - - private suspend fun checkMangaList(list: List, cause: String) { - assert(list.isNotEmpty()) { "Manga list for '$cause' is empty" } - assert(list.isDistinctBy { it.id }) { "Manga list for '$cause' contains duplicated ids" } - for (item in list) { - assert(item.url.isNotEmpty()) { "Url is empty" } - assert(!item.url.isUrlAbsolute()) { "Url looks like absolute: ${item.url}" } - assert(item.coverUrl.isUrlAbsolute()) { "Cover url is not absolute: ${item.coverUrl}" } - assert(item.title.isNotEmpty()) { "Title for ${item.publicUrl} is empty" } - assert(item.publicUrl.isUrlAbsolute()) - } - val testItem = list.random() - checkImageRequest(testItem.coverUrl, testItem.publicUrl) - } - - private suspend fun checkImageRequest(url: String, referer: String?) { - context.doRequest(url, referer).use { - assert(it.isSuccessful) { "Request failed: ${it.code}(${it.message}): $url" } - assert(it.mimeType?.startsWith("image/") == true) { - "Wrong response mime type: ${it.mimeType}" - } - } - } + assert("" !in titles) + assert(tags.all { it.source == source }) + + val tag = tags.last() + val list = parser.getList(offset = 0, tags = setOf(tag), sortOrder = null) + checkMangaList(list, "${tag.title} (${tag.key})") + assert(list.all { it.source == source }) + } + + @ParameterizedTest(name = "{index}|details|{0}") + @MangaSources + fun details(source: MangaSource) = runTest { + val parser = source.newParser(context) + val list = parser.getList(20, sortOrder = SortOrder.POPULARITY, tags = null) + val manga = list[3] + parser.getDetails(manga).apply { + assert(!chapters.isNullOrEmpty()) { "Chapters are null or empty" } + assert(publicUrl.isUrlAbsolute()) { "Manga public url is not absolute: '$publicUrl'" } + assert(description != null) { "Detailed description is null: '$publicUrl'" } + assert(title.startsWith(manga.title)) { + "Titles are mismatch: '$title' and '${manga.title}' for $publicUrl" + } + assert(this.source == source) + val c = checkNotNull(chapters) + assert(c.isDistinctBy { it.id }) { + "Chapters are not distinct by id: ${c.maxDuplicates { it.id }} for $publicUrl" + } + assert(c.isDistinctBy { it.number to it.branch }) { + "Chapters are not distinct by number: ${c.maxDuplicates { it.number to it.branch }} for $publicUrl" + } + assert(c.all { it.source == source }) + checkImageRequest(coverUrl, source) + largeCoverUrl?.let { + checkImageRequest(it, source) + } + } + } + + @ParameterizedTest(name = "{index}|pages|{0}") + @MangaSources + fun pages(source: MangaSource) = runTest { + val parser = source.newParser(context) + val list = parser.getList(20, sortOrder = SortOrder.POPULARITY, tags = null) + val manga = list.first() + val chapter = parser.getDetails(manga).chapters?.firstOrNull() ?: error("Chapter is null") + val pages = parser.getPages(chapter) + + assert(pages.isNotEmpty()) + assert(pages.isDistinctBy { it.id }) + assert(pages.all { it.source == source }) + + val page = pages.medianOrNull() ?: error("No page") + val pageUrl = parser.getPageUrl(page) + assert(pageUrl.isNotEmpty()) + assert(pageUrl.isUrlAbsolute()) + checkImageRequest(pageUrl, page.source) + } + + @ParameterizedTest(name = "{index}|favicon|{0}") + @MangaSources + fun favicon(source: MangaSource) = runTest { + val parser = source.newParser(context) + val favicons = parser.getFavicons() + val types = setOf("png", "svg", "ico", "gif", "jpg", "jpeg") + assert(favicons.isNotEmpty()) + favicons.forEach { + assert(it.url.isUrlAbsolute()) { "Favicon url is not absolute: ${it.url}" } + assert(it.type in types) { "Unknown icon type: ${it.type}" } + } + val favicon = favicons.find(24) + checkNotNull(favicon) + checkImageRequest(favicon.url, source) + } + + @ParameterizedTest(name = "{index}|domain|{0}") + @MangaSources + fun domain(source: MangaSource) = runTest { + val parser = source.newParser(context) + val defaultDomain = parser.domain + val url = HttpUrl.Builder() + .host(defaultDomain) + .scheme("https") + .toString() + val response = context.doRequest(url, source) + val realUrl = response.request.url + val realDomain = realUrl.topPrivateDomain() + val realHost = realUrl.host + assert(defaultDomain == realHost || defaultDomain == realDomain) { + "Domain mismatch:\nRequired:\t\t\t$defaultDomain\nActual:\t\t\t$realDomain\nHost:\t\t\t$realHost" + } + } + + @ParameterizedTest(name = "{index}|authorization|{0}") + @MangaSources + @Disabled + fun authorization(source: MangaSource) = runTest { + val parser = source.newParser(context) + if (parser is MangaParserAuthProvider) { + val username = parser.getUsername() + assert(username.isNotBlank()) { "Username is blank" } + println("Signed in to ${source.name} as $username") + } + } + + private suspend fun checkMangaList(list: List, cause: String) { + assert(list.isNotEmpty()) { "Manga list for '$cause' is empty" } + assert(list.isDistinctBy { it.id }) { "Manga list for '$cause' contains duplicated ids" } + for (item in list) { + assert(item.url.isNotEmpty()) { "Url is empty" } + assert(!item.url.isUrlAbsolute()) { "Url looks like absolute: ${item.url}" } + assert(item.coverUrl.isUrlAbsolute()) { "Cover url is not absolute: ${item.coverUrl}" } + assert(item.title.isNotEmpty()) { "Title for ${item.publicUrl} is empty" } + assert(item.publicUrl.isUrlAbsolute()) + } + val testItem = list.random() + checkImageRequest(testItem.coverUrl, testItem.source) + } + + private suspend fun checkImageRequest(url: String, source: MangaSource) { + context.doRequest(url, source).use { + assert(it.isSuccessful) { "Request failed: ${it.code}(${it.message}): $url" } + assert(it.mimeType?.startsWith("image/") == true) { + "Wrong response mime type: ${it.mimeType}" + } + } + } } From f62d0243425a8cb2b2b78b6d5b88bc3b81355c85 Mon Sep 17 00:00:00 2001 From: Koitharu Date: Sat, 25 Feb 2023 17:10:41 +0200 Subject: [PATCH 078/121] Fix Manga District & Hentai4free parsers #157 --- .../parsers/site/madara/Hentai4Free.kt | 69 ++ .../parsers/site/madara/MadaraParser.kt | 857 +++++++++--------- .../parsers/site/madara/MangaDistrict.kt | 49 + 3 files changed, 524 insertions(+), 451 deletions(-) create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Hentai4Free.kt create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangaDistrict.kt diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Hentai4Free.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Hentai4Free.kt new file mode 100644 index 00000000..bec398fb --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Hentai4Free.kt @@ -0,0 +1,69 @@ +package org.koitharu.kotatsu.parsers.site.madara + +import org.jsoup.nodes.Document +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.model.Manga +import org.koitharu.kotatsu.parsers.model.MangaChapter +import org.koitharu.kotatsu.parsers.model.MangaSource +import org.koitharu.kotatsu.parsers.model.MangaTag +import org.koitharu.kotatsu.parsers.util.* +import java.text.SimpleDateFormat +import java.util.* + +@MangaSourceParser("HENTAI_4FREE", "Hentai4Free", "en") +internal class Hentai4Free(context: MangaLoaderContext) : + MadaraParser(context, MangaSource.HENTAI_4FREE, "hentai4free.net") { + + override val tagPrefix = "hentai-tag/" + + override val isNsfwSource = true + + override suspend fun getTags(): Set { + val doc = webClient.httpGet("https://$domain/").parseHtml() + val body = doc.body() + val root1 = body.selectFirst("header")?.selectFirst("ul.second-menu") + val list = root1?.select("li").orEmpty() + val keySet = HashSet(list.size) + return list.mapNotNullToSet { li -> + val a = li.selectFirst("a") ?: return@mapNotNullToSet null + val href = a.attr("href").removeSuffix("/") + .substringAfterLast(tagPrefix, "") + if (href.isEmpty() || !keySet.add(href)) { + return@mapNotNullToSet null + } + MangaTag( + key = href, + title = a.ownText().trim().toTitleCase(), + source = source, + ) + } + } + + override suspend fun getChapters(manga: Manga, doc: Document): List { + val slug = manga.url.removeSuffix('/').substringAfterLast('/') + val doc2 = webClient.httpPost( + "https://$domain/hentai/$slug/ajax/chapters/", + mapOf(), + ).parseHtml() + val ul = doc2.body().selectFirstOrThrow("ul") + val dateFormat = SimpleDateFormat(datePattern, Locale.US) + return ul.select("li").asReversed().mapChapters { i, li -> + val a = li.selectFirst("a") + val href = a?.attrAsRelativeUrlOrNull("href") ?: li.parseFailed("Link is missing") + MangaChapter( + id = generateUid(href), + name = a.ownText(), + number = i + 1, + url = href, + uploadDate = parseChapterDate( + dateFormat, + li.selectFirst("span.chapter-release-date i")?.text(), + ), + source = source, + scanlator = null, + branch = null, + ) + } + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt index f6c6dac7..8cd86ff0 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt @@ -16,498 +16,453 @@ import java.text.SimpleDateFormat import java.util.* internal abstract class MadaraParser( - context: MangaLoaderContext, - source: MangaSource, - domain: String, + context: MangaLoaderContext, + source: MangaSource, + domain: String, ) : PagedMangaParser(context, source, pageSize = 12) { - override val configKeyDomain = ConfigKey.Domain(domain, null) - - override val sortOrders: Set = EnumSet.of( - SortOrder.UPDATED, - SortOrder.POPULARITY, - ) - - protected open val tagPrefix = "manga-genre/" - protected open val isNsfwSource = false - protected open val datePattern = "MMMM dd, yyyy" - - init { - paginator.firstPage = 0 - searchPaginator.firstPage = 0 - } - - override suspend fun getListPage( - page: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - val tag = tags.oneOrThrowIfMany() - val payload = createRequestTemplate() - payload["page"] = page.toString() - payload["vars[meta_key]"] = when (sortOrder) { - SortOrder.POPULARITY -> "_wp_manga_views" - SortOrder.UPDATED -> "_latest_update" - else -> "_wp_manga_views" - } - payload["vars[wp-manga-genre]"] = tag?.key.orEmpty() - payload["vars[s]"] = query?.urlEncoded().orEmpty() - val doc = webClient.httpPost( - "https://$domain/wp-admin/admin-ajax.php", - payload, - ).parseHtml() - return doc.select("div.row.c-tabs-item__content").ifEmpty { - doc.select("div.page-item-detail.manga") - }.map { div -> - val href = div.selectFirst("a")?.attrAsRelativeUrlOrNull("href") - ?: div.parseFailed("Link not found") - val summary = div.selectFirst(".tab-summary") ?: div.selectFirst(".item-summary") - Manga( - id = generateUid(href), - url = href, - publicUrl = href.toAbsoluteUrl(div.host ?: domain), - coverUrl = div.selectFirst("img")?.src().orEmpty(), - title = (summary?.selectFirst("h3") ?: summary?.selectFirst("h4"))?.text().orEmpty(), - altTitle = null, - rating = div.selectFirst("span.total_votes")?.ownText() - ?.toFloatOrNull()?.div(5f) ?: -1f, - tags = summary?.selectFirst(".mg_genres")?.select("a")?.mapToSet { a -> - MangaTag( - key = a.attr("href").removeSuffix("/").substringAfterLast('/'), - title = a.text().toTitleCase(), - source = source, - ) - }.orEmpty(), - author = summary?.selectFirst(".mg_author")?.selectFirst("a")?.ownText(), - state = when ( - summary?.selectFirst(".mg_status")?.selectFirst(".summary-content") - ?.ownText()?.trim()?.lowercase() - ) { - "ongoing" -> MangaState.ONGOING - "completed" -> MangaState.FINISHED - else -> null - }, - source = source, - isNsfw = isNsfwSource, - ) - } - } - - override suspend fun getTags(): Set { - val doc = webClient.httpGet("https://$domain/manga/").parseHtml() - val body = doc.body() - val root1 = body.selectFirst("header")?.selectFirst("ul.second-menu") - val root2 = body.selectFirst("div.genres_wrap")?.selectFirst("ul.list-unstyled") - if (root1 == null && root2 == null) { - doc.parseFailed("Root not found") - } - val list = root1?.select("li").orEmpty() + root2?.select("li").orEmpty() - val keySet = HashSet(list.size) - return list.mapNotNullToSet { li -> - val a = li.selectFirst("a") ?: return@mapNotNullToSet null - val href = a.attr("href").removeSuffix("/") - .substringAfterLast(tagPrefix, "") - if (href.isEmpty() || !keySet.add(href)) { - return@mapNotNullToSet null - } - MangaTag( - key = href, - title = a.ownText().trim().ifEmpty { - a.selectFirst(".menu-image-title")?.text()?.trim() ?: return@mapNotNullToSet null - }.toTitleCase(), - source = source, - ) - } - } - - override suspend fun getDetails(manga: Manga): Manga = coroutineScope { - val fullUrl = manga.url.toAbsoluteUrl(domain) - val doc = webClient.httpGet(fullUrl).parseHtml() - val chaptersDeferred = async { getChapters(manga, doc) } - val root = doc.body().selectFirst("div.profile-manga") - ?.selectFirst("div.summary_content") - ?.selectFirst("div.post-content") - ?: throw ParseException("Root not found", fullUrl) - val root2 = doc.body().selectFirst("div.content-area") - ?.selectFirst("div.c-page") - ?: throw ParseException("Root2 not found", fullUrl) - manga.copy( - tags = root.selectFirst("div.genres-content")?.select("a") - ?.mapNotNullToSet { a -> - MangaTag( - key = a.attr("href").removeSuffix("/").substringAfterLast('/'), - title = a.text().toTitleCase(), - source = source, - ) - } ?: manga.tags, - description = root2.selectFirst("div.description-summary") - ?.selectFirst("div.summary__content") - ?.select("p") - ?.filterNot { it.ownText().startsWith("A brief description") } - ?.joinToString { it.html() }, - chapters = chaptersDeferred.await(), - ) - } - - protected open suspend fun getChapters(manga: Manga, doc: Document): List { - val root2 = doc.body().selectFirstOrThrow("div.content-area") - .selectFirstOrThrow("div.c-page") - val dateFormat = SimpleDateFormat(datePattern, sourceLocale ?: Locale.US) - return root2.select("li").asReversed().mapChapters { i, li -> - val a = li.selectFirst("a") - val href = a?.attrAsRelativeUrlOrNull("href") ?: li.parseFailed("Link is missing") - MangaChapter( - id = generateUid(href), - name = a.ownText(), - number = i + 1, - url = href, - uploadDate = parseChapterDate( - dateFormat, - li.selectFirst("span.chapter-release-date i")?.text(), - ), - source = source, - scanlator = null, - branch = null, - ) - } - } - - override suspend fun getPages(chapter: MangaChapter): List { - val fullUrl = chapter.url.toAbsoluteUrl(domain) - val doc = webClient.httpGet(fullUrl).parseHtml() - val root = doc.body().selectFirst("div.main-col-inner") - ?.selectFirst("div.reading-content") - ?: throw ParseException("Root not found", fullUrl) - return root.select("div.page-break").map { div -> - val img = div.selectFirst("img") ?: div.parseFailed("Page image not found") - val url = img.src()?.toRelativeUrl(domain) ?: div.parseFailed("Image src not found") - MangaPage( - id = generateUid(url), - url = url, - preview = null, - referer = fullUrl, - source = source, - ) - } - } - - protected fun parseChapterDate(dateFormat: DateFormat, date: String?): Long { - date ?: return 0 - return when { - date.endsWith(" ago", ignoreCase = true) -> { - parseRelativeDate(date) - } - // Handle translated 'ago' in Portuguese. - date.endsWith(" atrás", ignoreCase = true) -> { - parseRelativeDate(date) - } - // Handle translated 'ago' in Turkish. - date.endsWith(" önce", ignoreCase = true) -> { - parseRelativeDate(date) - } - // Handle 'yesterday' and 'today', using midnight - date.startsWith("year", ignoreCase = true) -> { - Calendar.getInstance().apply { - add(Calendar.DAY_OF_MONTH, -1) // yesterday - set(Calendar.HOUR_OF_DAY, 0) - set(Calendar.MINUTE, 0) - set(Calendar.SECOND, 0) - set(Calendar.MILLISECOND, 0) - }.timeInMillis - } - - date.startsWith("today", ignoreCase = true) -> { - Calendar.getInstance().apply { - set(Calendar.HOUR_OF_DAY, 0) - set(Calendar.MINUTE, 0) - set(Calendar.SECOND, 0) - set(Calendar.MILLISECOND, 0) - }.timeInMillis - } - - date.contains(Regex("""\d(st|nd|rd|th)""")) -> { - // Clean date (e.g. 5th December 2019 to 5 December 2019) before parsing it - date.split(" ").map { - if (it.contains(Regex("""\d\D\D"""))) { - it.replace(Regex("""\D"""), "") - } else { - it - } - } - .let { dateFormat.tryParse(it.joinToString(" ")) } - } - - else -> dateFormat.tryParse(date) - } - } - - // Parses dates in this form: - // 21 hours ago - private fun parseRelativeDate(date: String): Long { - val number = Regex("""(\d+)""").find(date)?.value?.toIntOrNull() ?: return 0 - val cal = Calendar.getInstance() - - return when { - WordSet( - "hari", - "gün", - "jour", - "día", - "dia", - "day", - ).anyWordIn(date) -> cal.apply { add(Calendar.DAY_OF_MONTH, -number) }.timeInMillis - - WordSet("jam", "saat", "heure", "hora", "hour").anyWordIn(date) -> cal.apply { - add( - Calendar.HOUR, - -number, - ) - }.timeInMillis - - WordSet("menit", "dakika", "min", "minute", "minuto").anyWordIn(date) -> cal.apply { - add( - Calendar.MINUTE, - -number, - ) - }.timeInMillis - - WordSet("detik", "segundo", "second").anyWordIn(date) -> cal.apply { - add( - Calendar.SECOND, - -number, - ) - }.timeInMillis - - WordSet("month").anyWordIn(date) -> cal.apply { add(Calendar.MONTH, -number) }.timeInMillis - WordSet("year").anyWordIn(date) -> cal.apply { add(Calendar.YEAR, -number) }.timeInMillis - else -> 0 - } - } - - private fun Element.src(): String? { - var result = absUrl("data-src") - if (result.isEmpty()) result = absUrl("data-cfsrc") - if (result.isEmpty()) result = absUrl("src") - return if (result.isEmpty()) null else result - } - - private fun createRequestTemplate() = - ( - "action=madara_load_more&page=1&template=madara-core%2Fcontent%2Fcontent-search&vars%5Bs%5D=&vars%5B" + - "orderby%5D=meta_value_num&vars%5Bpaged%5D=1&vars%5Btemplate%5D=search&vars%5Bmeta_query" + - "%5D%5B0%5D%5Brelation%5D=AND&vars%5Bmeta_query%5D%5Brelation%5D=OR&vars%5Bpost_type" + - "%5D=wp-manga&vars%5Bpost_status%5D=publish&vars%5Bmeta_key%5D=_latest_update&vars%5Border" + - "%5D=desc&vars%5Bmanga_archives_item_layout%5D=default" - ).split('&') - .map { - val pos = it.indexOf('=') - it.substring(0, pos) to it.substring(pos + 1) - }.toMutableMap() - - @MangaSourceParser("MANGAREAD", "MangaRead", "en") - class MangaRead(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGAREAD, "www.mangaread.org") { - override val tagPrefix = "genres/" - override val datePattern = "dd.MM.yyyy" - } - - @MangaSourceParser("MANGAWEEBS", "MangaWeebs", "en") - class MangaWeebs(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGAWEEBS, "mangaweebs.in") { - override val datePattern = "dd MMMM HH:mm" - } - - @MangaSourceParser("HACHIMANGA", "HachiManga", "ja") - class HachiManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.HACHIMANGA, "hachimanga.com") { - - override val sourceLocale: Locale = Locale.ENGLISH - } - - @MangaSourceParser("PIANMANGA", "PianManga", "en") - class PianManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.PIANMANGA, "pianmanga.me") - - @MangaSourceParser("MANGAROSIE", "MangaRosie", "en") - class MangaRosie(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGAROSIE, "mangarosie.in") - - @MangaSourceParser("MANGATX", "MangaTx", "en") - class MangaTx(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGATX, "mangatx.com") - - @MangaSourceParser("MANGAEFFECT", "MangaEffect", "en") - class MangaEffect(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGAEFFECT, "mangaeffect.com") { - override val datePattern = "dd.MM.yyyy" - } - - @MangaSourceParser("AQUAMANGA", "AquaManga", "en") - class AquaManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.AQUAMANGA, "aquamanga.com") { - - } - - @MangaSourceParser("MANGALEK", "MangaLek", "ar") - class MangaLek(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGALEK, "mangalek.com") { - - } - - @MangaSourceParser("HARIMANGA", "HariManga", "en") - class HariManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.HARIMANGA, "harimanga.com") { - override val datePattern = "MM/dd/yyyy" - } - - @MangaSourceParser("KISSMANGA", "KissManga", "en") - class KissManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.KISSMANGA, "kissmanga.in") - - @MangaSourceParser("MANGAROCK", "MangaRock", "en") - class MangaRock(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGAROCK, "mangarockteam.com") - - @MangaSourceParser("FREEMANGA", "FreeManga", "en") - class FreeManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.FREEMANGA, "freemanga.me") - - @MangaSourceParser("MANGA_KOMI", "MangaKomi", "en") - class MangaKomi(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGA_KOMI, "mangakomi.io") - - @MangaSourceParser("NEATMANGA", "NeatManga", "en") - class NeatManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.NEATMANGA, "neatmangas.com") - - @MangaSourceParser("MANHWACLAN", "ManhwaClan", "en") - class ManhwaClan(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANHWACLAN, "manhwaclan.com") + override val configKeyDomain = ConfigKey.Domain(domain, null) - @MangaSourceParser("MANGA_3S", "Manga3s", "en") - class Manga3s(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGA_3S, "manga3s.com") { - override val tagPrefix = "manhwa-genre/" - } + override val sortOrders: Set = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + ) - @MangaSourceParser("MANHWAKOOL", "Manhwa Kool", "en") - class ManhwaKool(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANHWAKOOL, "manhwakool.com") { - override val datePattern: String = "MM/dd" - } + protected open val tagPrefix = "manga-genre/" + protected open val isNsfwSource = false + protected open val datePattern = "MMMM dd, yyyy" - @MangaSourceParser("TOPMANHUA", "Top Manhua", "en") - class TopManhua(context: MangaLoaderContext) : MadaraParser(context, MangaSource.TOPMANHUA, "www.topmanhua.com") { - override val tagPrefix = "manhua-genre/" - override val datePattern = "MM/dd/yyyy" - } + init { + paginator.firstPage = 0 + searchPaginator.firstPage = 0 + } + + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + val tag = tags.oneOrThrowIfMany() + val payload = createRequestTemplate() + payload["page"] = page.toString() + payload["vars[meta_key]"] = when (sortOrder) { + SortOrder.POPULARITY -> "_wp_manga_views" + SortOrder.UPDATED -> "_latest_update" + else -> "_wp_manga_views" + } + payload["vars[wp-manga-genre]"] = tag?.key.orEmpty() + payload["vars[s]"] = query?.urlEncoded().orEmpty() + val doc = webClient.httpPost( + "https://$domain/wp-admin/admin-ajax.php", + payload, + ).parseHtml() + return doc.select("div.row.c-tabs-item__content").ifEmpty { + doc.select("div.page-item-detail.manga") + }.map { div -> + val href = div.selectFirst("a")?.attrAsRelativeUrlOrNull("href") + ?: div.parseFailed("Link not found") + val summary = div.selectFirst(".tab-summary") ?: div.selectFirst(".item-summary") + Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(div.host ?: domain), + coverUrl = div.selectFirst("img")?.src().orEmpty(), + title = (summary?.selectFirst("h3") ?: summary?.selectFirst("h4"))?.text().orEmpty(), + altTitle = null, + rating = div.selectFirst("span.total_votes")?.ownText() + ?.toFloatOrNull()?.div(5f) ?: -1f, + tags = summary?.selectFirst(".mg_genres")?.select("a")?.mapToSet { a -> + MangaTag( + key = a.attr("href").removeSuffix("/").substringAfterLast('/'), + title = a.text().toTitleCase(), + source = source, + ) + }.orEmpty(), + author = summary?.selectFirst(".mg_author")?.selectFirst("a")?.ownText(), + state = when ( + summary?.selectFirst(".mg_status")?.selectFirst(".summary-content") + ?.ownText()?.trim()?.lowercase() + ) { + "ongoing" -> MangaState.ONGOING + "completed" -> MangaState.FINISHED + else -> null + }, + source = source, + isNsfw = isNsfwSource, + ) + } + } + + override suspend fun getTags(): Set { + val doc = webClient.httpGet("https://$domain/manga/").parseHtml() + val body = doc.body() + val root1 = body.selectFirst("header")?.selectFirst("ul.second-menu") + val root2 = body.selectFirst("div.genres_wrap")?.selectFirst("ul.list-unstyled") + if (root1 == null && root2 == null) { + doc.parseFailed("Root not found") + } + val list = root1?.select("li").orEmpty() + root2?.select("li").orEmpty() + val keySet = HashSet(list.size) + return list.mapNotNullToSet { li -> + val a = li.selectFirst("a") ?: return@mapNotNullToSet null + val href = a.attr("href").removeSuffix("/") + .substringAfterLast(tagPrefix, "") + if (href.isEmpty() || !keySet.add(href)) { + return@mapNotNullToSet null + } + MangaTag( + key = href, + title = a.ownText().trim().ifEmpty { + a.selectFirst(".menu-image-title")?.text()?.trim() ?: return@mapNotNullToSet null + }.toTitleCase(), + source = source, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { + val fullUrl = manga.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + val chaptersDeferred = async { getChapters(manga, doc) } + val root = doc.body().selectFirst("div.profile-manga") + ?.selectFirst("div.summary_content") + ?.selectFirst("div.post-content") + ?: throw ParseException("Root not found", fullUrl) + val root2 = doc.body().selectFirst("div.content-area") + ?.selectFirst("div.c-page") + ?: throw ParseException("Root2 not found", fullUrl) + manga.copy( + tags = root.selectFirst("div.genres-content")?.select("a") + ?.mapNotNullToSet { a -> + MangaTag( + key = a.attr("href").removeSuffix("/").substringAfterLast('/'), + title = a.text().toTitleCase(), + source = source, + ) + } ?: manga.tags, + description = root2.selectFirst("div.description-summary") + ?.selectFirst("div.summary__content") + ?.select("p") + ?.filterNot { it.ownText().startsWith("A brief description") } + ?.joinToString { it.html() }, + chapters = chaptersDeferred.await(), + ) + } + + protected open suspend fun getChapters(manga: Manga, doc: Document): List { + val root2 = doc.body().selectFirstOrThrow("div.content-area") + .selectFirstOrThrow("div.c-page") + val dateFormat = SimpleDateFormat(datePattern, sourceLocale ?: Locale.US) + return root2.select("li").asReversed().mapChapters { i, li -> + val a = li.selectFirst("a") + val href = a?.attrAsRelativeUrlOrNull("href") ?: li.parseFailed("Link is missing") + MangaChapter( + id = generateUid(href), + name = a.ownText(), + number = i + 1, + url = href, + uploadDate = parseChapterDate( + dateFormat, + li.selectFirst("span.chapter-release-date i")?.text(), + ), + source = source, + scanlator = null, + branch = null, + ) + } + } + + override suspend fun getPages(chapter: MangaChapter): List { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + val root = doc.body().selectFirst("div.main-col-inner") + ?.selectFirst("div.reading-content") + ?: throw ParseException("Root not found", fullUrl) + return root.select("div.page-break").map { div -> + val img = div.selectFirst("img") ?: div.parseFailed("Page image not found") + val url = img.src()?.toRelativeUrl(domain) ?: div.parseFailed("Image src not found") + MangaPage( + id = generateUid(url), + url = url, + preview = null, + referer = fullUrl, + source = source, + ) + } + } + + protected fun parseChapterDate(dateFormat: DateFormat, date: String?): Long { + date ?: return 0 + return when { + date.endsWith(" ago", ignoreCase = true) -> { + parseRelativeDate(date) + } + // Handle translated 'ago' in Portuguese. + date.endsWith(" atrás", ignoreCase = true) -> { + parseRelativeDate(date) + } + // Handle translated 'ago' in Turkish. + date.endsWith(" önce", ignoreCase = true) -> { + parseRelativeDate(date) + } + // Handle 'yesterday' and 'today', using midnight + date.startsWith("year", ignoreCase = true) -> { + Calendar.getInstance().apply { + add(Calendar.DAY_OF_MONTH, -1) // yesterday + set(Calendar.HOUR_OF_DAY, 0) + set(Calendar.MINUTE, 0) + set(Calendar.SECOND, 0) + set(Calendar.MILLISECOND, 0) + }.timeInMillis + } + + date.startsWith("today", ignoreCase = true) -> { + Calendar.getInstance().apply { + set(Calendar.HOUR_OF_DAY, 0) + set(Calendar.MINUTE, 0) + set(Calendar.SECOND, 0) + set(Calendar.MILLISECOND, 0) + }.timeInMillis + } + + date.contains(Regex("""\d(st|nd|rd|th)""")) -> { + // Clean date (e.g. 5th December 2019 to 5 December 2019) before parsing it + date.split(" ").map { + if (it.contains(Regex("""\d\D\D"""))) { + it.replace(Regex("""\D"""), "") + } else { + it + } + } + .let { dateFormat.tryParse(it.joinToString(" ")) } + } + + else -> dateFormat.tryParse(date) + } + } + + // Parses dates in this form: + // 21 hours ago + private fun parseRelativeDate(date: String): Long { + val number = Regex("""(\d+)""").find(date)?.value?.toIntOrNull() ?: return 0 + val cal = Calendar.getInstance() + + return when { + WordSet( + "hari", + "gün", + "jour", + "día", + "dia", + "day", + ).anyWordIn(date) -> cal.apply { add(Calendar.DAY_OF_MONTH, -number) }.timeInMillis + + WordSet("jam", "saat", "heure", "hora", "hour").anyWordIn(date) -> cal.apply { + add( + Calendar.HOUR, + -number, + ) + }.timeInMillis + + WordSet("menit", "dakika", "min", "minute", "minuto").anyWordIn(date) -> cal.apply { + add( + Calendar.MINUTE, + -number, + ) + }.timeInMillis + + WordSet("detik", "segundo", "second").anyWordIn(date) -> cal.apply { + add( + Calendar.SECOND, + -number, + ) + }.timeInMillis + + WordSet("month").anyWordIn(date) -> cal.apply { add(Calendar.MONTH, -number) }.timeInMillis + WordSet("year").anyWordIn(date) -> cal.apply { add(Calendar.YEAR, -number) }.timeInMillis + else -> 0 + } + } + + private fun Element.src(): String? { + var result = absUrl("data-src") + if (result.isEmpty()) result = absUrl("data-cfsrc") + if (result.isEmpty()) result = absUrl("src") + return if (result.isEmpty()) null else result + } + + private fun createRequestTemplate() = + ( + "action=madara_load_more&page=1&template=madara-core%2Fcontent%2Fcontent-search&vars%5Bs%5D=&vars%5B" + + "orderby%5D=meta_value_num&vars%5Bpaged%5D=1&vars%5Btemplate%5D=search&vars%5Bmeta_query" + + "%5D%5B0%5D%5Brelation%5D=AND&vars%5Bmeta_query%5D%5Brelation%5D=OR&vars%5Bpost_type" + + "%5D=wp-manga&vars%5Bpost_status%5D=publish&vars%5Bmeta_key%5D=_latest_update&vars%5Border" + + "%5D=desc&vars%5Bmanga_archives_item_layout%5D=default" + ).split('&') + .map { + val pos = it.indexOf('=') + it.substring(0, pos) to it.substring(pos + 1) + }.toMutableMap() + + @MangaSourceParser("MANGAREAD", "MangaRead", "en") + class MangaRead(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGAREAD, "www.mangaread.org") { + override val tagPrefix = "genres/" + override val datePattern = "dd.MM.yyyy" + } + + @MangaSourceParser("MANGAWEEBS", "MangaWeebs", "en") + class MangaWeebs(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGAWEEBS, "mangaweebs.in") { + override val datePattern = "dd MMMM HH:mm" + } + + @MangaSourceParser("HACHIMANGA", "HachiManga", "ja") + class HachiManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.HACHIMANGA, "hachimanga.com") { + + override val sourceLocale: Locale = Locale.ENGLISH + } + + @MangaSourceParser("PIANMANGA", "PianManga", "en") + class PianManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.PIANMANGA, "pianmanga.me") - @MangaSourceParser("S2MANGA", "S2Manga", "en") - class S2Manga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.S2MANGA, "s2manga.com") + @MangaSourceParser("MANGAROSIE", "MangaRosie", "en") + class MangaRosie(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGAROSIE, "mangarosie.in") - @MangaSourceParser("SKY_MANGA", "Sky Manga", "en") - class SkyManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.SKY_MANGA, "skymanga.xyz") { + @MangaSourceParser("MANGATX", "MangaTx", "en") + class MangaTx(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGATX, "mangatx.com") - override val isNsfwSource = true + @MangaSourceParser("MANGAEFFECT", "MangaEffect", "en") + class MangaEffect(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGAEFFECT, "mangaeffect.com") { + override val datePattern = "dd.MM.yyyy" + } - } + @MangaSourceParser("AQUAMANGA", "AquaManga", "en") + class AquaManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.AQUAMANGA, "aquamanga.com") - @MangaSourceParser("BAKAMAN", "BakaMan", "th") - class BakaMan(context: MangaLoaderContext) : MadaraParser(context, MangaSource.BAKAMAN, "bakaman.net") { + @MangaSourceParser("MANGALEK", "MangaLek", "ar") + class MangaLek(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGALEK, "mangalek.com") - override val isNsfwSource = true - } + @MangaSourceParser("HARIMANGA", "HariManga", "en") + class HariManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.HARIMANGA, "harimanga.com") { + override val datePattern = "MM/dd/yyyy" + } - @MangaSourceParser("MANGA_DISTRICT", "Manga District", "en") - class MangaDistrict(context: MangaLoaderContext) : - MadaraParser(context, MangaSource.MANGA_DISTRICT, "mangadistrict.com") { + @MangaSourceParser("KISSMANGA", "KissManga", "en") + class KissManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.KISSMANGA, "kissmanga.in") - override val tagPrefix = "read-scan/" + @MangaSourceParser("MANGAROCK", "MangaRock", "en") + class MangaRock(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGAROCK, "mangarockteam.com") - override val isNsfwSource = true - } + @MangaSourceParser("FREEMANGA", "FreeManga", "en") + class FreeManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.FREEMANGA, "freemanga.me") - @MangaSourceParser("HENTAI20", "Hentai20", "en") - class Hentai20(context: MangaLoaderContext) : - MadaraParser(context, MangaSource.HENTAI20, "hentai20.io") { + @MangaSourceParser("MANGA_KOMI", "MangaKomi", "en") + class MangaKomi(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGA_KOMI, "mangakomi.io") - override val tagPrefix = "manga-genre/" + @MangaSourceParser("NEATMANGA", "NeatManga", "en") + class NeatManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.NEATMANGA, "neatmangas.com") - override val isNsfwSource = true - } + @MangaSourceParser("MANHWACLAN", "ManhwaClan", "en") + class ManhwaClan(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANHWACLAN, "manhwaclan.com") - @MangaSourceParser("HENTAI_4FREE", "Hentai4Free", "en") - class Hentai4Free(context: MangaLoaderContext) : - MadaraParser(context, MangaSource.HENTAI_4FREE, "hentai4free.net") { + @MangaSourceParser("MANGA_3S", "Manga3s", "en") + class Manga3s(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGA_3S, "manga3s.com") { + override val tagPrefix = "manhwa-genre/" + } - override val tagPrefix = "hentai-genre/" + @MangaSourceParser("MANHWAKOOL", "Manhwa Kool", "en") + class ManhwaKool(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANHWAKOOL, "manhwakool.com") { + override val datePattern: String = "MM/dd" + } - override val isNsfwSource = true + @MangaSourceParser("TOPMANHUA", "Top Manhua", "en") + class TopManhua(context: MangaLoaderContext) : MadaraParser(context, MangaSource.TOPMANHUA, "www.topmanhua.com") { + override val tagPrefix = "manhua-genre/" + override val datePattern = "MM/dd/yyyy" + } - override suspend fun getTags(): Set { - val doc = webClient.httpGet("https://$domain/").parseHtml() - val body = doc.body() - val root1 = body.selectFirst("header")?.selectFirst("ul.second-menu") - val list = root1?.select("li").orEmpty() - val keySet = HashSet(list.size) - return list.mapNotNullToSet { li -> - val a = li.selectFirst("a") ?: return@mapNotNullToSet null - val href = a.attr("href").removeSuffix("/") - .substringAfterLast(tagPrefix, "") - if (href.isEmpty() || !keySet.add(href)) { - return@mapNotNullToSet null - } - MangaTag( - key = href, - title = a.ownText().trim().toTitleCase(), - source = source, - ) - } - } - } + @MangaSourceParser("S2MANGA", "S2Manga", "en") + class S2Manga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.S2MANGA, "s2manga.com") - @MangaSourceParser("ALLPORN_COMIC", "All Porn Comic", "en") - class AllPornComic(context: MangaLoaderContext) : - MadaraParser(context, MangaSource.ALLPORN_COMIC, "allporncomic.com") { + @MangaSourceParser("SKY_MANGA", "Sky Manga", "en") + class SkyManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.SKY_MANGA, "skymanga.xyz") { - override val tagPrefix = "porncomic-genre/" + override val isNsfwSource = true - override val isNsfwSource = true + } - } + @MangaSourceParser("BAKAMAN", "BakaMan", "th") + class BakaMan(context: MangaLoaderContext) : MadaraParser(context, MangaSource.BAKAMAN, "bakaman.net") { - @MangaSourceParser("CAT_300", "Cat300", "th") - class Cat300(context: MangaLoaderContext) : MadaraParser(context, MangaSource.CAT_300, "cat300.com") { + override val isNsfwSource = true + } - override val isNsfwSource = true - } + @MangaSourceParser("HENTAI20", "Hentai20", "en") + class Hentai20(context: MangaLoaderContext) : + MadaraParser(context, MangaSource.HENTAI20, "hentai20.io") { - @MangaSourceParser("BIBIMANGA", "BibiManga", "en") - class BibiManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.BIBIMANGA, "bibimanga.com") { + override val tagPrefix = "manga-genre/" - override val isNsfwSource = true - } + override val isNsfwSource = true + } - @MangaSourceParser("TREE_MANGA", "Tree Manga", "en") - class TreeManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.TREE_MANGA, "treemanga.com") { + @MangaSourceParser("ALLPORN_COMIC", "All Porn Comic", "en") + class AllPornComic(context: MangaLoaderContext) : + MadaraParser(context, MangaSource.ALLPORN_COMIC, "allporncomic.com") { - override val datePattern = "MM/dd/yyyy" + override val tagPrefix = "porncomic-genre/" - } + override val isNsfwSource = true - @MangaSourceParser("MANGACV", "Manga Cv", "en") - class MangaCv(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGACV, "mangacv.com") { + } - } + @MangaSourceParser("CAT_300", "Cat300", "th") + class Cat300(context: MangaLoaderContext) : MadaraParser(context, MangaSource.CAT_300, "cat300.com") { - @MangaSourceParser("TOONILY", "Toonily", "en") - class Toonily(context: MangaLoaderContext) : MadaraParser(context, MangaSource.TOONILY, "toonily.com") { + override val isNsfwSource = true + } - override val tagPrefix = "webtoon-genre/" + @MangaSourceParser("BIBIMANGA", "BibiManga", "en") + class BibiManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.BIBIMANGA, "bibimanga.com") { - override val isNsfwSource = true + override val isNsfwSource = true } - @MangaSourceParser("MANGA_MANHUA", "Manga Manhua", "en") - class MangaManhua(context: MangaLoaderContext) : - MadaraParser(context, MangaSource.MANGA_MANHUA, "mangamanhua.online") + @MangaSourceParser("TREE_MANGA", "Tree Manga", "en") + class TreeManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.TREE_MANGA, "treemanga.com") { - @MangaSourceParser("MANGA_247", "247MANGA", "en") - class Manga247(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGA_247, "247manga.com") { - override val tagPrefix = "manhwa-genre/" - } + override val datePattern = "MM/dd/yyyy" - @MangaSourceParser("MANGA_365", "365Manga", "en") - class Manga365(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGA_365, "365manga.com") + } + + @MangaSourceParser("MANGACV", "Manga Cv", "en") + class MangaCv(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGACV, "mangacv.com") + + @MangaSourceParser("TOONILY", "Toonily", "en") + class Toonily(context: MangaLoaderContext) : MadaraParser(context, MangaSource.TOONILY, "toonily.com") { + + override val tagPrefix = "webtoon-genre/" + + override val isNsfwSource = true + } - @MangaSourceParser("MANGACLASH", "Mangaclash", "en") - class Mangaclash(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGACLASH, "mangaclash.com") { - override val datePattern = "MM/dd/yyyy" - } + @MangaSourceParser("MANGA_MANHUA", "Manga Manhua", "en") + class MangaManhua(context: MangaLoaderContext) : + MadaraParser(context, MangaSource.MANGA_MANHUA, "mangamanhua.online") + + @MangaSourceParser("MANGA_247", "247MANGA", "en") + class Manga247(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGA_247, "247manga.com") { + override val tagPrefix = "manhwa-genre/" + } + + @MangaSourceParser("MANGA_365", "365Manga", "en") + class Manga365(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGA_365, "365manga.com") + + @MangaSourceParser("MANGACLASH", "Mangaclash", "en") + class Mangaclash(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGACLASH, "mangaclash.com") { + override val datePattern = "MM/dd/yyyy" + } - @MangaSourceParser("ZINMANGA", "ZINMANGA", "en") - class Zinmanga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.ZINMANGA, "zinmanga.com") + @MangaSourceParser("ZINMANGA", "ZINMANGA", "en") + class Zinmanga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.ZINMANGA, "zinmanga.com") } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangaDistrict.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangaDistrict.kt new file mode 100644 index 00000000..75439adc --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangaDistrict.kt @@ -0,0 +1,49 @@ +package org.koitharu.kotatsu.parsers.site.madara + +import org.jsoup.nodes.Document +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.model.Manga +import org.koitharu.kotatsu.parsers.model.MangaChapter +import org.koitharu.kotatsu.parsers.model.MangaSource +import org.koitharu.kotatsu.parsers.util.* +import java.text.SimpleDateFormat +import java.util.* + +@MangaSourceParser("MANGA_DISTRICT", "Manga District", "en") +internal class MangaDistrict(context: MangaLoaderContext) : + MadaraParser(context, MangaSource.MANGA_DISTRICT, "mangadistrict.com") { + + override val tagPrefix = "publication-genre/" + + override val isNsfwSource = true + + override val datePattern = "MMM dd,yyyy" + + override suspend fun getChapters(manga: Manga, doc: Document): List { + val slug = manga.url.removeSuffix('/').substringAfterLast('/') + val doc2 = webClient.httpPost( + "https://$domain/read-scan/$slug/ajax/chapters/", + mapOf(), + ).parseHtml() + val ul = doc2.body().selectFirstOrThrow("ul") + val dateFormat = SimpleDateFormat(datePattern, Locale.US) + return ul.select("li").asReversed().mapChapters { i, li -> + val a = li.selectFirst("a") + val href = a?.attrAsRelativeUrlOrNull("href") ?: li.parseFailed("Link is missing") + MangaChapter( + id = generateUid(href), + name = a.ownText(), + number = i + 1, + url = href, + uploadDate = parseChapterDate( + dateFormat, + li.selectFirst("span.chapter-release-date i")?.text(), + ), + source = source, + scanlator = null, + branch = null, + ) + } + } +} From 109358420230b4f12a38b24ae45cdc6471fbd15c Mon Sep 17 00:00:00 2001 From: Koitharu Date: Sat, 25 Feb 2023 18:36:05 +0200 Subject: [PATCH 079/121] [Grouple] Add UserAgent config --- .../koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt index e99784af..fd975206 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt @@ -283,6 +283,11 @@ internal abstract class GroupleParser( } else res } + override fun onCreateConfig(keys: MutableCollection>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + private fun getSortKey(sortOrder: SortOrder) = when (sortOrder) { SortOrder.ALPHABETICAL -> "name" From f2614dd3d7b35f8cd54dfe29773253e7cb7e9a5a Mon Sep 17 00:00:00 2001 From: Saksham Shekher <95137948+OshekharO@users.noreply.github.com> Date: Sun, 26 Feb 2023 23:03:43 +0530 Subject: [PATCH 080/121] Fix --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 35186f09..0debf1c8 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ This library provides manga sources. val parser = MangaSource.MANGADEX.newParser(mangaLoaderContext) ``` - `mangaLoaderContext` is an implementation of the MangaLoaderContext class. + `mangaLoaderContext` is an implementation of the `MangaLoaderContext` class. See examples of [Android](https://github.com/KotatsuApp/Kotatsu/blob/devel/app/src/main/java/org/koitharu/kotatsu/core/parser/MangaLoaderContextImpl.kt) and [Non-Android](https://github.com/KotatsuApp/kotatsu-dl/blob/master/src/main/kotlin/org/koitharu/kotatsu_dl/env/MangaLoaderContextImpl.kt) implementation. From 489a4ab5869643e21a4ab909d85a8c2b45ac8576 Mon Sep 17 00:00:00 2001 From: Koitharu Date: Sun, 5 Mar 2023 07:58:44 +0200 Subject: [PATCH 081/121] [NeatManga] Fix chapters loading --- .../parsers/site/madara/IsekaiScanEu.kt | 47 ------------ .../parsers/site/madara/IsekaiScanEuParser.kt | 47 ++++++++++++ .../parsers/site/madara/MadaraParser.kt | 3 - .../kotatsu/parsers/site/madara/NeatManga.kt | 75 +++++++++++++++++++ 4 files changed, 122 insertions(+), 50 deletions(-) delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/IsekaiScanEu.kt create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/IsekaiScanEuParser.kt create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/NeatManga.kt diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/IsekaiScanEu.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/IsekaiScanEu.kt deleted file mode 100644 index 526353e5..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/IsekaiScanEu.kt +++ /dev/null @@ -1,47 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.madara - -import org.jsoup.nodes.Document -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.Manga -import org.koitharu.kotatsu.parsers.model.MangaChapter -import org.koitharu.kotatsu.parsers.model.MangaSource -import org.koitharu.kotatsu.parsers.util.* -import java.text.SimpleDateFormat -import java.util.* - -@MangaSourceParser("ISEKAISCAN_EU", "IsekaiScan (eu)", "en") -internal class IsekaiScanEuParser(context: MangaLoaderContext) : - MadaraParser(context, MangaSource.ISEKAISCAN_EU, "isekaiscan.eu") { - - override val datePattern = "MM/dd/yyyy" - - override suspend fun getChapters(manga: Manga, doc: Document): List { - val mangaId = doc.body().requireElementById("manga-chapters-holder").attr("data-id") - val ul = webClient.httpPost( - "https://${domain}/wp-admin/admin-ajax.php", - mapOf( - "action" to "manga_get_chapters", - "manga" to mangaId, - ), - ).parseHtml().body().selectFirstOrThrow("ul") - val dateFormat = SimpleDateFormat(datePattern, Locale.US) - return ul.select("li").asReversed().mapChapters { i, li -> - val a = li.selectFirst("a") - val href = a?.attrAsRelativeUrlOrNull("href") ?: li.parseFailed("Link is missing") - MangaChapter( - id = generateUid(href), - name = a.ownText(), - number = i + 1, - url = href, - uploadDate = parseChapterDate( - dateFormat, - li.selectFirst("span.chapter-release-date i")?.text(), - ), - source = source, - scanlator = null, - branch = null, - ) - } - } -} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/IsekaiScanEuParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/IsekaiScanEuParser.kt new file mode 100644 index 00000000..3ec177c7 --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/IsekaiScanEuParser.kt @@ -0,0 +1,47 @@ +package org.koitharu.kotatsu.parsers.site.madara + +import org.jsoup.nodes.Document +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.model.Manga +import org.koitharu.kotatsu.parsers.model.MangaChapter +import org.koitharu.kotatsu.parsers.model.MangaSource +import org.koitharu.kotatsu.parsers.util.* +import java.text.SimpleDateFormat +import java.util.* + +@MangaSourceParser("ISEKAISCAN_EU", "IsekaiScan (eu)", "en") +internal class IsekaiScanEuParser(context: MangaLoaderContext) : + MadaraParser(context, MangaSource.ISEKAISCAN_EU, "isekaiscan.eu") { + + override val datePattern = "MM/dd/yyyy" + + override suspend fun getChapters(manga: Manga, doc: Document): List { + val mangaId = doc.body().requireElementById("manga-chapters-holder").attr("data-id") + val ul = webClient.httpPost( + "https://${domain}/wp-admin/admin-ajax.php", + mapOf( + "action" to "manga_get_chapters", + "manga" to mangaId, + ), + ).parseHtml().body().selectFirstOrThrow("ul") + val dateFormat = SimpleDateFormat(datePattern, Locale.US) + return ul.select("li").asReversed().mapChapters { i, li -> + val a = li.selectFirst("a") + val href = a?.attrAsRelativeUrlOrNull("href") ?: li.parseFailed("Link is missing") + MangaChapter( + id = generateUid(href), + name = a.ownText(), + number = i + 1, + url = href, + uploadDate = parseChapterDate( + dateFormat, + li.selectFirst("span.chapter-release-date i")?.text(), + ), + source = source, + scanlator = null, + branch = null, + ) + } + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt index 8cd86ff0..d5e40952 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt @@ -359,9 +359,6 @@ internal abstract class MadaraParser( @MangaSourceParser("MANGA_KOMI", "MangaKomi", "en") class MangaKomi(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGA_KOMI, "mangakomi.io") - @MangaSourceParser("NEATMANGA", "NeatManga", "en") - class NeatManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.NEATMANGA, "neatmangas.com") - @MangaSourceParser("MANHWACLAN", "ManhwaClan", "en") class ManhwaClan(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANHWACLAN, "manhwaclan.com") diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/NeatManga.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/NeatManga.kt new file mode 100644 index 00000000..2e325c6b --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/NeatManga.kt @@ -0,0 +1,75 @@ +package org.koitharu.kotatsu.parsers.site.madara + +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.exception.ParseException +import org.koitharu.kotatsu.parsers.model.Manga +import org.koitharu.kotatsu.parsers.model.MangaChapter +import org.koitharu.kotatsu.parsers.model.MangaSource +import org.koitharu.kotatsu.parsers.model.MangaTag +import org.koitharu.kotatsu.parsers.util.* +import java.text.SimpleDateFormat +import java.util.* + +@MangaSourceParser("NEATMANGA", "NeatManga", "en") +internal class NeatManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.NEATMANGA, "neatmangas.com") { + + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { + val chaptersDeferred = async { getChapters(manga) } + val fullUrl = manga.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + val root = doc.body().selectFirst("div.profile-manga") + ?.selectFirst("div.summary_content") + ?.selectFirst("div.post-content") + ?: throw ParseException("Root not found", fullUrl) + val root2 = doc.body().selectFirst("div.content-area") + ?.selectFirst("div.c-page") + ?: throw ParseException("Root2 not found", fullUrl) + manga.copy( + tags = root.selectFirst("div.genres-content")?.select("a") + ?.mapNotNullToSet { a -> + MangaTag( + key = a.attr("href").removeSuffix("/").substringAfterLast('/'), + title = a.text().toTitleCase(), + source = source, + ) + } ?: manga.tags, + description = root2.getElementsMatchingOwnText("Summary") + .firstOrNull() + ?.nextElementSibling() + ?.select("p") + ?.filterNot { it.ownText().startsWith("A brief description") } + ?.joinToString { it.html() }, + chapters = chaptersDeferred.await(), + ) + } + + private suspend fun getChapters(manga: Manga): List { + val slug = manga.url.removeSuffix('/').substringAfterLast('/') + val doc2 = webClient.httpPost( + "https://$domain/manga/$slug/ajax/chapters/", + mapOf(), + ).parseHtml() + val ul = doc2.body().selectFirstOrThrow("ul") + val dateFormat = SimpleDateFormat(datePattern, Locale.US) + return ul.select("li").asReversed().mapChapters { i, li -> + val a = li.selectFirst("a") + val href = a?.attrAsRelativeUrlOrNull("href") ?: li.parseFailed("Link is missing") + MangaChapter( + id = generateUid(href), + name = a.ownText(), + number = i + 1, + url = href, + uploadDate = parseChapterDate( + dateFormat, + li.selectFirst("span.chapter-release-date i")?.text(), + ), + source = source, + scanlator = null, + branch = null, + ) + } + } +} From e8d299782ba0123264dac3573388b6ac29df1f43 Mon Sep 17 00:00:00 2001 From: Koitharu Date: Sun, 5 Mar 2023 08:04:59 +0200 Subject: [PATCH 082/121] [Tempestfansub] Update domain #159 --- .../site/mangareader/MangaReaderParser.kt | 1294 ++++++++--------- 1 file changed, 647 insertions(+), 647 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt index 5a1b8499..215a6e11 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt @@ -16,653 +16,653 @@ import java.text.SimpleDateFormat import java.util.* internal abstract class MangaReaderParser( - context: MangaLoaderContext, - source: MangaSource, - pageSize: Int, - searchPageSize: Int + context: MangaLoaderContext, + source: MangaSource, + pageSize: Int, + searchPageSize: Int, ) : PagedMangaParser(context, source, pageSize, searchPageSize) { - override val sortOrders: Set - get() = EnumSet.of(SortOrder.UPDATED, SortOrder.POPULARITY, SortOrder.ALPHABETICAL, SortOrder.NEWEST) - - protected val idLocale - get() = Locale("in", "ID") - - abstract val listUrl: String - abstract val tableMode: Boolean - open val chapterDateFormat = SimpleDateFormat("MMM d, yyyy", idLocale) - - private var tagCache: ArrayMap? = null - private val mutex = Mutex() - private var lastSearchPage = 1 - - override suspend fun getDetails(manga: Manga): Manga { - val docs = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - val chapters = docs.select("#chapterlist > ul > li").reversed().mapChapters { index, element -> - val url = element.selectFirst("a")?.attrAsRelativeUrl("href") ?: return@mapChapters null - MangaChapter( - id = generateUid(url), - name = element.selectFirst(".chapternum")?.text() ?: "Chapter ${index + 1}", - url = url, - number = index + 1, - scanlator = null, - uploadDate = chapterDateFormat.tryParse(element.selectFirst(".chapterdate")?.text()), - branch = null, - source = source - ) - } - return if (tableMode) parseInfoTable(docs, manga, chapters) else parseInfoList(docs, manga, chapters) - } - - open suspend fun parseInfoTable(docs: Document, manga: Manga, chapters: List): Manga { - val mangaInfo = docs.selectFirst("div.seriestucontent > div.seriestucontentr") - val mangaState = mangaInfo?.selectFirst(".infotable td:contains(Status)")?.lastElementSibling()?.let { - when (it.text()) { - "Ongoing" -> MangaState.ONGOING - "Completed" -> MangaState.FINISHED - else -> null - } - } - val tagMap = getOrCreateTagMap() - val tags = mangaInfo?.select(".seriestugenre > a")?.mapNotNullToSet { tagMap[it.text()] } - - return manga.copy( - description = mangaInfo?.selectFirst("div.entry-content")?.html(), - state = mangaState, - author = mangaInfo?.selectFirst(".infotable td:contains(Author)")?.lastElementSibling()?.text(), - isNsfw = docs.selectFirst(".restrictcontainer") != null, - tags = tags.orEmpty(), - chapters = chapters, - ) - } - - open suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { - val mangaState = docs.selectFirst(".info-left .tsinfo div:contains(Status)")?.lastElementChild()?.let { - when (it.text()) { - "Ongoing" -> MangaState.ONGOING - "Completed" -> MangaState.FINISHED - else -> null - } - } - val tagMap = getOrCreateTagMap() - val tags = docs.select(".info-right .mgen > a").mapNotNullToSet { tagMap[it.text()] } - - return manga.copy( - description = docs.selectFirst(".info-right div.entry-content > p")?.html(), - state = mangaState, - author = docs.selectFirst(".info-left .tsinfo div:contains(Author)")?.lastElementChild()?.text(), - isNsfw = docs.selectFirst(".info-right .alr") != null, - tags = tags, - chapters = chapters, - ) - } - - override suspend fun getListPage( - page: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder - ): List { - if (!query.isNullOrEmpty()) { - if (page > lastSearchPage) { - return emptyList() - } - - val url = buildString { - append("https://") - append(domain) - append("/page/") - append(page) - append("/?s=") - append(query.urlEncoded()) - } - - val docs = webClient.httpGet(url).parseHtml() - lastSearchPage = docs.selectFirst(".pagination .next") - ?.previousElementSibling() - ?.text()?.toIntOrNull() ?: 1 - return parseMangaList(docs) - } - - val sortQuery = when (sortOrder) { - SortOrder.ALPHABETICAL -> "title" - SortOrder.NEWEST -> "latest" - SortOrder.POPULARITY -> "popular" - SortOrder.UPDATED -> "update" - else -> "" - } - val tagKey = "genre[]".urlEncoded() - val tagQuery = - if (tags.isNullOrEmpty()) "" else tags.joinToString(separator = "&", prefix = "&") { "$tagKey=${it.key}" } - val url = buildString { - append("https://") - append(domain) - append(listUrl) - append("/?order=") - append(sortQuery) - append(tagQuery) - append("&page=") - append(page) - } - - return parseMangaList(webClient.httpGet(url).parseHtml()) - } - - private fun parseMangaList(docs: Document): List { - return docs.select(".postbody .listupd .bs .bsx").mapNotNull { - val a = it.selectFirst("a") ?: return@mapNotNull null - val relativeUrl = a.attrAsRelativeUrl("href") - val rating = it.selectFirst(".numscore")?.text() - ?.toFloatOrNull()?.div(10) ?: RATING_UNKNOWN - - Manga( - id = generateUid(relativeUrl), - url = relativeUrl, - title = a.attr("title"), - altTitle = null, - publicUrl = a.attrAsAbsoluteUrl("href"), - rating = rating, - isNsfw = false, - coverUrl = it.selectFirst("img.ts-post-image")?.imageUrl().orEmpty(), - tags = emptySet(), - state = null, - author = null, - source = source - ) - } - } - - override suspend fun getPages(chapter: MangaChapter): List { - val chapterUrl = chapter.url.toAbsoluteUrl(domain) - val docs = webClient.httpGet(chapterUrl).parseHtml() - val script = docs.selectFirstOrThrow("script:containsData(ts_reader)") - val images = JSONObject(script.data().substringAfter('(').substringBeforeLast(')')) - .getJSONArray("sources") - .getJSONObject(0) - .getJSONArray("images") - - val pages = ArrayList(images.length()) - for (i in 0 until images.length()) { - pages.add( - MangaPage( - id = generateUid(images.getString(i)), - url = images.getString(i), - referer = chapterUrl, - preview = null, - source = source - ) - ) - } - - return pages - } - - override suspend fun getTags(): Set { - return getOrCreateTagMap().values.toSet() - } - - protected suspend fun getOrCreateTagMap(): Map = mutex.withLock { - tagCache?.let { return@withLock it } - val tagMap = ArrayMap() - - val url = listUrl.toAbsoluteUrl(domain) - val tagElements = webClient.httpGet(url).parseHtml().select("ul.genrez > li") - for (el in tagElements) { - if (el.text().isEmpty()) continue - - tagMap[el.text()] = MangaTag( - title = el.text(), - key = el.selectFirst("input")?.attr("value") ?: continue, - source = source - ) - } - - tagCache = tagMap - return@withLock tagMap - } - - private fun Element.imageUrl(): String { - return attrAsAbsoluteUrlOrNull("src") - ?: attrAsAbsoluteUrlOrNull("data-cfsrc") - ?: "" - } - - @MangaSourceParser("MANHWALAND", "Manhwaland", "id") - class ManhwaLandParser(context: MangaLoaderContext) : - MangaReaderParser(context, MangaSource.MANHWALAND, pageSize = 20, searchPageSize = 10) { - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("manhwaland.guru", null) - - override val listUrl: String - get() = "/series" - override val tableMode: Boolean - get() = false - } - - @MangaSourceParser("SEKAIKOMIK", "Sekaikomik", "id") - class SekaikomikParser(context: MangaLoaderContext) : - MangaReaderParser(context, MangaSource.SEKAIKOMIK, pageSize = 20, searchPageSize = 100) { - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("sekaikomik.pro", null) - - override val listUrl: String - get() = "/manga" - override val tableMode: Boolean - get() = false - override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMMM D, yyyy", idLocale) - } - - @MangaSourceParser("MANHWAINDO", "Manhwaindo", "id") - class ManhwaIndoParser(context: MangaLoaderContext) : - MangaReaderParser(context, MangaSource.MANHWAINDO, pageSize = 20, searchPageSize = 10) { - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("manhwaindo.id", null) - - override val listUrl: String get() = "/series" - override val tableMode: Boolean get() = false - } - - @MangaSourceParser("MANHWALIST", "Manhwalist", "id") - class ManhwalistParser(context: MangaLoaderContext) : - MangaReaderParser(context, MangaSource.MANHWALIST, pageSize = 24, searchPageSize = 10) { - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("manhwalist.in", null) - - override val listUrl: String = "/manga" - override val tableMode: Boolean get() = false - override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMMM d, yyyy", Locale.ENGLISH) - } - - @MangaSourceParser("KIRYUU", "Kiryuu", "id") - class KiryuuParser(context: MangaLoaderContext) : - MangaReaderParser(context, MangaSource.KIRYUU, pageSize = 20, searchPageSize = 10) { - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("kiryuu.id", null) - - override val listUrl: String - get() = "/manga" - override val tableMode: Boolean - get() = true - override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMMM d, yyyy", idLocale) - } - - @MangaSourceParser("TURKTOON", "Turktoon", "tr") - class TurktoonParser(context: MangaLoaderContext) : - MangaReaderParser(context, MangaSource.TURKTOON, pageSize = 20, searchPageSize = 10) { - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("turktoon.com", null) - - override val listUrl: String - get() = "/manga" - override val tableMode: Boolean - get() = false - override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMMM d, yyyy", Locale("tr", "TR")) - - override suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { - val mangaState = docs.selectFirst(".info-left .tsinfo div:contains(Durum)")?.lastElementChild()?.let { - when (it.text()) { - "Devam Ediyor" -> MangaState.ONGOING - "Tamamlandı" -> MangaState.FINISHED - else -> null - } - } - - return super.parseInfoList(docs, manga, chapters).copy(state = mangaState) - } - } - - @MangaSourceParser("WESTMANGA", "Westmanga", "id") - class WestmangaParser(context: MangaLoaderContext) : - MangaReaderParser(context, MangaSource.WESTMANGA, pageSize = 20, searchPageSize = 10) { - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("westmanga.info", null) - - override val listUrl: String - get() = "/manga" - override val tableMode: Boolean - get() = true - override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMMM d, yyyy", Locale.ENGLISH) - } - - @MangaSourceParser("TEMPESTFANSUB", "Tempestfansub", "tr") - class TempestfansubParser(context: MangaLoaderContext) : - MangaReaderParser(context, MangaSource.TEMPESTFANSUB, pageSize = 40, searchPageSize = 40) { - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("manga.tempestfansub.com", null) - - override val listUrl: String get() = "/manga" - override val tableMode: Boolean get() = true - override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMMM d, yyyy", Locale("tr", "TR")) - - override suspend fun parseInfoTable(docs: Document, manga: Manga, chapters: List): Manga { - val infoElement = docs.selectFirst("div.infox") - return manga.copy( - chapters = chapters, - description = infoElement?.selectFirst("div.entry-content")?.html(), - author = infoElement?.selectFirst(".flex-wrap div:contains(Artist)")?.lastElementSibling()?.text(), - tags = infoElement?.select(".wd-full .mgen > a") - ?.mapNotNullToSet { getOrCreateTagMap()[it.text()] } - .orEmpty(), - isNsfw = docs.selectFirst(".postbody .alr") != null, - ) - } - } - - @MangaSourceParser("MANHWADESU", "ManhwaDesu", "id") - class ManhwadesuParser(context: MangaLoaderContext) : - MangaReaderParser(context, MangaSource.MANHWADESU, pageSize = 20, searchPageSize = 10) { - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("manhwadesu.org", null) - - override val listUrl: String get() = "/komik" - override val tableMode: Boolean get() = false - } - - @MangaSourceParser("MANGATALE", "MangaTale", "id") - class MangaTaleParser(context: MangaLoaderContext) : - MangaReaderParser(context, MangaSource.MANGATALE, pageSize = 20, searchPageSize = 10) { - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("mangatale.co", null) - - override val listUrl: String get() = "/manga" - override val tableMode: Boolean get() = false - - override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMMM d, yyyy", Locale.ENGLISH) - - override suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { - val infoElement = docs.selectFirst("div.infox") - return manga.copy( - chapters = chapters, - description = infoElement?.selectFirst("div.entry-content")?.html(), - author = infoElement?.selectFirst(".flex-wrap div:contains(Author)")?.lastElementSibling()?.text(), - tags = infoElement?.select(".wd-full .mgen > a") - ?.mapNotNullToSet { getOrCreateTagMap()[it.text()] } - .orEmpty(), - isNsfw = docs.selectFirst(".postbody .alr") != null, - ) - } - } - - @MangaSourceParser("DRAGONTRANSLATION", "DragonTranslation", "es") - class DragonTranslationParser(context: MangaLoaderContext) : - MangaReaderParser(context, MangaSource.DRAGONTRANSLATION, pageSize = 20, searchPageSize = 10) { - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("dragontranslation.com", null) - - override val listUrl: String get() = "/manga" - override val tableMode: Boolean get() = false - - override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMMM d, yyyy", Locale.ENGLISH) - - override suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { - val infoElement = docs.selectFirst("div.infox") - return manga.copy( - chapters = chapters, - description = infoElement?.selectFirst("div.entry-content")?.html(), - author = infoElement?.selectFirst(".flex-wrap div:contains(Author)")?.lastElementSibling()?.text(), - tags = infoElement?.select(".wd-full .mgen > a") - ?.mapNotNullToSet { getOrCreateTagMap()[it.text()] } - .orEmpty(), - isNsfw = docs.selectFirst(".postbody .alr") != null, - ) - } - } - - @MangaSourceParser("ASURATR", "Asura Scans (tr)", "tr") - class AsuraTRParser(context: MangaLoaderContext) : - MangaReaderParser(context, MangaSource.ASURATR, pageSize = 30, searchPageSize = 10) { - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("asurascanstr.com", null) - - override val listUrl: String get() = "/manga" - override val tableMode: Boolean get() = false - - override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", Locale("tr")) - - override suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { - val infoElement = docs.selectFirst("div.infox") - return manga.copy( - chapters = chapters, - description = infoElement?.selectFirst("div.entry-content")?.html(), - author = infoElement?.selectFirst(".flex-wrap div:contains(Yazar)")?.lastElementSibling()?.text(), - tags = infoElement?.select(".wd-full .mgen > a") - ?.mapNotNullToSet { getOrCreateTagMap()[it.text()] } - .orEmpty(), - isNsfw = docs.selectFirst(".postbody .alr") != null, - ) - } - } - - @MangaSourceParser("KOMIKTAP", "KomikTap", "id") - class KomikTapParser(context: MangaLoaderContext) : - MangaReaderParser(context, MangaSource.KOMIKTAP, pageSize = 15, searchPageSize = 10) { - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("92.87.6.124", null) - - override val listUrl: String - get() = "/manga" - override val tableMode: Boolean - get() = true - - override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", idLocale) - } - - @MangaSourceParser("KUMAPOI", "KumaPoi", "id") - class KumaPoiParser(context: MangaLoaderContext) : - MangaReaderParser(context, MangaSource.KUMAPOI, pageSize = 15, searchPageSize = 10) { - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("kumapoi.me", null) - - override val listUrl: String - get() = "/manga" - override val tableMode: Boolean - get() = true - - override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH) - } - - @MangaSourceParser("ASURASCANS", "Asura Scans", "en") - class AsuraScansParser(context: MangaLoaderContext) : - MangaReaderParser(context, MangaSource.ASURASCANS, pageSize = 20, searchPageSize = 10) { - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("asurascans.com", null) - - override val listUrl: String - get() = "/manga" - override val tableMode: Boolean - get() = false - - override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH) - - override suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { - val infoElement = docs.selectFirst("div.infox") - return manga.copy( - chapters = chapters, - description = infoElement?.selectFirst("div.entry-content")?.html(), - author = infoElement?.selectFirst(".flex-wrap div:contains(Author)")?.lastElementSibling()?.text(), - tags = infoElement?.select(".wd-full .mgen > a") - ?.mapNotNullToSet { getOrCreateTagMap()[it.text()] } - .orEmpty(), - isNsfw = docs.selectFirst(".postbody .alr") != null, - ) - } - } - - @MangaSourceParser("TOONHUNTER", "Toon Hunter", "th") - class ToonHunterParser(context: MangaLoaderContext) : - MangaReaderParser(context, MangaSource.TOONHUNTER, pageSize = 20, searchPageSize = 10) { - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("toonhunter.com", null) - - override val listUrl: String - get() = "/manga" - override val tableMode: Boolean - get() = false - - override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH) - - override suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { - val infoElement = docs.selectFirst("div.infox") - return manga.copy( - chapters = chapters, - description = infoElement?.selectFirst("div.entry-content")?.html(), - author = infoElement?.selectFirst(".flex-wrap div:contains(Author)")?.lastElementSibling()?.text(), - isNsfw = docs.selectFirst(".postbody .alr") != null, - ) - } - } - - @MangaSourceParser("COSMICSCANS", "CosmicScans", "en") - class CosmicScansParser(context: MangaLoaderContext) : - MangaReaderParser(context, MangaSource.COSMICSCANS, pageSize = 20, searchPageSize = 10) { - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("cosmicscans.com", null) - - override val listUrl: String - get() = "/manga" - override val tableMode: Boolean - get() = false - - override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH) - - override suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { - val infoElement = docs.selectFirst("div.infox") - return manga.copy( - chapters = chapters, - description = infoElement?.selectFirst("div.entry-content")?.html(), - author = infoElement?.selectFirst(".flex-wrap div:contains(Author)")?.lastElementSibling()?.text(), - tags = infoElement?.select(".wd-full .mgen > a") - ?.mapNotNullToSet { getOrCreateTagMap()[it.text()] } - .orEmpty(), - ) - } - } - - @MangaSourceParser("KOMIKLOKAL", "KomikLokal", "id") - class KomikLokalParser(context: MangaLoaderContext) : - MangaReaderParser(context, MangaSource.KOMIKLOKAL, pageSize = 20, searchPageSize = 10) { - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("komiklokal.pics", null) - - override val listUrl: String - get() = "/manga" - override val tableMode: Boolean - get() = false - - override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH) - - override suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { - val infoElement = docs.selectFirst("div.infox") - return manga.copy( - chapters = chapters, - tags = infoElement?.select(".wd-full .mgen > a") - ?.mapNotNullToSet { getOrCreateTagMap()[it.text()] } - .orEmpty(), - ) - } - } - - @MangaSourceParser("KOMIKAV", "KomiKav", "id") - class KomiKavParser(context: MangaLoaderContext) : - MangaReaderParser(context, MangaSource.KOMIKAV, pageSize = 20, searchPageSize = 10) { - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("komikav.com", null) - - override val listUrl: String - get() = "/manga" - override val tableMode: Boolean - get() = false - - override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH) - - override suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { - val infoElement = docs.selectFirst("div.infox") - return manga.copy( - chapters = chapters, - description = infoElement?.selectFirst("div.entry-content")?.html(), - author = infoElement?.selectFirst(".flex-wrap div:contains(Author)")?.lastElementSibling()?.text(), - tags = infoElement?.select(".wd-full .mgen > a") - ?.mapNotNullToSet { getOrCreateTagMap()[it.text()] } - .orEmpty(), - ) - } - } - - @MangaSourceParser("KOMIKDEWASA", "KomikDewasa", "id") - class KomikDewasaParser(context: MangaLoaderContext) : - MangaReaderParser(context, MangaSource.KOMIKDEWASA, pageSize = 20, searchPageSize = 10) { - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("komikdewasa.club", null) - - override val listUrl: String - get() = "/manga" - override val tableMode: Boolean - get() = false - - override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH) - - override suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { - val infoElement = docs.selectFirst("div.infox") - return manga.copy( - chapters = chapters, - description = infoElement?.selectFirst("div.entry-content")?.html(), - ) - } - } - - @MangaSourceParser("MANGASUSU", "Mangasusu", "id") - class MangasusuParser(context: MangaLoaderContext) : - MangaReaderParser(context, MangaSource.MANGASUSU, pageSize = 20, searchPageSize = 10) { - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("mangasusu.co.in", null) - - override val listUrl: String - get() = "/project" - override val tableMode: Boolean - get() = true - - override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH) - } - - @MangaSourceParser("KOMIKLAB", "KomikLab", "id") - class KomikLabParser(context: MangaLoaderContext) : - MangaReaderParser(context, MangaSource.KOMIKLAB, pageSize = 20, searchPageSize = 10) { - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("komiklab.com", null) - - override val listUrl: String - get() = "/project" - override val tableMode: Boolean - get() = true - - override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH) - } - - @MangaSourceParser("KOMIKINDO", "KomikIndo", "id") - class KomikIndoParser(context: MangaLoaderContext) : - MangaReaderParser(context, MangaSource.KOMIKINDO, pageSize = 20, searchPageSize = 10) { - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("komikindo.co", null) - - override val listUrl: String - get() = "/project" - override val tableMode: Boolean - get() = true - - override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", idLocale) - } - - @MangaSourceParser("KOMIKMANGA", "KomikManga", "id") - class KomikMangaParser(context: MangaLoaderContext) : - MangaReaderParser(context, MangaSource.KOMIKMANGA, pageSize = 20, searchPageSize = 10) { - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("komikmanga.xyz", null) - - override val listUrl: String - get() = "/project" - override val tableMode: Boolean - get() = true - - override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH) - } + override val sortOrders: Set + get() = EnumSet.of(SortOrder.UPDATED, SortOrder.POPULARITY, SortOrder.ALPHABETICAL, SortOrder.NEWEST) + + protected val idLocale + get() = Locale("in", "ID") + + abstract val listUrl: String + abstract val tableMode: Boolean + open val chapterDateFormat = SimpleDateFormat("MMM d, yyyy", idLocale) + + private var tagCache: ArrayMap? = null + private val mutex = Mutex() + private var lastSearchPage = 1 + + override suspend fun getDetails(manga: Manga): Manga { + val docs = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val chapters = docs.select("#chapterlist > ul > li").reversed().mapChapters { index, element -> + val url = element.selectFirst("a")?.attrAsRelativeUrl("href") ?: return@mapChapters null + MangaChapter( + id = generateUid(url), + name = element.selectFirst(".chapternum")?.text() ?: "Chapter ${index + 1}", + url = url, + number = index + 1, + scanlator = null, + uploadDate = chapterDateFormat.tryParse(element.selectFirst(".chapterdate")?.text()), + branch = null, + source = source, + ) + } + return if (tableMode) parseInfoTable(docs, manga, chapters) else parseInfoList(docs, manga, chapters) + } + + open suspend fun parseInfoTable(docs: Document, manga: Manga, chapters: List): Manga { + val mangaInfo = docs.selectFirst("div.seriestucontent > div.seriestucontentr") + val mangaState = mangaInfo?.selectFirst(".infotable td:contains(Status)")?.lastElementSibling()?.let { + when (it.text()) { + "Ongoing" -> MangaState.ONGOING + "Completed" -> MangaState.FINISHED + else -> null + } + } + val tagMap = getOrCreateTagMap() + val tags = mangaInfo?.select(".seriestugenre > a")?.mapNotNullToSet { tagMap[it.text()] } + + return manga.copy( + description = mangaInfo?.selectFirst("div.entry-content")?.html(), + state = mangaState, + author = mangaInfo?.selectFirst(".infotable td:contains(Author)")?.lastElementSibling()?.text(), + isNsfw = docs.selectFirst(".restrictcontainer") != null, + tags = tags.orEmpty(), + chapters = chapters, + ) + } + + open suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { + val mangaState = docs.selectFirst(".info-left .tsinfo div:contains(Status)")?.lastElementChild()?.let { + when (it.text()) { + "Ongoing" -> MangaState.ONGOING + "Completed" -> MangaState.FINISHED + else -> null + } + } + val tagMap = getOrCreateTagMap() + val tags = docs.select(".info-right .mgen > a").mapNotNullToSet { tagMap[it.text()] } + + return manga.copy( + description = docs.selectFirst(".info-right div.entry-content > p")?.html(), + state = mangaState, + author = docs.selectFirst(".info-left .tsinfo div:contains(Author)")?.lastElementChild()?.text(), + isNsfw = docs.selectFirst(".info-right .alr") != null, + tags = tags, + chapters = chapters, + ) + } + + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + if (!query.isNullOrEmpty()) { + if (page > lastSearchPage) { + return emptyList() + } + + val url = buildString { + append("https://") + append(domain) + append("/page/") + append(page) + append("/?s=") + append(query.urlEncoded()) + } + + val docs = webClient.httpGet(url).parseHtml() + lastSearchPage = docs.selectFirst(".pagination .next") + ?.previousElementSibling() + ?.text()?.toIntOrNull() ?: 1 + return parseMangaList(docs) + } + + val sortQuery = when (sortOrder) { + SortOrder.ALPHABETICAL -> "title" + SortOrder.NEWEST -> "latest" + SortOrder.POPULARITY -> "popular" + SortOrder.UPDATED -> "update" + else -> "" + } + val tagKey = "genre[]".urlEncoded() + val tagQuery = + if (tags.isNullOrEmpty()) "" else tags.joinToString(separator = "&", prefix = "&") { "$tagKey=${it.key}" } + val url = buildString { + append("https://") + append(domain) + append(listUrl) + append("/?order=") + append(sortQuery) + append(tagQuery) + append("&page=") + append(page) + } + + return parseMangaList(webClient.httpGet(url).parseHtml()) + } + + private fun parseMangaList(docs: Document): List { + return docs.select(".postbody .listupd .bs .bsx").mapNotNull { + val a = it.selectFirst("a") ?: return@mapNotNull null + val relativeUrl = a.attrAsRelativeUrl("href") + val rating = it.selectFirst(".numscore")?.text() + ?.toFloatOrNull()?.div(10) ?: RATING_UNKNOWN + + Manga( + id = generateUid(relativeUrl), + url = relativeUrl, + title = a.attr("title"), + altTitle = null, + publicUrl = a.attrAsAbsoluteUrl("href"), + rating = rating, + isNsfw = false, + coverUrl = it.selectFirst("img.ts-post-image")?.imageUrl().orEmpty(), + tags = emptySet(), + state = null, + author = null, + source = source, + ) + } + } + + override suspend fun getPages(chapter: MangaChapter): List { + val chapterUrl = chapter.url.toAbsoluteUrl(domain) + val docs = webClient.httpGet(chapterUrl).parseHtml() + val script = docs.selectFirstOrThrow("script:containsData(ts_reader)") + val images = JSONObject(script.data().substringAfter('(').substringBeforeLast(')')) + .getJSONArray("sources") + .getJSONObject(0) + .getJSONArray("images") + + val pages = ArrayList(images.length()) + for (i in 0 until images.length()) { + pages.add( + MangaPage( + id = generateUid(images.getString(i)), + url = images.getString(i), + referer = chapterUrl, + preview = null, + source = source, + ), + ) + } + + return pages + } + + override suspend fun getTags(): Set { + return getOrCreateTagMap().values.toSet() + } + + protected suspend fun getOrCreateTagMap(): Map = mutex.withLock { + tagCache?.let { return@withLock it } + val tagMap = ArrayMap() + + val url = listUrl.toAbsoluteUrl(domain) + val tagElements = webClient.httpGet(url).parseHtml().select("ul.genrez > li") + for (el in tagElements) { + if (el.text().isEmpty()) continue + + tagMap[el.text()] = MangaTag( + title = el.text(), + key = el.selectFirst("input")?.attr("value") ?: continue, + source = source, + ) + } + + tagCache = tagMap + return@withLock tagMap + } + + private fun Element.imageUrl(): String { + return attrAsAbsoluteUrlOrNull("src") + ?: attrAsAbsoluteUrlOrNull("data-cfsrc") + ?: "" + } + + @MangaSourceParser("MANHWALAND", "Manhwaland", "id") + class ManhwaLandParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.MANHWALAND, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("manhwaland.guru", null) + + override val listUrl: String + get() = "/series" + override val tableMode: Boolean + get() = false + } + + @MangaSourceParser("SEKAIKOMIK", "Sekaikomik", "id") + class SekaikomikParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.SEKAIKOMIK, pageSize = 20, searchPageSize = 100) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("sekaikomik.pro", null) + + override val listUrl: String + get() = "/manga" + override val tableMode: Boolean + get() = false + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMMM D, yyyy", idLocale) + } + + @MangaSourceParser("MANHWAINDO", "Manhwaindo", "id") + class ManhwaIndoParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.MANHWAINDO, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("manhwaindo.id", null) + + override val listUrl: String get() = "/series" + override val tableMode: Boolean get() = false + } + + @MangaSourceParser("MANHWALIST", "Manhwalist", "id") + class ManhwalistParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.MANHWALIST, pageSize = 24, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("manhwalist.in", null) + + override val listUrl: String = "/manga" + override val tableMode: Boolean get() = false + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMMM d, yyyy", Locale.ENGLISH) + } + + @MangaSourceParser("KIRYUU", "Kiryuu", "id") + class KiryuuParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.KIRYUU, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("kiryuu.id", null) + + override val listUrl: String + get() = "/manga" + override val tableMode: Boolean + get() = true + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMMM d, yyyy", idLocale) + } + + @MangaSourceParser("TURKTOON", "Turktoon", "tr") + class TurktoonParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.TURKTOON, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("turktoon.com", null) + + override val listUrl: String + get() = "/manga" + override val tableMode: Boolean + get() = false + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMMM d, yyyy", Locale("tr", "TR")) + + override suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { + val mangaState = docs.selectFirst(".info-left .tsinfo div:contains(Durum)")?.lastElementChild()?.let { + when (it.text()) { + "Devam Ediyor" -> MangaState.ONGOING + "Tamamlandı" -> MangaState.FINISHED + else -> null + } + } + + return super.parseInfoList(docs, manga, chapters).copy(state = mangaState) + } + } + + @MangaSourceParser("WESTMANGA", "Westmanga", "id") + class WestmangaParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.WESTMANGA, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("westmanga.info", null) + + override val listUrl: String + get() = "/manga" + override val tableMode: Boolean + get() = true + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMMM d, yyyy", Locale.ENGLISH) + } + + @MangaSourceParser("TEMPESTFANSUB", "Tempestfansub", "tr") + class TempestfansubParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.TEMPESTFANSUB, pageSize = 40, searchPageSize = 40) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("tempestscans.com", null) + + override val listUrl: String get() = "/manga" + override val tableMode: Boolean get() = true + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMMM d, yyyy", Locale("tr", "TR")) + + override suspend fun parseInfoTable(docs: Document, manga: Manga, chapters: List): Manga { + val infoElement = docs.selectFirst("div.infox") + return manga.copy( + chapters = chapters, + description = infoElement?.selectFirst("div.entry-content")?.html(), + author = infoElement?.selectFirst(".flex-wrap div:contains(Artist)")?.lastElementSibling()?.text(), + tags = infoElement?.select(".wd-full .mgen > a") + ?.mapNotNullToSet { getOrCreateTagMap()[it.text()] } + .orEmpty(), + isNsfw = docs.selectFirst(".postbody .alr") != null, + ) + } + } + + @MangaSourceParser("MANHWADESU", "ManhwaDesu", "id") + class ManhwadesuParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.MANHWADESU, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("manhwadesu.org", null) + + override val listUrl: String get() = "/komik" + override val tableMode: Boolean get() = false + } + + @MangaSourceParser("MANGATALE", "MangaTale", "id") + class MangaTaleParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.MANGATALE, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("mangatale.co", null) + + override val listUrl: String get() = "/manga" + override val tableMode: Boolean get() = false + + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMMM d, yyyy", Locale.ENGLISH) + + override suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { + val infoElement = docs.selectFirst("div.infox") + return manga.copy( + chapters = chapters, + description = infoElement?.selectFirst("div.entry-content")?.html(), + author = infoElement?.selectFirst(".flex-wrap div:contains(Author)")?.lastElementSibling()?.text(), + tags = infoElement?.select(".wd-full .mgen > a") + ?.mapNotNullToSet { getOrCreateTagMap()[it.text()] } + .orEmpty(), + isNsfw = docs.selectFirst(".postbody .alr") != null, + ) + } + } + + @MangaSourceParser("DRAGONTRANSLATION", "DragonTranslation", "es") + class DragonTranslationParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.DRAGONTRANSLATION, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("dragontranslation.com", null) + + override val listUrl: String get() = "/manga" + override val tableMode: Boolean get() = false + + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMMM d, yyyy", Locale.ENGLISH) + + override suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { + val infoElement = docs.selectFirst("div.infox") + return manga.copy( + chapters = chapters, + description = infoElement?.selectFirst("div.entry-content")?.html(), + author = infoElement?.selectFirst(".flex-wrap div:contains(Author)")?.lastElementSibling()?.text(), + tags = infoElement?.select(".wd-full .mgen > a") + ?.mapNotNullToSet { getOrCreateTagMap()[it.text()] } + .orEmpty(), + isNsfw = docs.selectFirst(".postbody .alr") != null, + ) + } + } + + @MangaSourceParser("ASURATR", "Asura Scans (tr)", "tr") + class AsuraTRParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.ASURATR, pageSize = 30, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("asurascanstr.com", null) + + override val listUrl: String get() = "/manga" + override val tableMode: Boolean get() = false + + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", Locale("tr")) + + override suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { + val infoElement = docs.selectFirst("div.infox") + return manga.copy( + chapters = chapters, + description = infoElement?.selectFirst("div.entry-content")?.html(), + author = infoElement?.selectFirst(".flex-wrap div:contains(Yazar)")?.lastElementSibling()?.text(), + tags = infoElement?.select(".wd-full .mgen > a") + ?.mapNotNullToSet { getOrCreateTagMap()[it.text()] } + .orEmpty(), + isNsfw = docs.selectFirst(".postbody .alr") != null, + ) + } + } + + @MangaSourceParser("KOMIKTAP", "KomikTap", "id") + class KomikTapParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.KOMIKTAP, pageSize = 15, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("92.87.6.124", null) + + override val listUrl: String + get() = "/manga" + override val tableMode: Boolean + get() = true + + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", idLocale) + } + + @MangaSourceParser("KUMAPOI", "KumaPoi", "id") + class KumaPoiParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.KUMAPOI, pageSize = 15, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("kumapoi.me", null) + + override val listUrl: String + get() = "/manga" + override val tableMode: Boolean + get() = true + + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH) + } + + @MangaSourceParser("ASURASCANS", "Asura Scans", "en") + class AsuraScansParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.ASURASCANS, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("asurascans.com", null) + + override val listUrl: String + get() = "/manga" + override val tableMode: Boolean + get() = false + + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH) + + override suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { + val infoElement = docs.selectFirst("div.infox") + return manga.copy( + chapters = chapters, + description = infoElement?.selectFirst("div.entry-content")?.html(), + author = infoElement?.selectFirst(".flex-wrap div:contains(Author)")?.lastElementSibling()?.text(), + tags = infoElement?.select(".wd-full .mgen > a") + ?.mapNotNullToSet { getOrCreateTagMap()[it.text()] } + .orEmpty(), + isNsfw = docs.selectFirst(".postbody .alr") != null, + ) + } + } + + @MangaSourceParser("TOONHUNTER", "Toon Hunter", "th") + class ToonHunterParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.TOONHUNTER, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("toonhunter.com", null) + + override val listUrl: String + get() = "/manga" + override val tableMode: Boolean + get() = false + + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH) + + override suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { + val infoElement = docs.selectFirst("div.infox") + return manga.copy( + chapters = chapters, + description = infoElement?.selectFirst("div.entry-content")?.html(), + author = infoElement?.selectFirst(".flex-wrap div:contains(Author)")?.lastElementSibling()?.text(), + isNsfw = docs.selectFirst(".postbody .alr") != null, + ) + } + } + + @MangaSourceParser("COSMICSCANS", "CosmicScans", "en") + class CosmicScansParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.COSMICSCANS, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("cosmicscans.com", null) + + override val listUrl: String + get() = "/manga" + override val tableMode: Boolean + get() = false + + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH) + + override suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { + val infoElement = docs.selectFirst("div.infox") + return manga.copy( + chapters = chapters, + description = infoElement?.selectFirst("div.entry-content")?.html(), + author = infoElement?.selectFirst(".flex-wrap div:contains(Author)")?.lastElementSibling()?.text(), + tags = infoElement?.select(".wd-full .mgen > a") + ?.mapNotNullToSet { getOrCreateTagMap()[it.text()] } + .orEmpty(), + ) + } + } + + @MangaSourceParser("KOMIKLOKAL", "KomikLokal", "id") + class KomikLokalParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.KOMIKLOKAL, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("komiklokal.pics", null) + + override val listUrl: String + get() = "/manga" + override val tableMode: Boolean + get() = false + + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH) + + override suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { + val infoElement = docs.selectFirst("div.infox") + return manga.copy( + chapters = chapters, + tags = infoElement?.select(".wd-full .mgen > a") + ?.mapNotNullToSet { getOrCreateTagMap()[it.text()] } + .orEmpty(), + ) + } + } + + @MangaSourceParser("KOMIKAV", "KomiKav", "id") + class KomiKavParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.KOMIKAV, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("komikav.com", null) + + override val listUrl: String + get() = "/manga" + override val tableMode: Boolean + get() = false + + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH) + + override suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { + val infoElement = docs.selectFirst("div.infox") + return manga.copy( + chapters = chapters, + description = infoElement?.selectFirst("div.entry-content")?.html(), + author = infoElement?.selectFirst(".flex-wrap div:contains(Author)")?.lastElementSibling()?.text(), + tags = infoElement?.select(".wd-full .mgen > a") + ?.mapNotNullToSet { getOrCreateTagMap()[it.text()] } + .orEmpty(), + ) + } + } + + @MangaSourceParser("KOMIKDEWASA", "KomikDewasa", "id") + class KomikDewasaParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.KOMIKDEWASA, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("komikdewasa.club", null) + + override val listUrl: String + get() = "/manga" + override val tableMode: Boolean + get() = false + + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH) + + override suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { + val infoElement = docs.selectFirst("div.infox") + return manga.copy( + chapters = chapters, + description = infoElement?.selectFirst("div.entry-content")?.html(), + ) + } + } + + @MangaSourceParser("MANGASUSU", "Mangasusu", "id") + class MangasusuParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.MANGASUSU, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("mangasusu.co.in", null) + + override val listUrl: String + get() = "/project" + override val tableMode: Boolean + get() = true + + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH) + } + + @MangaSourceParser("KOMIKLAB", "KomikLab", "id") + class KomikLabParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.KOMIKLAB, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("komiklab.com", null) + + override val listUrl: String + get() = "/project" + override val tableMode: Boolean + get() = true + + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH) + } + + @MangaSourceParser("KOMIKINDO", "KomikIndo", "id") + class KomikIndoParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.KOMIKINDO, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("komikindo.co", null) + + override val listUrl: String + get() = "/project" + override val tableMode: Boolean + get() = true + + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", idLocale) + } + + @MangaSourceParser("KOMIKMANGA", "KomikManga", "id") + class KomikMangaParser(context: MangaLoaderContext) : + MangaReaderParser(context, MangaSource.KOMIKMANGA, pageSize = 20, searchPageSize = 10) { + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("komikmanga.xyz", null) + + override val listUrl: String + get() = "/project" + override val tableMode: Boolean + get() = true + + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH) + } } From 413f4a2f102a89ea73270cf04f3120f581ab4c97 Mon Sep 17 00:00:00 2001 From: Koitharu Date: Wed, 8 Mar 2023 19:43:23 +0200 Subject: [PATCH 083/121] Batch parsers fixes --- .../koitharu/kotatsu/parsers/MangaParser.kt | 218 +++++------ .../kotatsu/parsers/site/CloneMangaParser.kt | 164 ++++---- .../kotatsu/parsers/site/Manhwa18Parser.kt | 356 +++++++++--------- .../kotatsu/parsers/site/UnionMangasParser.kt | 302 +++++++-------- .../parsers/site/madara/IsekaiScanEuParser.kt | 30 +- .../parsers/site/madara/Madara5Parser.kt | 336 ++++++++--------- .../parsers/site/madara/Madara6Parser.kt | 99 +++-- .../parsers/site/madara/MadaraParser.kt | 6 +- .../kotatsu/parsers/site/madara/NeatManga.kt | 29 +- .../parsers/site/madara/PrismaScansParser.kt | 67 ++-- .../site/mangareader/MangaReaderParser.kt | 10 +- .../koitharu/kotatsu/parsers/util/Jsoup.kt | 6 +- .../kotatsu/parsers/MangaParserTest.kt | 8 +- 13 files changed, 844 insertions(+), 787 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaParser.kt index a99d6d64..84ee386a 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaParser.kt @@ -12,115 +12,115 @@ import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl import java.util.* abstract class MangaParser @InternalParsersApi constructor( - @property:InternalParsersApi val context: MangaLoaderContext, - val source: MangaSource, + @property:InternalParsersApi val context: MangaLoaderContext, + val source: MangaSource, ) { - /** - * Supported [SortOrder] variants. Must not be empty. - * - * For better performance use [EnumSet] for more than one item. - */ - abstract val sortOrders: Set - - val config by lazy { context.getConfig(source) } - - open val sourceLocale: Locale? - get() = source.locale?.let { Locale(it) } - - /** - * Provide default domain and available alternatives, if any. - * - * Never hardcode domain in requests, use [getDomain] instead. - */ - @InternalParsersApi - abstract val configKeyDomain: ConfigKey.Domain - - open val headers: Headers? = null - - /** - * Used as fallback if value of `sortOrder` passed to [getList] is null - */ - protected open val defaultSortOrder: SortOrder - get() { - val supported = sortOrders - return SortOrder.values().first { it in supported } - } - - protected val webClient: WebClient = OkHttpWebClient(context.httpClient, source) - - /** - * Parse list of manga by specified criteria - * - * @param offset starting from 0 and used for pagination. - * Note than passed value may not be divisible by internal page size, so you should adjust it manually. - * @param query search query, may be null or empty if no search needed - * @param tags genres for filtering, values from [getTags] and [Manga.tags]. May be null or empty - * @param sortOrder one of [sortOrders] or null for default value - */ - @JvmSynthetic - @InternalParsersApi - abstract suspend fun getList( - offset: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List - - /** - * Parse list of manga with search by text query - * - * @param offset starting from 0 and used for pagination. - * @param query search query - */ - open suspend fun getList(offset: Int, query: String): List { - return getList(offset, query, null, defaultSortOrder) - } - - /** - * Parse list of manga by specified criteria - * - * @param offset starting from 0 and used for pagination. - * Note than passed value may not be divisible by internal page size, so you should adjust it manually. - * @param tags genres for filtering, values from [getTags] and [Manga.tags]. May be null or empty - * @param sortOrder one of [sortOrders] or null for default value - */ - open suspend fun getList(offset: Int, tags: Set?, sortOrder: SortOrder?): List { - return getList(offset, null, tags, sortOrder ?: defaultSortOrder) - } - - /** - * Parse details for [Manga]: chapters list, description, large cover, etc. - * Must return the same manga, may change any fields excepts id, url and source - * @see Manga.copy - */ - abstract suspend fun getDetails(manga: Manga): Manga - - /** - * Parse pages list for specified chapter. - * @see MangaPage for details - */ - abstract suspend fun getPages(chapter: MangaChapter): List - - /** - * Fetch direct link to the page image. - */ - open suspend fun getPageUrl(page: MangaPage): String = page.url.toAbsoluteUrl(domain) - - /** - * Fetch available tags (genres) for source - */ - abstract suspend fun getTags(): Set - - /** - * Parse favicons from the main page of the source`s website - */ - open suspend fun getFavicons(): Favicons { - return FaviconParser(webClient, domain).parseFavicons() - } - - @CallSuper - open fun onCreateConfig(keys: MutableCollection>) { - keys.add(configKeyDomain) - } + /** + * Supported [SortOrder] variants. Must not be empty. + * + * For better performance use [EnumSet] for more than one item. + */ + abstract val sortOrders: Set + + val config by lazy { context.getConfig(source) } + + open val sourceLocale: Locale + get() = source.locale?.let { Locale(it) } ?: Locale.ROOT + + /** + * Provide default domain and available alternatives, if any. + * + * Never hardcode domain in requests, use [getDomain] instead. + */ + @InternalParsersApi + abstract val configKeyDomain: ConfigKey.Domain + + open val headers: Headers? = null + + /** + * Used as fallback if value of `sortOrder` passed to [getList] is null + */ + protected open val defaultSortOrder: SortOrder + get() { + val supported = sortOrders + return SortOrder.values().first { it in supported } + } + + protected val webClient: WebClient = OkHttpWebClient(context.httpClient, source) + + /** + * Parse list of manga by specified criteria + * + * @param offset starting from 0 and used for pagination. + * Note than passed value may not be divisible by internal page size, so you should adjust it manually. + * @param query search query, may be null or empty if no search needed + * @param tags genres for filtering, values from [getTags] and [Manga.tags]. May be null or empty + * @param sortOrder one of [sortOrders] or null for default value + */ + @JvmSynthetic + @InternalParsersApi + abstract suspend fun getList( + offset: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List + + /** + * Parse list of manga with search by text query + * + * @param offset starting from 0 and used for pagination. + * @param query search query + */ + open suspend fun getList(offset: Int, query: String): List { + return getList(offset, query, null, defaultSortOrder) + } + + /** + * Parse list of manga by specified criteria + * + * @param offset starting from 0 and used for pagination. + * Note than passed value may not be divisible by internal page size, so you should adjust it manually. + * @param tags genres for filtering, values from [getTags] and [Manga.tags]. May be null or empty + * @param sortOrder one of [sortOrders] or null for default value + */ + open suspend fun getList(offset: Int, tags: Set?, sortOrder: SortOrder?): List { + return getList(offset, null, tags, sortOrder ?: defaultSortOrder) + } + + /** + * Parse details for [Manga]: chapters list, description, large cover, etc. + * Must return the same manga, may change any fields excepts id, url and source + * @see Manga.copy + */ + abstract suspend fun getDetails(manga: Manga): Manga + + /** + * Parse pages list for specified chapter. + * @see MangaPage for details + */ + abstract suspend fun getPages(chapter: MangaChapter): List + + /** + * Fetch direct link to the page image. + */ + open suspend fun getPageUrl(page: MangaPage): String = page.url.toAbsoluteUrl(domain) + + /** + * Fetch available tags (genres) for source + */ + abstract suspend fun getTags(): Set + + /** + * Parse favicons from the main page of the source`s website + */ + open suspend fun getFavicons(): Favicons { + return FaviconParser(webClient, domain).parseFavicons() + } + + @CallSuper + open fun onCreateConfig(keys: MutableCollection>) { + keys.add(configKeyDomain) + } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/CloneMangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/CloneMangaParser.kt index 889aa81c..56fd66e0 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/CloneMangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/CloneMangaParser.kt @@ -10,94 +10,92 @@ import java.util.* @MangaSourceParser("CLONEMANGA", "CloneManga", "en") internal class CloneMangaParser(context: MangaLoaderContext) : PagedMangaParser( - context, - MangaSource.CLONEMANGA, - pageSize = 1, + context, + MangaSource.CLONEMANGA, + pageSize = 1, ) { - override val sortOrders: Set = Collections.singleton( - SortOrder.POPULARITY, - ) + override val sortOrders: Set = Collections.singleton( + SortOrder.POPULARITY, + ) - override val configKeyDomain = ConfigKey.Domain("manga.clone-army.org", null) + override val configKeyDomain = ConfigKey.Domain("manga.clone-army.org", null) - override suspend fun getListPage( - page: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - if (query != null || page > 1) { - return emptyList() - } - val link = "https://${domain}/viewer_landing.php" - val doc = webClient.httpGet(link).parseHtml() - val mangas = doc.getElementsByClass("comicPreviewContainer") - return mangas.mapNotNull { item -> - val attr = item.getElementsByClass("comicPreview").attr("style") - val href = item.selectFirst("a")?.attrAsAbsoluteUrl("href") ?: return@mapNotNull null - val cover = attr.substring(attr.indexOf("site/themes"), attr.indexOf(")")) - Manga( - id = generateUid(href), - title = item.selectFirst("h3")?.text() ?: return@mapNotNull null, - coverUrl = "https://${domain}/$cover", - altTitle = null, - author = "Dan Kim", - rating = RATING_UNKNOWN, - url = href, - isNsfw = false, - tags = emptySet(), - state = null, - publicUrl = href.toAbsoluteUrl(domain), - source = source, - ) - } - } + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + if (query != null || page > 1) { + return emptyList() + } + val link = "https://${domain}/viewer_landing.php" + val doc = webClient.httpGet(link).parseHtml() + val mangas = doc.getElementsByClass("comicPreviewContainer") + return mangas.mapNotNull { item -> + val background = item.selectFirstOrThrow(".comicPreview").styleValueOrNull("background") + val href = item.selectFirst("a")?.attrAsAbsoluteUrl("href") ?: return@mapNotNull null + val cover = background?.substring(background.indexOf("site/themes"), background.indexOf(")")) + Manga( + id = generateUid(href), + title = item.selectFirst("h3")?.text() ?: return@mapNotNull null, + coverUrl = "https://${domain}/$cover", + altTitle = null, + author = "Dan Kim", + rating = RATING_UNKNOWN, + url = href, + isNsfw = false, + tags = emptySet(), + state = null, + publicUrl = href.toAbsoluteUrl(domain), + source = source, + ) + } + } - override suspend fun getDetails(manga: Manga): Manga { - val doc = webClient.httpGet(manga.publicUrl).parseHtml() - val series = doc.location() - val numChapters = Regex( - pattern = "&page=(.*)&lang=", - ).findAll( - input = doc.getElementsByTag("script")[3].toString(), - ) - .elementAt(3).destructured.component1() - .toInt() - val chapters = ArrayList() - for (i in 0..numChapters) { - val chapter = MangaChapter( - id = generateUid("$series&page=$i"), - name = "Chapter ${i + 1}", - number = i + 1, - url = "$series&page=$i", - scanlator = null, - branch = null, - uploadDate = 0L, - source = MangaSource.DUMMY, - ) - chapters.add(chapter) - } - return manga.copy(chapters = chapters) - } + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.publicUrl).parseHtml() + val series = doc.location() + val numChapters = Regex( + pattern = "&page=(.*)&lang=", + ).findAll( + input = doc.getElementsByTag("script")[3].toString(), + ) + .elementAt(3).destructured.component1() + .toInt() + val chapters = ArrayList() + for (i in 0..numChapters) { + val chapter = MangaChapter( + id = generateUid("$series&page=$i"), + name = "Chapter ${i + 1}", + number = i + 1, + url = "$series&page=$i", + scanlator = null, + branch = null, + uploadDate = 0L, + source = source, + ) + chapters.add(chapter) + } + return manga.copy(chapters = chapters) + } - override suspend fun getPages(chapter: MangaChapter): List { - val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() - val imgUrl = doc.getElementsByClass("subsectionContainer")[0] - .selectFirst("img") - ?.attrAsAbsoluteUrlOrNull("src") ?: doc.parseFailed("Something broken") - return listOf( - MangaPage( - id = generateUid(imgUrl), - url = imgUrl, - referer = imgUrl, - preview = null, - source = MangaSource.DUMMY, - ), - ) - } + override suspend fun getPages(chapter: MangaChapter): List { + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + val imgUrl = doc.getElementsByClass("subsectionContainer")[0] + .selectFirst("img") + ?.attrAsAbsoluteUrlOrNull("src") ?: doc.parseFailed("Something broken") + return listOf( + MangaPage( + id = generateUid(imgUrl), + url = imgUrl, + referer = imgUrl, + preview = null, + source = source, + ), + ) + } - override suspend fun getTags(): Set { - return emptySet() - } + override suspend fun getTags(): Set = emptySet() } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/Manhwa18Parser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/Manhwa18Parser.kt index 97a40b94..55bc37cd 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/Manhwa18Parser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/Manhwa18Parser.kt @@ -1,5 +1,6 @@ package org.koitharu.kotatsu.parsers.site +import androidx.collection.ArrayMap import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaSourceParser import org.koitharu.kotatsu.parsers.PagedMangaParser @@ -10,176 +11,187 @@ import java.util.* @MangaSourceParser("MANHWA18", "Manhwa18", "en") class Manhwa18Parser(context: MangaLoaderContext) : - PagedMangaParser(context, MangaSource.MANHWA18, pageSize = 20, searchPageSize = 20) { - - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("manhwa18.net", null) - - override val sortOrders: Set - get() = EnumSet.of(SortOrder.UPDATED, SortOrder.POPULARITY, SortOrder.ALPHABETICAL) - - override suspend fun getFavicons(): Favicons { - return Favicons( - listOf( - Favicon("https://${domain}/uploads/logos/logo-mini.png", 92, null), - ), - domain, - ) - } - - override suspend fun getDetails(manga: Manga): Manga { - val docs = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - val cardInfoElement = docs.selectFirst(".card .manga-info") - val author = cardInfoElement?.selectFirst("b:contains(Author(s))")?.parent() - ?.select("a.btn") - ?.joinToString(", ") { it.text() } - val tags = cardInfoElement?.selectFirst("b:contains(Genre(s))")?.parent() - ?.select("a.btn") - ?.mapToSet { MangaTag(it.text(), it.text().lowercase(), MangaSource.MANHWA18) } - val state = cardInfoElement?.selectFirst("b:contains(Status)")?.parent() - ?.selectFirst("a.btn") - ?.let { - when (it.text()) { - "On going" -> MangaState.ONGOING - "Completed" -> MangaState.FINISHED - else -> null - } - } - - return manga.copy( - altTitle = cardInfoElement?.selectFirst("b:contains(Other names)")?.parent()?.ownText()?.removePrefix(": "), - author = author, - description = docs.selectFirst(".series-summary .summary-content")?.html(), - tags = tags.orEmpty(), - state = state, - chapters = docs.select(".card-body > .list-chapters > a").asReversed().mapChapters { index, element -> - // attrAsRelativeUrl only return page url without the '/' - val chapterUrl = element.attrAsAbsoluteUrlOrNull("href")?.toRelativeUrl(domain) - ?: return@mapChapters null - val uploadDate = parseUploadDate(element.selectFirst(".chapter-time")?.text()) - MangaChapter( - id = generateUid(chapterUrl), - name = element.selectFirst(".chapter-name")?.text().orEmpty(), - number = index + 1, - url = chapterUrl, - scanlator = null, - uploadDate = uploadDate, - branch = null, - source = MangaSource.MANHWA18, - ) - }, - ) - } - - // 7 minutes ago - // 5 hours ago - // 2 days ago - // 2 weeks ago - // 4 years ago - private fun parseUploadDate(timeStr: String?): Long { - timeStr ?: return 0 - - val timeWords = timeStr.split(' ') - if (timeWords.size != 3) return 0 - val timeWord = timeWords[1] - val timeAmount = timeWords[0].toIntOrNull() ?: return 0 - val timeUnit = when (timeWord) { - "minute", "minutes" -> Calendar.MINUTE - "hour", "hours" -> Calendar.HOUR - "day", "days" -> Calendar.DAY_OF_YEAR - "week", "weeks" -> Calendar.WEEK_OF_YEAR - "month", "months" -> Calendar.MONTH - "year", "years" -> Calendar.YEAR - else -> return 0 - } - val cal = Calendar.getInstance() - cal.add(timeUnit, -timeAmount) - return cal.time.time - } - - override suspend fun getListPage( - page: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - val sortQuery = when (sortOrder) { - SortOrder.ALPHABETICAL -> "name" - SortOrder.POPULARITY -> "views" - SortOrder.UPDATED -> "last_update" - else -> "" - } - - val sortType = if (sortOrder == SortOrder.ALPHABETICAL) "ASC" else "DESC" - val tagQuery = tags?.joinToString(",") { it.key }.orEmpty() - val url = buildString { - append("https://") - append(domain) - append("/manga-list.html?listType=pagination&page=") - append(page) - append("&artist=&author=&group=&m_status=&name=") - append(query?.urlEncoded().orEmpty()) - append("&genre=$tagQuery") - append("&ungenre=") - append("&sort=") - append(sortQuery) - append("&sort_type=") - append(sortType) - } - - val docs = webClient.httpGet(url).parseHtml() - val actualPage = docs.selectFirst("ul.pagination a.active")?.text()?.toIntOrNull() - if (actualPage != page) { - return emptyList() - } - - return docs.select(".card-body .thumb-item-flow") - .map { - val titleElement = it.selectFirstOrThrow(".thumb_attr.series-title > a") - val absUrl = titleElement.attrAsAbsoluteUrl("href") - Manga( - id = generateUid(absUrl.toRelativeUrl(domain)), - title = titleElement.text(), - altTitle = null, - url = absUrl.toRelativeUrl(domain), - publicUrl = absUrl, - rating = RATING_UNKNOWN, - isNsfw = true, - coverUrl = it.selectFirst("div.img-in-ratio")?.attrAsAbsoluteUrl("data-bg").orEmpty(), - tags = emptySet(), - state = null, - author = null, - largeCoverUrl = null, - description = null, - source = MangaSource.MANHWA18, - ) - } - } - - override suspend fun getPages(chapter: MangaChapter): List { - val chapterUrl = chapter.url.toAbsoluteUrl(domain) - return webClient.httpGet(chapterUrl).parseHtml() - .select("div.chapter-content > img").map { - val url = it.attrAsRelativeUrlOrNull("src").orEmpty() - MangaPage( - id = generateUid(url), - url = url, - referer = chapterUrl, - preview = null, - source = MangaSource.MANHWA18, - ) - } - } - - override suspend fun getTags(): Set { - return webClient.httpGet("https://${domain}/").parseHtml().selectFirstOrThrow(".genres-menu") - .select("a.genres-item").orEmpty() - .mapToSet { - MangaTag( - title = it.text(), - key = it.text().lowercase(), - source = MangaSource.MANHWA18, - ) - } - } + PagedMangaParser(context, MangaSource.MANHWA18, pageSize = 20, searchPageSize = 20) { + + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("manhwa18.net", null) + + override val sortOrders: Set + get() = EnumSet.of(SortOrder.UPDATED, SortOrder.POPULARITY, SortOrder.ALPHABETICAL, SortOrder.NEWEST) + + private val tagsMap = SuspendLazy(::parseTags) + + override suspend fun getFavicons(): Favicons { + return Favicons( + listOf( + Favicon("https://${domain}/uploads/logos/logo-mini.png", 92, null), + ), + domain, + ) + } + + override suspend fun getDetails(manga: Manga): Manga { + val docs = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val cardInfoElement = docs.selectFirst(".card .manga-info") + val author = cardInfoElement?.selectFirst("b:contains(Author(s))")?.parent() + ?.select("a.btn") + ?.joinToString(", ") { it.text() } + val availableTags = tagsMap.get() + val tags = cardInfoElement?.selectFirst("b:contains(Genre(s))")?.parent() + ?.select("a.btn") + ?.mapNotNullToSet { availableTags[it.text().lowercase(Locale.ENGLISH)] } + val state = cardInfoElement?.selectFirst("b:contains(Status)")?.parent() + ?.selectFirst("a.btn") + ?.let { + when (it.text()) { + "On going" -> MangaState.ONGOING + "Completed" -> MangaState.FINISHED + else -> null + } + } + + return manga.copy( + altTitle = cardInfoElement?.selectFirst("b:contains(Other names)")?.parent()?.ownText()?.removePrefix(": "), + author = author, + description = docs.selectFirst(".series-summary .summary-content")?.html(), + tags = tags.orEmpty(), + state = state, + chapters = docs.select(".card-body > .list-chapters > a").asReversed().mapChapters { index, element -> + // attrAsRelativeUrl only return page url without the '/' + val chapterUrl = element.attrAsAbsoluteUrlOrNull("href")?.toRelativeUrl(domain) + ?: return@mapChapters null + val uploadDate = parseUploadDate(element.selectFirst(".chapter-time")?.text()) + MangaChapter( + id = generateUid(chapterUrl), + name = element.selectFirst(".chapter-name")?.text().orEmpty(), + number = index + 1, + url = chapterUrl, + scanlator = null, + uploadDate = uploadDate, + branch = null, + source = MangaSource.MANHWA18, + ) + }, + ) + } + + // 7 minutes ago + // 5 hours ago + // 2 days ago + // 2 weeks ago + // 4 years ago + private fun parseUploadDate(timeStr: String?): Long { + timeStr ?: return 0 + + val timeWords = timeStr.split(' ') + if (timeWords.size != 3) return 0 + val timeWord = timeWords[1] + val timeAmount = timeWords[0].toIntOrNull() ?: return 0 + val timeUnit = when (timeWord) { + "minute", "minutes" -> Calendar.MINUTE + "hour", "hours" -> Calendar.HOUR + "day", "days" -> Calendar.DAY_OF_YEAR + "week", "weeks" -> Calendar.WEEK_OF_YEAR + "month", "months" -> Calendar.MONTH + "year", "years" -> Calendar.YEAR + else -> return 0 + } + val cal = Calendar.getInstance() + cal.add(timeUnit, -timeAmount) + return cal.time.time + } + + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + val sortQuery = when (sortOrder) { + SortOrder.ALPHABETICAL -> "az" + SortOrder.POPULARITY -> "top" + SortOrder.UPDATED -> "update" + SortOrder.NEWEST -> "new" + else -> "" + } + + val tagQuery = tags?.joinToString(",") { it.key }.orEmpty() + val url = buildString { + append("https://") + append(domain) + append("/tim-kiem?page=") + append(page) + if (!query.isNullOrEmpty()) { + append("&q=") + append(query.urlEncoded()) + } + append("&accept_genres=$tagQuery") + append("&sort=") + append(sortQuery) + } + + val docs = webClient.httpGet(url).parseHtml() + + return docs.select(".card-body .thumb-item-flow") + .map { + val titleElement = it.selectFirstOrThrow(".thumb_attr.series-title > a") + val absUrl = titleElement.attrAsAbsoluteUrl("href") + Manga( + id = generateUid(absUrl.toRelativeUrl(domain)), + title = titleElement.text(), + altTitle = null, + url = absUrl.toRelativeUrl(domain), + publicUrl = absUrl, + rating = RATING_UNKNOWN, + isNsfw = true, + coverUrl = it.selectFirst("div.img-in-ratio")?.attrAsAbsoluteUrl("data-bg").orEmpty(), + tags = emptySet(), + state = null, + author = null, + largeCoverUrl = null, + description = null, + source = MangaSource.MANHWA18, + ) + } + } + + override suspend fun getPages(chapter: MangaChapter): List { + val chapterUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(chapterUrl).parseHtml() + return doc.requireElementById("chapter-content").select("img").mapNotNull { + val url = it.attrAsRelativeUrlOrNull("data-src") + ?: it.attrAsRelativeUrlOrNull("src") + ?: return@mapNotNull null + MangaPage( + id = generateUid(url), + url = url, + referer = chapterUrl, + preview = null, + source = MangaSource.MANHWA18, + ) + } + } + + override suspend fun getTags(): Set { + return tagsMap.get().values.toSet() + } + + private suspend fun parseTags(): Map { + val doc = webClient.httpGet("https://$domain/tim-kiem?q=").parseHtml() + val list = doc.getElementsByAttribute("data-genre-id") + if (list.isEmpty()) { + return emptyMap() + } + val result = ArrayMap(list.size) + for (item in list) { + val id = item.attr("data-genre-id") + val name = item.text() + result[name.lowercase(Locale.ENGLISH)] = MangaTag( + title = name.toTitleCase(Locale.ENGLISH), + key = id, + source = source, + ) + } + return result + } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/UnionMangasParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/UnionMangasParser.kt index ea07c0fc..53b959d0 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/UnionMangasParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/UnionMangasParser.kt @@ -15,163 +15,163 @@ import java.util.* @MangaSourceParser("UNION_MANGAS", "Union Mangás", "pt") class UnionMangasParser(context: MangaLoaderContext) : PagedMangaParser(context, MangaSource.UNION_MANGAS, 40) { - override val sortOrders = EnumSet.of( - SortOrder.ALPHABETICAL, - SortOrder.POPULARITY, - ) + override val sortOrders = EnumSet.of( + SortOrder.ALPHABETICAL, + SortOrder.POPULARITY, + ) - override val configKeyDomain = ConfigKey.Domain("unionleitor.top", emptyArray()) + override val configKeyDomain = ConfigKey.Domain("unionleitor.top", emptyArray()) - override suspend fun getListPage( - page: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - if (!query.isNullOrEmpty()) { - return if (page == searchPaginator.firstPage) { - search(query) - } else { - emptyList() - } - } - val tag = tags.oneOrThrowIfMany() - val url = urlBuilder() - .addPathSegment("lista-mangas") - .addPathSegment( - when { - tag != null -> tag.key - sortOrder == SortOrder.ALPHABETICAL -> "a-z" - else -> "visualizacoes" - }, - ).addPathSegment(page.toString()) - val doc = webClient.httpGet(url.build()).parseHtml() - val root = doc.selectFirstOrThrow("div.tamanho-bloco-perfil") - return root.select(".lista-mangas-novos").map { div -> - val a = div.selectFirstOrThrow("a") - val img = div.selectFirstOrThrow("img") - val href = a.attrAsRelativeUrl("href") - Manga( - id = generateUid(href), - url = href, - publicUrl = a.attrAsAbsoluteUrl("href"), - title = div.selectLastOrThrow("a").text(), - coverUrl = img.attrAsAbsoluteUrl("src"), - altTitle = null, - rating = RATING_UNKNOWN, - tags = emptySet(), - description = div.selectLast("div")?.ownText(), - state = null, - author = null, - isNsfw = false, - source = source, - ) - } - } + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + if (!query.isNullOrEmpty()) { + return if (page == searchPaginator.firstPage) { + search(query) + } else { + emptyList() + } + } + val tag = tags.oneOrThrowIfMany() + val url = urlBuilder() + .addPathSegment("lista-mangas") + .addPathSegment( + when { + tag != null -> tag.key + sortOrder == SortOrder.ALPHABETICAL -> "a-z" + else -> "visualizacoes" + }, + ).addPathSegment(page.toString()) + val doc = webClient.httpGet(url.build()).parseHtml() + val root = doc.selectFirstOrThrow("div.tamanho-bloco-perfil") + return root.select(".lista-mangas-novos").map { div -> + val a = div.selectFirstOrThrow("a") + val img = div.selectFirstOrThrow("img") + val href = a.attrAsRelativeUrl("href") + Manga( + id = generateUid(href), + url = href, + publicUrl = a.attrAsAbsoluteUrl("href"), + title = div.selectLastOrThrow("a").text(), + coverUrl = img.attrAsAbsoluteUrl("src"), + altTitle = null, + rating = RATING_UNKNOWN, + tags = emptySet(), + description = div.selectLast("div")?.ownText(), + state = null, + author = null, + isNsfw = false, + source = source, + ) + } + } - override suspend fun getDetails(manga: Manga): Manga { - val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - val root = doc.selectFirstOrThrow(".perfil-manga") - val dateFormat = SimpleDateFormat("dd/MM/yyyy", Locale.ROOT) - return manga.copy( - rating = root.select("h2") - .find { it.ownText().startsWith('#') } - ?.ownText()?.drop(1)?.toFloatOrNull()?.div(10f) ?: manga.rating, - largeCoverUrl = root.selectFirst("img.img-thumbnail")?.attrAsAbsoluteUrlOrNull("src"), - description = root.selectFirst(".panel-default")?.selectFirst(".panel-body")?.html(), - author = root.tableValue("Autor")?.ownText(), - altTitle = root.tableValue("Título(s) Alternativo(s)")?.ownText(), - state = when (root.tableValue("Status")?.selectLast(".label")?.text()) { - "Completo" -> MangaState.FINISHED - "Ativo" -> MangaState.ONGOING - else -> null - }, - tags = root.tableValue("Gênero(s)")?.select("a")?.mapToSet { - it.toMangaTag() - } ?: manga.tags, - isNsfw = root.selectFirst(".alert-danger")?.html()?.contains("18 anos") == true, - chapters = root.select("div.row.capitulos").asReversed().mapChapters { i, div -> - val a = div.selectFirstOrThrow("a") - val href = a.attrAsRelativeUrl("href") - val title = a.text() - MangaChapter( - id = generateUid(href), - name = title, - number = i + 1, - url = href, - scanlator = div.selectLast("a")?.text()?.takeUnless { it == title }, - uploadDate = dateFormat.tryParse( - a.nextElementSibling()?.text()?.removeSurrounding("(", ")"), - ), - branch = null, - source = source, - ) - }, - ) - } + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val root = doc.selectFirstOrThrow(".perfil-manga") + val dateFormat = SimpleDateFormat("dd/MM/yyyy", Locale.ROOT) + return manga.copy( + rating = root.select("h2") + .find { it.ownText().startsWith('#') } + ?.ownText()?.drop(1)?.toFloatOrNull()?.div(10f) ?: manga.rating, + largeCoverUrl = root.selectFirst("img.img-thumbnail")?.attrAsAbsoluteUrlOrNull("src"), + description = root.selectFirst(".panel-default")?.selectFirst(".panel-body")?.html(), + author = root.tableValue("Autor")?.ownText(), + altTitle = root.tableValue("Título(s) Alternativo(s)")?.ownText(), + state = when (root.tableValue("Status")?.selectLast(".label")?.text()) { + "Completo" -> MangaState.FINISHED + "Ativo" -> MangaState.ONGOING + else -> null + }, + tags = root.tableValue("Gênero(s)")?.select("a")?.mapToSet { + it.toMangaTag() + } ?: manga.tags, + isNsfw = root.selectFirst(".alert-danger")?.html()?.contains("18 anos") == true, + chapters = root.select("div.row.capitulos").asReversed().mapChapters { i, div -> + val a = div.selectFirstOrThrow("a") + val href = a.attrAsRelativeUrl("href") + val title = a.text() + MangaChapter( + id = generateUid(href), + name = title, + number = i + 1, + url = href, + scanlator = div.selectLast("a")?.text()?.takeUnless { it == title }, + uploadDate = dateFormat.tryParse( + a.nextElementSibling()?.text()?.removeSurrounding("(", ")"), + ), + branch = null, + source = source, + ) + }, + ) + } - override suspend fun getPages(chapter: MangaChapter): List { - val fullUrl = chapter.url.toAbsoluteUrl(domain) - val doc = webClient.httpGet(fullUrl).parseHtml() - val root = doc.body().selectFirstOrThrow("article") - return root.selectOrThrow("img[pag]").mapNotNull { img -> - val href = img.attrAsRelativeUrl("src") - if (href.startsWith("/images/banner")) { - return@mapNotNull null - } - MangaPage( - id = generateUid(href), - url = href, - referer = fullUrl, - preview = null, - source = source, - ) - } - } + override suspend fun getPages(chapter: MangaChapter): List { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + val root = doc.body().selectFirstOrThrow("article") + return root.selectOrThrow("img[pag]").mapNotNull { img -> + val href = img.attrAsRelativeUrl("src") + if (href.startsWith("/images/banner")) { + return@mapNotNull null + } + MangaPage( + id = generateUid(href), + url = href, + referer = fullUrl, + preview = null, + source = source, + ) + } + } - override suspend fun getTags(): Set { - val doc = webClient.httpGet(urlBuilder().addPathSegment("lista-mangas").build()).parseHtml() - val ul = doc.body().selectFirstOrThrow(".nav-tabs").selectFirstOrThrow("ul.dropdown-menu") - return ul.select("li").mapToSet { li -> - li.selectFirstOrThrow("a").toMangaTag() - } - } + override suspend fun getTags(): Set { + val doc = webClient.httpGet(urlBuilder().addPathSegment("lista-mangas").build()).parseHtml() + val ul = doc.body().selectFirstOrThrow(".nav-tabs").selectFirstOrThrow("ul.dropdown-menu") + return ul.select("li").mapToSet { li -> + li.selectFirstOrThrow("a").toMangaTag() + } + } - private suspend fun search(query: String): List { - val domain = domain - val json = webClient.httpGet( - urlBuilder() - .addPathSegments("assets/busca.php") - .addQueryParameter("nomeManga", query) - .build(), - ).parseJson() - return json.getJSONArray("items").mapJSON { jo -> - val href = "/pagina-manga/" + jo.getString("url") - Manga( - id = generateUid(href), - url = href, - publicUrl = href.toAbsoluteUrl(domain), - title = jo.getString("titulo"), - rating = RATING_UNKNOWN, - tags = emptySet(), - author = jo.getStringOrNull("autor"), - coverUrl = jo.getString("imagem"), - state = null, - isNsfw = false, - altTitle = null, - source = source, - ) - } - } + private suspend fun search(query: String): List { + val domain = domain + val json = webClient.httpGet( + urlBuilder() + .addPathSegments("assets/busca.php") + .addQueryParameter("nomeManga", query) + .build(), + ).parseJson() + return json.getJSONArray("items").mapJSON { jo -> + val href = "/pagina-manga/" + jo.getString("url") + Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + title = jo.getString("titulo"), + rating = RATING_UNKNOWN, + tags = emptySet(), + author = jo.getStringOrNull("autor"), + coverUrl = jo.getString("imagem"), + state = null, + isNsfw = false, + altTitle = null, + source = source, + ) + } + } - private fun Element.tableValue(title: String): Element? { - return select("h4.media-heading") - .find { it.selectFirst("label.subtit-manga")?.text()?.contains(title, ignoreCase = true) == true } - } + private fun Element.tableValue(title: String): Element? { + return select("h4.media-heading") + .find { it.selectFirst("label.subtit-manga")?.text()?.contains(title, ignoreCase = true) == true } + } - private fun Element.toMangaTag() = MangaTag( - title = text().toTitleCase(sourceLocale ?: Locale.ROOT), - key = attr("href").removeSuffix('/').substringAfterLast('/'), - source = source, - ) + private fun Element.toMangaTag() = MangaTag( + title = text().toTitleCase(sourceLocale), + key = attr("href").removeSuffix('/').substringAfterLast('/'), + source = source, + ) } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/IsekaiScanEuParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/IsekaiScanEuParser.kt index 3ec177c7..49b506a6 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/IsekaiScanEuParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/IsekaiScanEuParser.kt @@ -6,13 +6,14 @@ import org.koitharu.kotatsu.parsers.MangaSourceParser import org.koitharu.kotatsu.parsers.model.Manga import org.koitharu.kotatsu.parsers.model.MangaChapter import org.koitharu.kotatsu.parsers.model.MangaSource +import org.koitharu.kotatsu.parsers.model.MangaTag import org.koitharu.kotatsu.parsers.util.* import java.text.SimpleDateFormat import java.util.* -@MangaSourceParser("ISEKAISCAN_EU", "IsekaiScan (eu)", "en") +@MangaSourceParser("ISEKAISCAN_EU", "IsekaiScan", "en") internal class IsekaiScanEuParser(context: MangaLoaderContext) : - MadaraParser(context, MangaSource.ISEKAISCAN_EU, "isekaiscan.eu") { + MadaraParser(context, MangaSource.ISEKAISCAN_EU, "isekaiscan.to") { override val datePattern = "MM/dd/yyyy" @@ -44,4 +45,29 @@ internal class IsekaiScanEuParser(context: MangaLoaderContext) : ) } } + + override suspend fun getTags(): Set { + val doc = webClient.httpGet("https://$domain/mangax/").parseHtml() + val body = doc.body() + val root1 = body.selectFirst("header")?.selectFirst("ul.second-menu") + val root2 = body.selectFirst("div.genres_wrap")?.selectFirst("ul.list-unstyled") + if (root1 == null && root2 == null) { + doc.parseFailed("Root not found") + } + val list = root1?.select("li").orEmpty() + root2?.select("li").orEmpty() + val keySet = HashSet(list.size) + return list.mapNotNullToSet { li -> + val a = li.selectFirst("a") ?: return@mapNotNullToSet null + val href = a.attr("href").removeSuffix("/") + .substringAfterLast(tagPrefix, "") + if (href.isEmpty() || !keySet.add(href)) { + return@mapNotNullToSet null + } + MangaTag( + key = href, + title = a.ownText().toTitleCase(Locale.ENGLISH), + source = source, + ) + } + } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Madara5Parser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Madara5Parser.kt index 3cc26196..1b480d01 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Madara5Parser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Madara5Parser.kt @@ -13,174 +13,174 @@ import java.text.SimpleDateFormat import java.util.* abstract class Madara5Parser @InternalParsersApi constructor( - context: MangaLoaderContext, - source: MangaSource, - domain: String, + context: MangaLoaderContext, + source: MangaSource, + domain: String, ) : PagedMangaParser(context, source, pageSize = 22) { - protected open val datePattern = "MMMM dd, HH:mm" - protected open val tagPrefix = "/mangas/" - protected open val nsfwTags = arraySetOf("yaoi", "yuri", "mature") - - override val sortOrders: Set = EnumSet.of(SortOrder.UPDATED) - - override val configKeyDomain = ConfigKey.Domain(domain, null) - - override suspend fun getListPage( - page: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - val domain = domain - val url = buildString { - append("https://") - append(domain) - append("/search?s=") - if (!query.isNullOrEmpty()) { - append(query.urlEncoded()) - } - append("&post_type=wp-manga") - if (!tags.isNullOrEmpty()) { - for (tag in tags) { - append("&genre%5B%5D=") - append(tag.key) - } - } - append("&op=1&author=&artist=&page=") - append(page) - } - val root = webClient.httpGet(url).parseHtml().body().selectFirstOrThrow(".search-wrap") - return root.select(".c-tabs-item__content").map { div -> - val a = div.selectFirstOrThrow("a") - val img = div.selectLastOrThrow("img") - val href = a.attrAsRelativeUrl("href") - val postContent = root.selectFirstOrThrow(".post-content") - val tagSet = postContent.getElementsContainingOwnText("Genre") - .firstOrNull()?.tableValue() - ?.getElementsByAttributeValueContaining("href", tagPrefix) - ?.mapToSet { it.asMangaTag() }.orEmpty() - Manga( - id = generateUid(href), - title = a.attr("title"), - altTitle = postContent.getElementsContainingOwnText("Alternative") - .firstOrNull()?.tableValue()?.text()?.trim(), - url = href, - publicUrl = a.attrAsAbsoluteUrl("href"), - coverUrl = img.src().orEmpty(), - author = postContent.getElementsContainingOwnText("Author") - .firstOrNull()?.tableValue()?.text()?.trim(), - state = postContent.getElementsContainingOwnText("Status") - .firstOrNull()?.tableValue()?.text()?.asMangaState(), - isNsfw = isNsfw(tagSet), - rating = div.selectFirstOrThrow(".score").text() - .toFloatOrNull()?.div(5f) ?: RATING_UNKNOWN, - tags = tagSet, - source = source, - ) - } - } - - override suspend fun getDetails(manga: Manga): Manga { - val root = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml().body() - .selectFirstOrThrow(".site-content") - val postContent = root.selectFirstOrThrow(".post-content") - val tags = postContent.getElementsContainingOwnText("Genre") - .firstOrNull()?.tableValue() - ?.getElementsByAttributeValueContaining("href", tagPrefix) - ?.mapToSet { a -> a.asMangaTag() } ?: manga.tags - val mangaId = root.getElementById("manga-chapters-holder")?.attr("data-id")?.toLongOrNull() - ?: root.parseFailed("Cannot find mangaId") - return manga.copy( - description = (root.selectFirst(".detail-content") - ?: root.selectFirstOrThrow(".description-summary")).html(), - author = postContent.getElementsContainingOwnText("Author") - .firstOrNull()?.tableValue()?.text()?.trim(), - state = postContent.getElementsContainingOwnText("Status") - .firstOrNull()?.tableValue()?.text()?.asMangaState(), - tags = tags, - isNsfw = isNsfw(tags), - chapters = loadChapters(mangaId), - ) - } - - override suspend fun getPages(chapter: MangaChapter): List { - val fullUrl = chapter.url.toAbsoluteUrl(domain) - val doc = webClient.httpGet(fullUrl).parseHtml() - val arrayData = doc.getElementById("arraydata") ?: doc.parseFailed("#arraydata not found") - return arrayData.html().split(',').map { url -> - MangaPage( - id = generateUid(url), - url = url, - referer = fullUrl, - preview = null, - source = source, - ) - } - } - - override suspend fun getTags(): Set { - val doc = webClient.httpGet("http://${domain}/").parseHtml().body() - return doc.getElementsByAttributeValueContaining("href", tagPrefix) - .mapToSet { it.asMangaTag() } - } - - private suspend fun loadChapters(mangaId: Long): List { - val dateFormat = SimpleDateFormat(datePattern, sourceLocale ?: Locale.US) - val doc = webClient.httpGet("https://${domain}/ajax-list-chapter?mangaID=$mangaId").parseHtml() - return doc.select("li.wp-manga-chapter").asReversed().mapChapters { i, li -> - val a = li.selectFirstOrThrow("a") - val href = a.attrAsRelativeUrl("href") - MangaChapter( - id = generateUid(href), - url = href, - name = a.text(), - number = i + 1, - branch = null, - uploadDate = dateFormat.tryParse( - li.selectFirst(".chapter-release-date")?.text()?.trim(), - ), - scanlator = null, - source = source, - ) - } - } - - protected fun isNsfw(tags: Set): Boolean { - return tags.any { it.key in nsfwTags } - } - - private fun Element.src(): String? { - return absUrl("data-src").ifEmpty { - absUrl("src") - }.takeUnless { it.isEmpty() } - } - - private fun Element.tableValue(): Element { - for (p in parents()) { - val children = p.children() - if (children.size == 2) { - return children[1] - } - } - parseFailed("Cannot find tableValue for node ${text()}") - } - - private fun String.asMangaState() = when (trim().lowercase(sourceLocale ?: Locale.US)) { - "ongoing" -> MangaState.ONGOING - "completed" -> MangaState.FINISHED - else -> null - } - - private fun Element.asMangaTag() = MangaTag( - title = ownText(), - key = attr("href").removeSuffix('/').substringAfterLast('/') - .replace('-', '+'), - source = source, - ) - - @MangaSourceParser("MANGAOWLS", "BeautyManga", "en") - class BeautyManga(context: MangaLoaderContext) : Madara5Parser(context, MangaSource.MANGAOWLS, "beautymanga.com") { - - } + protected open val datePattern = "MMMM dd, HH:mm" + protected open val tagPrefix = "/mangas/" + protected open val nsfwTags = arraySetOf("yaoi", "yuri", "mature") + + override val sortOrders: Set = EnumSet.of(SortOrder.UPDATED) + + override val configKeyDomain = ConfigKey.Domain(domain, null) + + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + val domain = domain + val url = buildString { + append("https://") + append(domain) + append("/search?s=") + if (!query.isNullOrEmpty()) { + append(query.urlEncoded()) + } + append("&post_type=wp-manga") + if (!tags.isNullOrEmpty()) { + for (tag in tags) { + append("&genre%5B%5D=") + append(tag.key) + } + } + append("&op=1&author=&artist=&page=") + append(page) + } + val root = webClient.httpGet(url).parseHtml().body().selectFirstOrThrow(".search-wrap") + return root.select(".c-tabs-item__content").map { div -> + val a = div.selectFirstOrThrow("a") + val img = div.selectLastOrThrow("img") + val href = a.attrAsRelativeUrl("href") + val postContent = root.selectFirstOrThrow(".post-content") + val tagSet = postContent.getElementsContainingOwnText("Genre") + .firstOrNull()?.tableValue() + ?.getElementsByAttributeValueContaining("href", tagPrefix) + ?.mapToSet { it.asMangaTag() }.orEmpty() + Manga( + id = generateUid(href), + title = a.attr("title"), + altTitle = postContent.getElementsContainingOwnText("Alternative") + .firstOrNull()?.tableValue()?.text()?.trim(), + url = href, + publicUrl = a.attrAsAbsoluteUrl("href"), + coverUrl = img.src().orEmpty(), + author = postContent.getElementsContainingOwnText("Author") + .firstOrNull()?.tableValue()?.text()?.trim(), + state = postContent.getElementsContainingOwnText("Status") + .firstOrNull()?.tableValue()?.text()?.asMangaState(), + isNsfw = isNsfw(tagSet), + rating = div.selectFirstOrThrow(".score").text() + .toFloatOrNull()?.div(5f) ?: RATING_UNKNOWN, + tags = tagSet, + source = source, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val root = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml().body() + .selectFirstOrThrow(".site-content") + val postContent = root.selectFirstOrThrow(".post-content") + val tags = postContent.getElementsContainingOwnText("Genre") + .firstOrNull()?.tableValue() + ?.getElementsByAttributeValueContaining("href", tagPrefix) + ?.mapToSet { a -> a.asMangaTag() } ?: manga.tags + val mangaId = root.getElementById("manga-chapters-holder")?.attr("data-id")?.toLongOrNull() + ?: root.parseFailed("Cannot find mangaId") + return manga.copy( + description = (root.selectFirst(".detail-content") + ?: root.selectFirstOrThrow(".description-summary")).html(), + author = postContent.getElementsContainingOwnText("Author") + .firstOrNull()?.tableValue()?.text()?.trim(), + state = postContent.getElementsContainingOwnText("Status") + .firstOrNull()?.tableValue()?.text()?.asMangaState(), + tags = tags, + isNsfw = isNsfw(tags), + chapters = loadChapters(mangaId), + ) + } + + override suspend fun getPages(chapter: MangaChapter): List { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + val arrayData = doc.getElementById("arraydata") ?: doc.parseFailed("#arraydata not found") + return arrayData.html().split(',').map { url -> + MangaPage( + id = generateUid(url), + url = url, + referer = fullUrl, + preview = null, + source = source, + ) + } + } + + override suspend fun getTags(): Set { + val doc = webClient.httpGet("http://${domain}/").parseHtml().body() + return doc.getElementsByAttributeValueContaining("href", tagPrefix) + .mapToSet { it.asMangaTag() } + } + + private suspend fun loadChapters(mangaId: Long): List { + val dateFormat = SimpleDateFormat(datePattern, sourceLocale) + val doc = webClient.httpGet("https://${domain}/ajax-list-chapter?mangaID=$mangaId").parseHtml() + return doc.select("li.wp-manga-chapter").asReversed().mapChapters { i, li -> + val a = li.selectFirstOrThrow("a") + val href = a.attrAsRelativeUrl("href") + MangaChapter( + id = generateUid(href), + url = href, + name = a.text(), + number = i + 1, + branch = null, + uploadDate = dateFormat.tryParse( + li.selectFirst(".chapter-release-date")?.text()?.trim(), + ), + scanlator = null, + source = source, + ) + } + } + + protected fun isNsfw(tags: Set): Boolean { + return tags.any { it.key in nsfwTags } + } + + private fun Element.src(): String? { + return absUrl("data-src").ifEmpty { + absUrl("src") + }.takeUnless { it.isEmpty() } + } + + private fun Element.tableValue(): Element { + for (p in parents()) { + val children = p.children() + if (children.size == 2) { + return children[1] + } + } + parseFailed("Cannot find tableValue for node ${text()}") + } + + private fun String.asMangaState() = when (trim().lowercase(sourceLocale)) { + "ongoing" -> MangaState.ONGOING + "completed" -> MangaState.FINISHED + else -> null + } + + private fun Element.asMangaTag() = MangaTag( + title = ownText(), + key = attr("href").removeSuffix('/').substringAfterLast('/') + .replace('-', '+'), + source = source, + ) + + @MangaSourceParser("MANGAOWLS", "BeautyManga", "en") + class BeautyManga(context: MangaLoaderContext) : Madara5Parser(context, MangaSource.MANGAOWLS, "beautymanga.com") { + + } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Madara6Parser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Madara6Parser.kt index a38f4b97..a476644b 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Madara6Parser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Madara6Parser.kt @@ -7,64 +7,63 @@ import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.model.* import org.koitharu.kotatsu.parsers.util.* import java.text.SimpleDateFormat -import java.util.* internal abstract class Madara6Parser( - context: MangaLoaderContext, - source: MangaSource, - domain: String, + context: MangaLoaderContext, + source: MangaSource, + domain: String, ) : MadaraParser(context, source, domain) { - override val datePattern: String = "dd MMMM yyyy" + override val datePattern: String = "dd MMMM yyyy" - override suspend fun getDetails(manga: Manga): Manga { - return coroutineScope { - val chapters = async { loadChapters(manga.url) } - val body = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml().body() - parseDetails(manga, body, chapters.await()) - } - } + override suspend fun getDetails(manga: Manga): Manga { + return coroutineScope { + val chapters = async { loadChapters(manga.url) } + val body = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml().body() + parseDetails(manga, body, chapters.await()) + } + } - protected fun Element.tableValue(): Element { - for (p in parents()) { - val children = p.children() - if (children.size == 2) { - return children[1] - } - } - parseFailed("Cannot find tableValue for node ${text()}") - } + protected fun Element.tableValue(): Element { + for (p in parents()) { + val children = p.children() + if (children.size == 2) { + return children[1] + } + } + parseFailed("Cannot find tableValue for node ${text()}") + } - protected abstract fun String.asMangaState(): MangaState? + protected abstract fun String.asMangaState(): MangaState? - protected fun Element.asMangaTag() = MangaTag( - title = ownText(), - key = attr("href").removeSuffix('/').substringAfterLast('/') - .replace('-', '+'), - source = source, - ) + protected fun Element.asMangaTag() = MangaTag( + title = ownText(), + key = attr("href").removeSuffix('/').substringAfterLast('/') + .replace('-', '+'), + source = source, + ) - protected open suspend fun loadChapters(mangaUrl: String): List { - val url = mangaUrl.toAbsoluteUrl(domain).removeSuffix('/') + "/ajax/chapters/" - val dateFormat = SimpleDateFormat(datePattern, sourceLocale ?: Locale.ROOT) - val doc = webClient.httpPost(url, emptyMap()).parseHtml() - return doc.select("li.wp-manga-chapter").asReversed().mapChapters { i, li -> - val a = li.selectFirstOrThrow("a") - val href = a.attrAsRelativeUrl("href") - MangaChapter( - id = generateUid(href), - url = href, - name = a.text(), - number = i + 1, - branch = null, - uploadDate = dateFormat.tryParse( - li.selectFirst(".chapter-release-date")?.text()?.trim(), - ), - scanlator = null, - source = source, - ) - } - } + protected open suspend fun loadChapters(mangaUrl: String): List { + val url = mangaUrl.toAbsoluteUrl(domain).removeSuffix('/') + "/ajax/chapters/" + val dateFormat = SimpleDateFormat(datePattern, sourceLocale) + val doc = webClient.httpPost(url, emptyMap()).parseHtml() + return doc.select("li.wp-manga-chapter").asReversed().mapChapters { i, li -> + val a = li.selectFirstOrThrow("a") + val href = a.attrAsRelativeUrl("href") + MangaChapter( + id = generateUid(href), + url = href, + name = a.text(), + number = i + 1, + branch = null, + uploadDate = dateFormat.tryParse( + li.selectFirst(".chapter-release-date")?.text()?.trim(), + ), + scanlator = null, + source = source, + ) + } + } - protected abstract fun parseDetails(manga: Manga, body: Element, chapters: List): Manga + protected abstract fun parseDetails(manga: Manga, body: Element, chapters: List): Manga } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt index d5e40952..e03e8550 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt @@ -153,7 +153,7 @@ internal abstract class MadaraParser( protected open suspend fun getChapters(manga: Manga, doc: Document): List { val root2 = doc.body().selectFirstOrThrow("div.content-area") .selectFirstOrThrow("div.c-page") - val dateFormat = SimpleDateFormat(datePattern, sourceLocale ?: Locale.US) + val dateFormat = SimpleDateFormat(datePattern, sourceLocale) return root2.select("li").asReversed().mapChapters { i, li -> val a = li.selectFirst("a") val href = a?.attrAsRelativeUrlOrNull("href") ?: li.parseFailed("Link is missing") @@ -285,11 +285,11 @@ internal abstract class MadaraParser( } } - private fun Element.src(): String? { + protected fun Element.src(): String? { var result = absUrl("data-src") if (result.isEmpty()) result = absUrl("data-cfsrc") if (result.isEmpty()) result = absUrl("src") - return if (result.isEmpty()) null else result + return result.ifEmpty { null } } private fun createRequestTemplate() = diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/NeatManga.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/NeatManga.kt index 2e325c6b..3e8e4bb7 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/NeatManga.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/NeatManga.kt @@ -5,10 +5,7 @@ import kotlinx.coroutines.coroutineScope import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaSourceParser import org.koitharu.kotatsu.parsers.exception.ParseException -import org.koitharu.kotatsu.parsers.model.Manga -import org.koitharu.kotatsu.parsers.model.MangaChapter -import org.koitharu.kotatsu.parsers.model.MangaSource -import org.koitharu.kotatsu.parsers.model.MangaTag +import org.koitharu.kotatsu.parsers.model.* import org.koitharu.kotatsu.parsers.util.* import java.text.SimpleDateFormat import java.util.* @@ -16,6 +13,8 @@ import java.util.* @MangaSourceParser("NEATMANGA", "NeatManga", "en") internal class NeatManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.NEATMANGA, "neatmangas.com") { + override val datePattern = "dd MMMM yyyy" + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { val chaptersDeferred = async { getChapters(manga) } val fullUrl = manga.url.toAbsoluteUrl(domain) @@ -72,4 +71,26 @@ internal class NeatManga(context: MangaLoaderContext) : MadaraParser(context, Ma ) } } + + override suspend fun getPages(chapter: MangaChapter): List { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + val root = doc.body().selectFirst("div.main-col-inner") + ?.selectFirst("div.reading-content") + ?: throw ParseException("Root not found", fullUrl) + return root.select("div.page-break").mapNotNull { div -> + val img = div.selectFirst("img") + if (img == null || img.attr("id").isNullOrEmpty()) { + return@mapNotNull null + } + val url = img.src()?.toRelativeUrl(domain) ?: div.parseFailed("Image src not found") + MangaPage( + id = generateUid(url), + url = url, + preview = null, + referer = fullUrl, + source = source, + ) + } + } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/PrismaScansParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/PrismaScansParser.kt index cea3be89..b9d0c822 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/PrismaScansParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/PrismaScansParser.kt @@ -10,45 +10,44 @@ import org.koitharu.kotatsu.parsers.model.MangaState import org.koitharu.kotatsu.parsers.util.attrAsAbsoluteUrlOrNull import org.koitharu.kotatsu.parsers.util.mapToSet import org.koitharu.kotatsu.parsers.util.selectFirstOrThrow -import java.util.* @MangaSourceParser("PRISMA_SCANS", "Prisma Scans", "pt") internal class PrismaScansParser(context: MangaLoaderContext) : - Madara6Parser(context, MangaSource.PRISMA_SCANS, "prismascans.net") { + Madara6Parser(context, MangaSource.PRISMA_SCANS, "prismascans.net") { - override val tagPrefix = "manga-genre/" - override val datePattern = "MMM dd, yyyy" + override val tagPrefix = "manga-genre/" + override val datePattern = "MMM dd, yyyy" - override fun parseDetails(manga: Manga, body: Element, chapters: List): Manga { - val root = body.selectFirstOrThrow(".site-content") - val postContent = root.selectFirstOrThrow(".post-content") - val tags = postContent.getElementsContainingOwnText("Gênero") - .firstOrNull()?.tableValue() - ?.getElementsByAttributeValueContaining("href", tagPrefix) - ?.mapToSet { a -> a.asMangaTag() } ?: manga.tags - return manga.copy( - largeCoverUrl = root.selectFirst("picture") - ?.selectFirst("img[data-src]") - ?.attrAsAbsoluteUrlOrNull("data-src"), - description = root.selectFirstOrThrow(".manga-excerpt").firstElementChild()?.html(), - author = postContent.getElementsContainingOwnText("Artista") - .firstOrNull()?.tableValue()?.text()?.trim(), - altTitle = postContent.getElementsContainingOwnText("Título Alternativo") - .firstOrNull()?.tableValue()?.text()?.trim(), - state = postContent.getElementsContainingOwnText("Status") - .firstOrNull()?.tableValue()?.text()?.asMangaState(), - tags = tags, - isNsfw = body.hasClass("adult-content"), - chapters = chapters, - ) - } + override fun parseDetails(manga: Manga, body: Element, chapters: List): Manga { + val root = body.selectFirstOrThrow(".site-content") + val postContent = root.selectFirstOrThrow(".post-content") + val tags = postContent.getElementsContainingOwnText("Gênero") + .firstOrNull()?.tableValue() + ?.getElementsByAttributeValueContaining("href", tagPrefix) + ?.mapToSet { a -> a.asMangaTag() } ?: manga.tags + return manga.copy( + largeCoverUrl = root.selectFirst("picture") + ?.selectFirst("img[data-src]") + ?.attrAsAbsoluteUrlOrNull("data-src"), + description = root.selectFirstOrThrow(".manga-excerpt").firstElementChild()?.html(), + author = postContent.getElementsContainingOwnText("Artista") + .firstOrNull()?.tableValue()?.text()?.trim(), + altTitle = postContent.getElementsContainingOwnText("Título Alternativo") + .firstOrNull()?.tableValue()?.text()?.trim(), + state = postContent.getElementsContainingOwnText("Status") + .firstOrNull()?.tableValue()?.text()?.asMangaState(), + tags = tags, + isNsfw = body.hasClass("adult-content"), + chapters = chapters, + ) + } - override fun String.asMangaState() = when (trim().lowercase(sourceLocale ?: Locale.ROOT)) { - "em lançamento" -> MangaState.ONGOING - "completo", - "cancelado", - -> MangaState.FINISHED + override fun String.asMangaState() = when (trim().lowercase(sourceLocale)) { + "em lançamento" -> MangaState.ONGOING + "completo", + "cancelado", + -> MangaState.FINISHED - else -> null - } + else -> null + } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt index 215a6e11..ef776dc5 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt @@ -223,6 +223,7 @@ internal abstract class MangaReaderParser( private fun Element.imageUrl(): String { return attrAsAbsoluteUrlOrNull("src") + ?: attrAsAbsoluteUrlOrNull("data-src") ?: attrAsAbsoluteUrlOrNull("data-cfsrc") ?: "" } @@ -258,6 +259,7 @@ internal abstract class MangaReaderParser( override val configKeyDomain: ConfigKey.Domain get() = ConfigKey.Domain("manhwaindo.id", null) + override val chapterDateFormat = SimpleDateFormat("MMMM dd, yyyy", Locale.ENGLISH) override val listUrl: String get() = "/series" override val tableMode: Boolean get() = false } @@ -499,7 +501,7 @@ internal abstract class MangaReaderParser( override val tableMode: Boolean get() = false - override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH) + override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", sourceLocale) override suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List): Manga { val infoElement = docs.selectFirst("div.infox") @@ -542,7 +544,7 @@ internal abstract class MangaReaderParser( class KomikLokalParser(context: MangaLoaderContext) : MangaReaderParser(context, MangaSource.KOMIKLOKAL, pageSize = 20, searchPageSize = 10) { override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("komiklokal.pics", null) + get() = ConfigKey.Domain("komikmirror.art", null) override val listUrl: String get() = "/manga" @@ -592,7 +594,7 @@ internal abstract class MangaReaderParser( class KomikDewasaParser(context: MangaLoaderContext) : MangaReaderParser(context, MangaSource.KOMIKDEWASA, pageSize = 20, searchPageSize = 10) { override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("komikdewasa.club", null) + get() = ConfigKey.Domain("komikdewasa.info", null) override val listUrl: String get() = "/manga" @@ -656,7 +658,7 @@ internal abstract class MangaReaderParser( class KomikMangaParser(context: MangaLoaderContext) : MangaReaderParser(context, MangaSource.KOMIKMANGA, pageSize = 20, searchPageSize = 10) { override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("komikmanga.xyz", null) + get() = ConfigKey.Domain("komikhentai.co", null) override val listUrl: String get() = "/project" diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Jsoup.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Jsoup.kt index f76e079e..6e5faeb8 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Jsoup.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Jsoup.kt @@ -32,7 +32,7 @@ fun Element.attrOrNull(attributeKey: String) = attr(attributeKey).takeUnless { i */ fun Element.attrAsRelativeUrlOrNull(attributeKey: String): String? { val attr = attr(attributeKey).trim() - if (attr.isEmpty()) { + if (attr.isEmpty() || attr.startsWith("data:")) { return null } if (attr.startsWith("/")) { @@ -63,7 +63,7 @@ fun Element.attrAsRelativeUrl(attributeKey: String): String { */ fun Element.attrAsAbsoluteUrlOrNull(attributeKey: String): String? { val attr = attr(attributeKey).trim() - if (attr.isEmpty()) { + if (attr.isEmpty() || attr.startsWith("data:")) { return null } return (baseUri().toHttpUrlOrNull()?.newBuilder(attr) ?: return null).toString() @@ -114,4 +114,4 @@ fun Element.selectLast(cssQuery: String): Element? { fun Element.selectLastOrThrow(cssQuery: String): Element { return selectLast(cssQuery) ?: throw ParseException("Cannot find \"$cssQuery\"", baseUri()) -} \ No newline at end of file +} diff --git a/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaParserTest.kt b/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaParserTest.kt index 337b484e..9ea50a30 100644 --- a/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaParserTest.kt +++ b/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaParserTest.kt @@ -23,7 +23,7 @@ internal class MangaParserTest { @MangaSources fun list(source: MangaSource) = runTest { val parser = source.newParser(context) - val list = parser.getList(20, sortOrder = SortOrder.POPULARITY, tags = null) + val list = parser.getList(0, sortOrder = SortOrder.POPULARITY, tags = null) checkMangaList(list, "list") assert(list.all { it.source == source }) } @@ -46,7 +46,7 @@ internal class MangaParserTest { @MangaSources fun search(source: MangaSource) = runTest { val parser = source.newParser(context) - val subject = parser.getList(20, sortOrder = SortOrder.POPULARITY, tags = null).minByOrNull { + val subject = parser.getList(0, sortOrder = SortOrder.POPULARITY, tags = null).minByOrNull { it.title.length } ?: error("No manga found") val query = subject.title @@ -84,7 +84,7 @@ internal class MangaParserTest { @MangaSources fun details(source: MangaSource) = runTest { val parser = source.newParser(context) - val list = parser.getList(20, sortOrder = SortOrder.POPULARITY, tags = null) + val list = parser.getList(0, sortOrder = SortOrder.POPULARITY, tags = null) val manga = list[3] parser.getDetails(manga).apply { assert(!chapters.isNullOrEmpty()) { "Chapters are null or empty" } @@ -113,7 +113,7 @@ internal class MangaParserTest { @MangaSources fun pages(source: MangaSource) = runTest { val parser = source.newParser(context) - val list = parser.getList(20, sortOrder = SortOrder.POPULARITY, tags = null) + val list = parser.getList(0, sortOrder = SortOrder.POPULARITY, tags = null) val manga = list.first() val chapter = parser.getDetails(manga).chapters?.firstOrNull() ?: error("Chapter is null") val pages = parser.getPages(chapter) From fc53b199150fbd2c25b175a344e17d82cfc839ea Mon Sep 17 00:00:00 2001 From: Koitharu Date: Sat, 18 Mar 2023 18:20:00 +0200 Subject: [PATCH 084/121] Add toString to models --- .../koitharu/kotatsu/parsers/model/Manga.kt | 6 +- .../kotatsu/parsers/model/MangaChapter.kt | 6 +- .../kotatsu/parsers/model/MangaPage.kt | 76 +++++++++---------- .../kotatsu/parsers/model/MangaTag.kt | 6 +- 4 files changed, 51 insertions(+), 43 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/model/Manga.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/model/Manga.kt index 9186be16..0f45a992 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/model/Manga.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/model/Manga.kt @@ -155,4 +155,8 @@ class Manga( result = 31 * result + source.hashCode() return result } -} \ No newline at end of file + + override fun toString(): String { + return "Manga($id - \"$title\" [$url] - $source)" + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/model/MangaChapter.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/model/MangaChapter.kt index 52b01e6b..4059be66 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/model/MangaChapter.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/model/MangaChapter.kt @@ -67,4 +67,8 @@ class MangaChapter( result = 31 * result + source.hashCode() return result } -} \ No newline at end of file + + override fun toString(): String { + return "MangaChapter($id - #$number [$url] - $source)" + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/model/MangaPage.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/model/MangaPage.kt index 0c5102a0..b741fac3 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/model/MangaPage.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/model/MangaPage.kt @@ -3,49 +3,45 @@ package org.koitharu.kotatsu.parsers.model import org.koitharu.kotatsu.parsers.MangaParser class MangaPage( - /** - * Unique identifier for manga - */ - val id: Long, - /** - * Relative url to page (**without** a domain) or any other uri. - * Used principally in parsers. - * May contain link to image or html page. - * @see MangaParser.getPageUrl - */ - val url: String, - /** - * Absolute link to the chapter or website home page. - * Used in Referer header - */ - @Deprecated("Referer header should be added based on source domain") - val referer: String, - /** - * Absolute url of the small page image if exists, null otherwise - */ - val preview: String?, - val source: MangaSource, + /** + * Unique identifier for manga + */ + val id: Long, + /** + * Relative url to page (**without** a domain) or any other uri. + * Used principally in parsers. + * May contain link to image or html page. + * @see MangaParser.getPageUrl + */ + val url: String, + /** + * Absolute url of the small page image if exists, null otherwise + */ + val preview: String?, + val source: MangaSource, ) { - override fun equals(other: Any?): Boolean { - if (this === other) return true - if (javaClass != other?.javaClass) return false + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (javaClass != other?.javaClass) return false - other as MangaPage + other as MangaPage - if (id != other.id) return false - if (url != other.url) return false - if (referer != other.referer) return false - if (preview != other.preview) return false - return source == other.source - } + if (id != other.id) return false + if (url != other.url) return false + if (preview != other.preview) return false + return source == other.source + } - override fun hashCode(): Int { - var result = id.hashCode() - result = 31 * result + url.hashCode() - result = 31 * result + referer.hashCode() - result = 31 * result + (preview?.hashCode() ?: 0) - result = 31 * result + source.hashCode() - return result - } + override fun hashCode(): Int { + var result = id.hashCode() + result = 31 * result + url.hashCode() + result = 31 * result + (preview?.hashCode() ?: 0) + result = 31 * result + source.hashCode() + return result + } + + override fun toString(): String { + return "MangaPage($id [$url] - $source)" + } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/model/MangaTag.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/model/MangaTag.kt index 0d9fa04b..be0f9083 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/model/MangaTag.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/model/MangaTag.kt @@ -34,4 +34,8 @@ class MangaTag( result = 31 * result + source.hashCode() return result } -} \ No newline at end of file + + override fun toString(): String { + return "MangaTag($key \"$title\" - $source)" + } +} From 93f5f70d7969c3a26a5e29b8674d8bfcd7ad8380 Mon Sep 17 00:00:00 2001 From: Koitharu Date: Sun, 19 Mar 2023 09:28:38 +0200 Subject: [PATCH 085/121] Remove referrer field from page model --- .../kotatsu/parsers/site/AnibelParser.kt | 343 ++++++----- .../kotatsu/parsers/site/BatoToParser.kt | 546 +++++++++--------- .../kotatsu/parsers/site/BlogTruyenParser.kt | 466 ++++++++------- .../kotatsu/parsers/site/CloneMangaParser.kt | 1 - .../kotatsu/parsers/site/ComickFunParser.kt | 357 ++++++------ .../kotatsu/parsers/site/DesuMeParser.kt | 1 - .../kotatsu/parsers/site/ExHentaiParser.kt | 529 +++++++++-------- .../kotatsu/parsers/site/MangaDexParser.kt | 464 ++++++++------- .../kotatsu/parsers/site/MangaInUaParser.kt | 257 ++++----- .../kotatsu/parsers/site/MangaTownParser.kt | 379 ++++++------ .../kotatsu/parsers/site/Manhwa18Parser.kt | 1 - .../kotatsu/parsers/site/NHentaiParser.kt | 317 +++++----- .../kotatsu/parsers/site/NetTruyenParser.kt | 409 +++++++------ .../parsers/site/NicovideoSeigaParser.kt | 315 +++++----- .../kotatsu/parsers/site/NineMangaParser.kt | 431 +++++++------- .../kotatsu/parsers/site/NudeMoonParser.kt | 413 +++++++------ .../kotatsu/parsers/site/RemangaParser.kt | 473 ++++++++------- .../parsers/site/TruyentranhLHParser.kt | 1 - .../kotatsu/parsers/site/UnionMangasParser.kt | 1 - .../parsers/site/grouple/GroupleParser.kt | 1 - .../parsers/site/madara/Madara5Parser.kt | 5 +- .../parsers/site/madara/MadaraParser.kt | 1 - .../kotatsu/parsers/site/madara/NeatManga.kt | 1 - .../site/mangareader/MangaReaderParser.kt | 1 - .../parsers/site/multichan/ChanParser.kt | 311 +++++----- .../parsers/site/rulib/MangaLibParser.kt | 525 +++++++++-------- 26 files changed, 3264 insertions(+), 3285 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/AnibelParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/AnibelParser.kt index dddfc47d..a50fc5b5 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/AnibelParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/AnibelParser.kt @@ -20,32 +20,32 @@ import java.util.* @MangaSourceParser("ANIBEL", "Anibel", "be") internal class AnibelParser(context: MangaLoaderContext) : MangaParser(context, MangaSource.ANIBEL) { - override val configKeyDomain = ConfigKey.Domain("anibel.net", null) + override val configKeyDomain = ConfigKey.Domain("anibel.net", null) - override val sortOrders: Set = EnumSet.of( - SortOrder.NEWEST, - ) + override val sortOrders: Set = EnumSet.of( + SortOrder.NEWEST, + ) - override suspend fun getList( - offset: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - if (!query.isNullOrEmpty()) { - return if (offset == 0) { - search(query) - } else { - emptyList() - } - } - val filters = tags?.takeUnless { it.isEmpty() }?.joinToString( - separator = ",", - prefix = "genres: [", - postfix = "]", - ) { "\"${it.key}\"" }.orEmpty() - val array = apiCall( - """ + override suspend fun getList( + offset: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + if (!query.isNullOrEmpty()) { + return if (offset == 0) { + search(query) + } else { + emptyList() + } + } + val filters = tags?.takeUnless { it.isEmpty() }?.joinToString( + separator = ",", + prefix = "genres: [", + postfix = "]", + ) { "\"${it.key}\"" }.orEmpty() + val array = apiCall( + """ getMediaList(offset: $offset, limit: 20, mediaType: manga, filters: {$filters}) { docs { mediaId @@ -62,37 +62,37 @@ internal class AnibelParser(context: MangaLoaderContext) : MangaParser(context, } } """.trimIndent(), - ).getJSONObject("getMediaList").getJSONArray("docs") - return array.mapJSON { jo -> - val mediaId = jo.getString("mediaId") - val title = jo.getJSONObject("title") - val href = "${jo.getString("mediaType")}/${jo.getString("slug")}" - Manga( - id = generateUid(mediaId), - title = title.getString("be"), - coverUrl = jo.getString("poster").removePrefix("/cdn") - .toAbsoluteUrl(getDomain("cdn")) + "?width=200&height=280", - altTitle = title.getString("alt").takeUnless(String::isEmpty), - author = null, - isNsfw = false, - rating = jo.getDouble("rating").toFloat() / 10f, - url = href, - publicUrl = "https://${domain}/$href", - tags = jo.getJSONArray("genres").mapToTags(), - state = when (jo.getString("status")) { - "ongoing" -> MangaState.ONGOING - "finished" -> MangaState.FINISHED - else -> null - }, - source = source, - ) - } - } + ).getJSONObject("getMediaList").getJSONArray("docs") + return array.mapJSON { jo -> + val mediaId = jo.getString("mediaId") + val title = jo.getJSONObject("title") + val href = "${jo.getString("mediaType")}/${jo.getString("slug")}" + Manga( + id = generateUid(mediaId), + title = title.getString("be"), + coverUrl = jo.getString("poster").removePrefix("/cdn") + .toAbsoluteUrl(getDomain("cdn")) + "?width=200&height=280", + altTitle = title.getString("alt").takeUnless(String::isEmpty), + author = null, + isNsfw = false, + rating = jo.getDouble("rating").toFloat() / 10f, + url = href, + publicUrl = "https://${domain}/$href", + tags = jo.getJSONArray("genres").mapToTags(), + state = when (jo.getString("status")) { + "ongoing" -> MangaState.ONGOING + "finished" -> MangaState.FINISHED + else -> null + }, + source = source, + ) + } + } - override suspend fun getDetails(manga: Manga): Manga { - val (type, slug) = manga.url.split('/') - val details = apiCall( - """ + override suspend fun getDetails(manga: Manga): Manga { + val (type, slug) = manga.url.split('/') + val details = apiCall( + """ media(mediaType: $type, slug: "$slug") { mediaId title { @@ -108,52 +108,52 @@ internal class AnibelParser(context: MangaLoaderContext) : MangaParser(context, genres } """.trimIndent(), - ).getJSONObject("media") - val title = details.getJSONObject("title") - val poster = details.getString("poster").removePrefix("/cdn") - .toAbsoluteUrl(getDomain("cdn")) - val chapters = apiCall( - """ + ).getJSONObject("media") + val title = details.getJSONObject("title") + val poster = details.getString("poster").removePrefix("/cdn") + .toAbsoluteUrl(getDomain("cdn")) + val chapters = apiCall( + """ chapters(mediaId: "${details.getString("mediaId")}") { id chapter released } """.trimIndent(), - ).getJSONArray("chapters") - return manga.copy( - title = title.getString("be"), - altTitle = title.getString("alt"), - coverUrl = "$poster?width=200&height=280", - largeCoverUrl = poster, - description = details.getJSONObject("description").getString("be"), - rating = details.getDouble("rating").toFloat() / 10f, - tags = details.getJSONArray("genres").mapToTags(), - state = when (details.getString("status")) { - "ongoing" -> MangaState.ONGOING - "finished" -> MangaState.FINISHED - else -> null - }, - chapters = chapters.mapJSON { jo -> - val number = jo.getInt("chapter") - MangaChapter( - id = generateUid(jo.getString("id")), - name = "Глава $number", - number = number, - url = "${manga.url}/read/$number", - scanlator = null, - uploadDate = jo.getLong("released"), - branch = null, - source = source, - ) - }, - ) - } + ).getJSONArray("chapters") + return manga.copy( + title = title.getString("be"), + altTitle = title.getString("alt"), + coverUrl = "$poster?width=200&height=280", + largeCoverUrl = poster, + description = details.getJSONObject("description").getString("be"), + rating = details.getDouble("rating").toFloat() / 10f, + tags = details.getJSONArray("genres").mapToTags(), + state = when (details.getString("status")) { + "ongoing" -> MangaState.ONGOING + "finished" -> MangaState.FINISHED + else -> null + }, + chapters = chapters.mapJSON { jo -> + val number = jo.getInt("chapter") + MangaChapter( + id = generateUid(jo.getString("id")), + name = "Глава $number", + number = number, + url = "${manga.url}/read/$number", + scanlator = null, + uploadDate = jo.getLong("released"), + branch = null, + source = source, + ) + }, + ) + } - override suspend fun getPages(chapter: MangaChapter): List { - val (_, slug, _, number) = chapter.url.split('/') - val chapterJson = apiCall( - """ + override suspend fun getPages(chapter: MangaChapter): List { + val (_, slug, _, number) = chapter.url.split('/') + val chapterJson = apiCall( + """ chapter(slug: "$slug", chapter: $number) { id images { @@ -162,35 +162,34 @@ internal class AnibelParser(context: MangaLoaderContext) : MangaParser(context, } } """.trimIndent(), - ).getJSONObject("chapter") - val pages = chapterJson.getJSONArray("images") - val chapterUrl = "https://${domain}/${chapter.url}" - return pages.mapJSONIndexed { i, jo -> - MangaPage( - id = generateUid("${chapter.url}/$i"), - url = jo.getString("large"), - referer = chapterUrl, - preview = jo.getString("thumbnail"), - source = source, - ) - } - } + ).getJSONObject("chapter") + val pages = chapterJson.getJSONArray("images") + val chapterUrl = "https://${domain}/${chapter.url}" + return pages.mapJSONIndexed { i, jo -> + MangaPage( + id = generateUid("${chapter.url}/$i"), + url = jo.getString("large"), + preview = jo.getString("thumbnail"), + source = source, + ) + } + } - override suspend fun getTags(): Set { - val json = apiCall( - """ + override suspend fun getTags(): Set { + val json = apiCall( + """ getFilters(mediaType: manga) { genres } """.trimIndent(), - ) - val array = json.getJSONObject("getFilters").getJSONArray("genres") - return array.mapToTags() - } + ) + val array = json.getJSONObject("getFilters").getJSONArray("genres") + return array.mapToTags() + } - private suspend fun search(query: String): List { - val json = apiCall( - """ + private suspend fun search(query: String): List { + val json = apiCall( + """ search(query: "$query", limit: 40) { id title { @@ -202,65 +201,65 @@ internal class AnibelParser(context: MangaLoaderContext) : MangaParser(context, type } """.trimIndent(), - ) - val array = json.getJSONArray("search") - return array.mapJSON { jo -> - val mediaId = jo.getString("id") - val title = jo.getJSONObject("title") - val href = "${jo.getString("type").lowercase()}/${jo.getString("url")}" - Manga( - id = generateUid(mediaId), - title = title.getString("be"), - coverUrl = jo.getString("poster").removePrefix("/cdn") - .toAbsoluteUrl(getDomain("cdn")) + "?width=200&height=280", - altTitle = title.getString("en").takeUnless(String::isEmpty), - author = null, - isNsfw = false, - rating = RATING_UNKNOWN, - url = href, - publicUrl = "https://${domain}/$href", - tags = emptySet(), - state = null, - source = source, - ) - } - } + ) + val array = json.getJSONArray("search") + return array.mapJSON { jo -> + val mediaId = jo.getString("id") + val title = jo.getJSONObject("title") + val href = "${jo.getString("type").lowercase()}/${jo.getString("url")}" + Manga( + id = generateUid(mediaId), + title = title.getString("be"), + coverUrl = jo.getString("poster").removePrefix("/cdn") + .toAbsoluteUrl(getDomain("cdn")) + "?width=200&height=280", + altTitle = title.getString("en").takeUnless(String::isEmpty), + author = null, + isNsfw = false, + rating = RATING_UNKNOWN, + url = href, + publicUrl = "https://${domain}/$href", + tags = emptySet(), + state = null, + source = source, + ) + } + } - private suspend fun apiCall(request: String): JSONObject { - return webClient.graphQLQuery("https://api.${domain}/graphql", request) - .getJSONObject("data") - } + private suspend fun apiCall(request: String): JSONObject { + return webClient.graphQLQuery("https://api.${domain}/graphql", request) + .getJSONObject("data") + } - private fun JSONArray.mapToTags(): Set { + private fun JSONArray.mapToTags(): Set { - fun toTitle(slug: String): String { - val builder = StringBuilder(slug) - var capitalize = true - for ((i, c) in builder.withIndex()) { - when { - c == '-' -> { - builder.setCharAt(i, ' ') - } + fun toTitle(slug: String): String { + val builder = StringBuilder(slug) + var capitalize = true + for ((i, c) in builder.withIndex()) { + when { + c == '-' -> { + builder.setCharAt(i, ' ') + } - capitalize -> { - builder.setCharAt(i, c.uppercaseChar()) - capitalize = false - } - } - } - return builder.toString() - } + capitalize -> { + builder.setCharAt(i, c.uppercaseChar()) + capitalize = false + } + } + } + return builder.toString() + } - val result = ArraySet(length()) - stringIterator().forEach { - result.add( - MangaTag( - title = toTitle(it), - key = it, - source = source, - ), - ) - } - return result - } + val result = ArraySet(length()) + stringIterator().forEach { + result.add( + MangaTag( + title = toTitle(it), + key = it, + source = source, + ), + ) + } + return result + } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BatoToParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BatoToParser.kt index 0f5e3603..127c455f 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BatoToParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BatoToParser.kt @@ -20,290 +20,298 @@ import javax.crypto.spec.SecretKeySpec @MangaSourceParser("BATOTO", "Bato.To") internal class BatoToParser(context: MangaLoaderContext) : PagedMangaParser( - context = context, - source = MangaSource.BATOTO, - pageSize = 60, - searchPageSize = 20, + context = context, + source = MangaSource.BATOTO, + pageSize = 60, + searchPageSize = 20, ) { - override val sortOrders: Set = EnumSet.of( - SortOrder.NEWEST, - SortOrder.UPDATED, - SortOrder.POPULARITY, - SortOrder.ALPHABETICAL, - ) + override val sortOrders: Set = EnumSet.of( + SortOrder.NEWEST, + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.ALPHABETICAL, + ) - override val configKeyDomain = ConfigKey.Domain( - "bato.to", - arrayOf("bato.to", "mto.to", "hto.to", "mangatoto.com", "battwo.com", "batotwo.com", "comiko.net", "batotoo.com"), - ) + override val configKeyDomain = ConfigKey.Domain( + "bato.to", + arrayOf( + "bato.to", + "mto.to", + "hto.to", + "mangatoto.com", + "battwo.com", + "batotwo.com", + "comiko.net", + "batotoo.com", + ), + ) - override suspend fun getListPage( - page: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - if (!query.isNullOrEmpty()) { - return search(page, query) - } - @Suppress("NON_EXHAUSTIVE_WHEN_STATEMENT") - val url = buildString { - append("https://") - append(domain) - append("/browse?sort=") - when (sortOrder) { - SortOrder.UPDATED, - -> append("update.za") + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + if (!query.isNullOrEmpty()) { + return search(page, query) + } + @Suppress("NON_EXHAUSTIVE_WHEN_STATEMENT") + val url = buildString { + append("https://") + append(domain) + append("/browse?sort=") + when (sortOrder) { + SortOrder.UPDATED, + -> append("update.za") - SortOrder.POPULARITY -> append("views_a.za") - SortOrder.NEWEST -> append("create.za") - SortOrder.ALPHABETICAL -> append("title.az") - } - if (!tags.isNullOrEmpty()) { - append("&genres=") - appendAll(tags, ",") { it.key } - } - append("&page=") - append(page) - } - return parseList(url, page) - } + SortOrder.POPULARITY -> append("views_a.za") + SortOrder.NEWEST -> append("create.za") + SortOrder.ALPHABETICAL -> append("title.az") + } + if (!tags.isNullOrEmpty()) { + append("&genres=") + appendAll(tags, ",") { it.key } + } + append("&page=") + append(page) + } + return parseList(url, page) + } - override suspend fun getDetails(manga: Manga): Manga { - val root = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - .requireElementById("mainer") - val details = root.selectFirstOrThrow(".detail-set") - val attrs = details.selectFirst(".attr-main")?.select(".attr-item")?.associate { - it.child(0).text().trim() to it.child(1) - }.orEmpty() - return manga.copy( - title = root.selectFirst("h3.item-title")?.text() ?: manga.title, - isNsfw = !root.selectFirst("alert")?.getElementsContainingOwnText("NSFW").isNullOrEmpty(), - largeCoverUrl = details.selectFirst("img[src]")?.absUrl("src"), - description = details.getElementById("limit-height-body-summary") - ?.selectFirst(".limit-html") - ?.html(), - tags = manga.tags + attrs["Genres:"]?.parseTags().orEmpty(), - state = when (attrs["Release status:"]?.text()) { - "Ongoing" -> MangaState.ONGOING - "Completed" -> MangaState.FINISHED - else -> manga.state - }, - author = attrs["Authors:"]?.text()?.trim() ?: manga.author, - chapters = root.selectFirst(".episode-list") - ?.selectFirst(".main") - ?.children() - ?.reversed() - ?.mapChapters { i, div -> - div.parseChapter(i) - }.orEmpty(), - ) - } + override suspend fun getDetails(manga: Manga): Manga { + val root = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + .requireElementById("mainer") + val details = root.selectFirstOrThrow(".detail-set") + val attrs = details.selectFirst(".attr-main")?.select(".attr-item")?.associate { + it.child(0).text().trim() to it.child(1) + }.orEmpty() + return manga.copy( + title = root.selectFirst("h3.item-title")?.text() ?: manga.title, + isNsfw = !root.selectFirst("alert")?.getElementsContainingOwnText("NSFW").isNullOrEmpty(), + largeCoverUrl = details.selectFirst("img[src]")?.absUrl("src"), + description = details.getElementById("limit-height-body-summary") + ?.selectFirst(".limit-html") + ?.html(), + tags = manga.tags + attrs["Genres:"]?.parseTags().orEmpty(), + state = when (attrs["Release status:"]?.text()) { + "Ongoing" -> MangaState.ONGOING + "Completed" -> MangaState.FINISHED + else -> manga.state + }, + author = attrs["Authors:"]?.text()?.trim() ?: manga.author, + chapters = root.selectFirst(".episode-list") + ?.selectFirst(".main") + ?.children() + ?.reversed() + ?.mapChapters { i, div -> + div.parseChapter(i) + }.orEmpty(), + ) + } - override suspend fun getPages(chapter: MangaChapter): List { - val fullUrl = chapter.url.toAbsoluteUrl(domain) - val scripts = webClient.httpGet(fullUrl).parseHtml().select("script") - for (script in scripts) { - val scriptSrc = script.html() - val p = scriptSrc.indexOf("const imgHttpLis =") - if (p == -1) continue - val start = scriptSrc.indexOf('[', p) - val end = scriptSrc.indexOf(';', start) - if (start == -1 || end == -1) { - continue - } - val images = JSONArray(scriptSrc.substring(start, end)) - val batoPass = scriptSrc.substringBetweenFirst("batoPass =", ";")?.trim(' ', '"', '\n') - ?: script.parseFailed("Cannot find batoPass") - val batoWord = scriptSrc.substringBetweenFirst("batoWord =", ";")?.trim(' ', '"', '\n') - ?: script.parseFailed("Cannot find batoWord") - val password = context.evaluateJs(batoPass)?.removeSurrounding('"') - ?: script.parseFailed("Cannot evaluate batoPass") - val args = JSONArray(decryptAES(batoWord, password)) - val result = ArrayList(images.length()) - repeat(images.length()) { i -> - val url = images.getString(i) - result += MangaPage( - id = generateUid(url), - url = url + "?" + args.getString(i), - referer = fullUrl, - preview = null, - source = source, - ) - } - return result - } - throw ParseException("Cannot find images list", fullUrl) - } + override suspend fun getPages(chapter: MangaChapter): List { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val scripts = webClient.httpGet(fullUrl).parseHtml().select("script") + for (script in scripts) { + val scriptSrc = script.html() + val p = scriptSrc.indexOf("const imgHttpLis =") + if (p == -1) continue + val start = scriptSrc.indexOf('[', p) + val end = scriptSrc.indexOf(';', start) + if (start == -1 || end == -1) { + continue + } + val images = JSONArray(scriptSrc.substring(start, end)) + val batoPass = scriptSrc.substringBetweenFirst("batoPass =", ";")?.trim(' ', '"', '\n') + ?: script.parseFailed("Cannot find batoPass") + val batoWord = scriptSrc.substringBetweenFirst("batoWord =", ";")?.trim(' ', '"', '\n') + ?: script.parseFailed("Cannot find batoWord") + val password = context.evaluateJs(batoPass)?.removeSurrounding('"') + ?: script.parseFailed("Cannot evaluate batoPass") + val args = JSONArray(decryptAES(batoWord, password)) + val result = ArrayList(images.length()) + repeat(images.length()) { i -> + val url = images.getString(i) + result += MangaPage( + id = generateUid(url), + url = url + "?" + args.getString(i), + preview = null, + source = source, + ) + } + return result + } + throw ParseException("Cannot find images list", fullUrl) + } - override suspend fun getTags(): Set { - val scripts = webClient.httpGet( - "https://${domain}/browse", - ).parseHtml().selectOrThrow("script") - for (script in scripts) { - val genres = script.html().substringBetweenFirst("const _genres =", ";") ?: continue - val jo = JSONObject(genres) - val result = ArraySet(jo.length()) - jo.keys().forEach { key -> - val item = jo.getJSONObject(key) - result += MangaTag( - title = item.getString("text").toTitleCase(), - key = item.getString("file"), - source = source, - ) - } - return result - } - throw ParseException("Cannot find gernes list", scripts[0].baseUri()) - } + override suspend fun getTags(): Set { + val scripts = webClient.httpGet( + "https://${domain}/browse", + ).parseHtml().selectOrThrow("script") + for (script in scripts) { + val genres = script.html().substringBetweenFirst("const _genres =", ";") ?: continue + val jo = JSONObject(genres) + val result = ArraySet(jo.length()) + jo.keys().forEach { key -> + val item = jo.getJSONObject(key) + result += MangaTag( + title = item.getString("text").toTitleCase(), + key = item.getString("file"), + source = source, + ) + } + return result + } + throw ParseException("Cannot find gernes list", scripts[0].baseUri()) + } - private suspend fun search(page: Int, query: String): List { - val url = buildString { - append("https://") - append(domain) - append("/search?word=") - append(query.replace(' ', '+')) - append("&page=") - append(page) - } - return parseList(url, page) - } + private suspend fun search(page: Int, query: String): List { + val url = buildString { + append("https://") + append(domain) + append("/search?word=") + append(query.replace(' ', '+')) + append("&page=") + append(page) + } + return parseList(url, page) + } - private fun getActivePage(body: Element): Int = body.select("nav ul.pagination > li.page-item.active") - .lastOrNull() - ?.text() - ?.toIntOrNull() ?: body.parseFailed("Cannot determine current page") + private fun getActivePage(body: Element): Int = body.select("nav ul.pagination > li.page-item.active") + .lastOrNull() + ?.text() + ?.toIntOrNull() ?: body.parseFailed("Cannot determine current page") - private suspend fun parseList(url: String, page: Int): List { - val body = webClient.httpGet(url).parseHtml().body() - if (body.selectFirst(".browse-no-matches") != null) { - return emptyList() - } - val activePage = getActivePage(body) - if (activePage != page) { - return emptyList() - } - val root = body.requireElementById("series-list") - return root.children().map { div -> - val a = div.selectFirstOrThrow("a") - val href = a.attrAsRelativeUrl("href") - val title = div.selectFirstOrThrow(".item-title").text() - Manga( - id = generateUid(href), - title = title, - altTitle = div.selectFirst(".item-alias")?.text()?.takeUnless { it == title }, - url = href, - publicUrl = a.absUrl("href"), - rating = RATING_UNKNOWN, - isNsfw = false, - coverUrl = div.selectFirst("img[src]")?.absUrl("src").orEmpty(), - largeCoverUrl = null, - description = null, - tags = div.selectFirst(".item-genre")?.parseTags().orEmpty(), - state = null, - author = null, - source = source, - ) - } - } + private suspend fun parseList(url: String, page: Int): List { + val body = webClient.httpGet(url).parseHtml().body() + if (body.selectFirst(".browse-no-matches") != null) { + return emptyList() + } + val activePage = getActivePage(body) + if (activePage != page) { + return emptyList() + } + val root = body.requireElementById("series-list") + return root.children().map { div -> + val a = div.selectFirstOrThrow("a") + val href = a.attrAsRelativeUrl("href") + val title = div.selectFirstOrThrow(".item-title").text() + Manga( + id = generateUid(href), + title = title, + altTitle = div.selectFirst(".item-alias")?.text()?.takeUnless { it == title }, + url = href, + publicUrl = a.absUrl("href"), + rating = RATING_UNKNOWN, + isNsfw = false, + coverUrl = div.selectFirst("img[src]")?.absUrl("src").orEmpty(), + largeCoverUrl = null, + description = null, + tags = div.selectFirst(".item-genre")?.parseTags().orEmpty(), + state = null, + author = null, + source = source, + ) + } + } - private fun Element.parseTags() = children().mapToSet { span -> - val text = span.ownText() - MangaTag( - title = text.toTitleCase(), - key = text.lowercase(Locale.ENGLISH).replace(' ', '_'), - source = source, - ) - } + private fun Element.parseTags() = children().mapToSet { span -> + val text = span.ownText() + MangaTag( + title = text.toTitleCase(), + key = text.lowercase(Locale.ENGLISH).replace(' ', '_'), + source = source, + ) + } - private fun Element.parseChapter(index: Int): MangaChapter? { - val a = selectFirst("a.chapt") ?: return null - val extra = selectFirst(".extra") - val href = a.attrAsRelativeUrl("href") - return MangaChapter( - id = generateUid(href), - name = a.text(), - number = index + 1, - url = href, - scanlator = extra?.getElementsByAttributeValueContaining("href", "/group/")?.text(), - uploadDate = runCatching { - parseChapterDate(extra?.select("i")?.lastOrNull()?.ownText()) - }.getOrDefault(0), - branch = null, - source = source, - ) - } + private fun Element.parseChapter(index: Int): MangaChapter? { + val a = selectFirst("a.chapt") ?: return null + val extra = selectFirst(".extra") + val href = a.attrAsRelativeUrl("href") + return MangaChapter( + id = generateUid(href), + name = a.text(), + number = index + 1, + url = href, + scanlator = extra?.getElementsByAttributeValueContaining("href", "/group/")?.text(), + uploadDate = runCatching { + parseChapterDate(extra?.select("i")?.lastOrNull()?.ownText()) + }.getOrDefault(0), + branch = null, + source = source, + ) + } - private fun parseChapterDate(date: String?): Long { - if (date.isNullOrEmpty()) { - return 0 - } - val value = date.substringBefore(' ').toInt() - val field = when { - "sec" in date -> Calendar.SECOND - "min" in date -> Calendar.MINUTE - "hour" in date -> Calendar.HOUR - "day" in date -> Calendar.DAY_OF_MONTH - "week" in date -> Calendar.WEEK_OF_YEAR - "month" in date -> Calendar.MONTH - "year" in date -> Calendar.YEAR - else -> return 0 - } - val calendar = Calendar.getInstance() - calendar.add(field, -value) - return calendar.timeInMillis - } + private fun parseChapterDate(date: String?): Long { + if (date.isNullOrEmpty()) { + return 0 + } + val value = date.substringBefore(' ').toInt() + val field = when { + "sec" in date -> Calendar.SECOND + "min" in date -> Calendar.MINUTE + "hour" in date -> Calendar.HOUR + "day" in date -> Calendar.DAY_OF_MONTH + "week" in date -> Calendar.WEEK_OF_YEAR + "month" in date -> Calendar.MONTH + "year" in date -> Calendar.YEAR + else -> return 0 + } + val calendar = Calendar.getInstance() + calendar.add(field, -value) + return calendar.timeInMillis + } - private fun decryptAES(encrypted: String, password: String): String { - val cipherData = context.decodeBase64(encrypted) - val saltData = cipherData.copyOfRange(8, 16) - val (key, iv) = generateKeyAndIV( - keyLength = 32, - ivLength = 16, - iterations = 1, - salt = saltData, - password = password.toByteArray(StandardCharsets.UTF_8), - md = MessageDigest.getInstance("MD5"), - ) - val encryptedData = cipherData.copyOfRange(16, cipherData.size) - val cipher = Cipher.getInstance("AES/CBC/PKCS5Padding") - cipher.init(Cipher.DECRYPT_MODE, key, iv) - return cipher.doFinal(encryptedData).toString(Charsets.UTF_8) - } + private fun decryptAES(encrypted: String, password: String): String { + val cipherData = context.decodeBase64(encrypted) + val saltData = cipherData.copyOfRange(8, 16) + val (key, iv) = generateKeyAndIV( + keyLength = 32, + ivLength = 16, + iterations = 1, + salt = saltData, + password = password.toByteArray(StandardCharsets.UTF_8), + md = MessageDigest.getInstance("MD5"), + ) + val encryptedData = cipherData.copyOfRange(16, cipherData.size) + val cipher = Cipher.getInstance("AES/CBC/PKCS5Padding") + cipher.init(Cipher.DECRYPT_MODE, key, iv) + return cipher.doFinal(encryptedData).toString(Charsets.UTF_8) + } - @Suppress("SameParameterValue") - private fun generateKeyAndIV( - keyLength: Int, - ivLength: Int, - iterations: Int, - salt: ByteArray, - password: ByteArray, - md: MessageDigest, - ): Pair { - val digestLength = md.digestLength - val requiredLength = (keyLength + ivLength + digestLength - 1) / digestLength * digestLength - val generatedData = ByteArray(requiredLength) - var generatedLength = 0 - md.reset() - while (generatedLength < keyLength + ivLength) { - if (generatedLength > 0) { - md.update(generatedData, generatedLength - digestLength, digestLength) - } - md.update(password) - md.update(salt, 0, 8) - md.digest(generatedData, generatedLength, digestLength) - repeat(iterations - 1) { - md.update(generatedData, generatedLength, digestLength) - md.digest(generatedData, generatedLength, digestLength) - } - generatedLength += digestLength - } + @Suppress("SameParameterValue") + private fun generateKeyAndIV( + keyLength: Int, + ivLength: Int, + iterations: Int, + salt: ByteArray, + password: ByteArray, + md: MessageDigest, + ): Pair { + val digestLength = md.digestLength + val requiredLength = (keyLength + ivLength + digestLength - 1) / digestLength * digestLength + val generatedData = ByteArray(requiredLength) + var generatedLength = 0 + md.reset() + while (generatedLength < keyLength + ivLength) { + if (generatedLength > 0) { + md.update(generatedData, generatedLength - digestLength, digestLength) + } + md.update(password) + md.update(salt, 0, 8) + md.digest(generatedData, generatedLength, digestLength) + repeat(iterations - 1) { + md.update(generatedData, generatedLength, digestLength) + md.digest(generatedData, generatedLength, digestLength) + } + generatedLength += digestLength + } - return SecretKeySpec(generatedData.copyOfRange(0, keyLength), "AES") to IvParameterSpec( - if (ivLength > 0) { - generatedData.copyOfRange(keyLength, keyLength + ivLength) - } else byteArrayOf(), - ) - } + return SecretKeySpec(generatedData.copyOfRange(0, keyLength), "AES") to IvParameterSpec( + if (ivLength > 0) { + generatedData.copyOfRange(keyLength, keyLength + ivLength) + } else byteArrayOf(), + ) + } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BlogTruyenParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BlogTruyenParser.kt index 05b7b22d..27ce2241 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BlogTruyenParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BlogTruyenParser.kt @@ -17,238 +17,236 @@ import java.util.* @MangaSourceParser("BLOGTRUYEN", "BlogTruyen", "vi") class BlogTruyenParser(context: MangaLoaderContext) : - PagedMangaParser(context, MangaSource.BLOGTRUYEN, pageSize = 20) { - - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("blogtruyen.vn", null) - - override val sortOrders: Set - get() = EnumSet.of(SortOrder.UPDATED) - - private val mutex = Mutex() - private val dateFormat = SimpleDateFormat("dd/MM/yyyy HH:mm", Locale.US) - private var cacheTags: ArrayMap? = null - - override suspend fun getDetails(manga: Manga): Manga { - val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - val descriptionElement = doc.selectFirstOrThrow("div.description") - val statusText = descriptionElement - .selectFirst("p:contains(Trạng thái) > span.color-red") - ?.text() - - val state = when (statusText) { - "Đang tiến hành" -> MangaState.ONGOING - "Đã hoàn thành" -> MangaState.FINISHED - else -> null - } - - val rating = doc.selectFirst("span.total-vote")?.attr("ng-init")?.let { text -> - val like = text.substringAfter("TotalLike=") - .substringBefore(';') - .toIntOrNull() ?: return@let RATING_UNKNOWN - val dislike = text.substringAfter("TotalDisLike=") - .toIntOrNull() ?: return@let RATING_UNKNOWN - - when { - like == 0 && dislike == 0 -> RATING_UNKNOWN - else -> like.toFloat() / (like + dislike) - } - } - - val tagMap = getOrCreateTagMap() - val tags = descriptionElement.select("p > span.category").mapNotNullToSet { - val tagName = it.selectFirst("a")?.text()?.trim() ?: return@mapNotNullToSet null - tagMap[tagName] - } - - return manga.copy( - tags = tags, - author = descriptionElement.selectFirst("p:contains(Tác giả) > a")?.text(), - description = doc.selectFirst(".detail .content")?.html(), - chapters = parseChapterList(doc), - largeCoverUrl = doc.selectLast("div.thumbnail > img")?.imageUrl(), - state = state, - rating = rating ?: RATING_UNKNOWN, - isNsfw = doc.getElementById("warningCategory") != null, - ) - } - - private fun parseChapterList(doc: Document): List { - val chapterList = doc.select("#list-chapters > p") - return chapterList.asReversed().mapChapters { index, element -> - val titleElement = element.selectFirst("span.title > a") ?: return@mapChapters null - val name = titleElement.text() - val relativeUrl = titleElement.attrAsRelativeUrl("href") - val id = relativeUrl.substringAfter('/').substringBefore('/') - val uploadDate = dateFormat.tryParse(element.select("span.publishedDate").text()) - MangaChapter( - id = generateUid(id), - name = name, - number = index + 1, - url = relativeUrl, - scanlator = null, - uploadDate = uploadDate, - branch = null, - source = source, - ) - } - } - - override suspend fun getListPage( - page: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - return when { - !query.isNullOrEmpty() -> { - val searchUrl = "https://${domain}/timkiem/nangcao/1/0/-1/-1?txt=${query.urlEncoded()}&p=$page" - val searchContent = webClient.httpGet(searchUrl).parseHtml() - .selectFirst("section.list-manga-bycate > div.list") - parseMangaList(searchContent) - } - - !tags.isNullOrEmpty() -> { - val tag = tags.oneOrThrowIfMany()!! - val categoryAjax = - "https://${domain}/ajax/Category/AjaxLoadMangaByCategory?id=${tag.key}&orderBy=5&p=$page" - val listContent = webClient.httpGet(categoryAjax).parseHtml().selectFirst("div.list") - parseMangaList(listContent) - } - - else -> getNormalList(page) - } - } - - private suspend fun getNormalList(page: Int): List { - val pageLink = "https://${domain}/page-$page" - val doc = webClient.httpGet(pageLink).parseHtml() - val listElements = doc.selectFirstOrThrow("section.list-mainpage.listview") - .select("div.bg-white.storyitem") - - return listElements.mapNotNull { - val linkTag = it.selectFirst("div.fl-l > a") ?: return@mapNotNull null - val relativeUrl = linkTag.attrAsRelativeUrl("href") - val tagMap = getOrCreateTagMap() - val tags = it.select("footer > div.category > a").mapNotNullToSet { a -> - tagMap[a.text()] - } - - Manga( - id = generateUid(relativeUrl), - title = linkTag.attr("title"), - altTitle = null, - description = it.selectFirst("p.al-j.break.line-height-15")?.text(), - url = relativeUrl, - publicUrl = relativeUrl.toAbsoluteUrl(domain), - coverUrl = linkTag.selectLast("img")?.imageUrl().orEmpty(), - source = source, - tags = tags, - isNsfw = false, - rating = RATING_UNKNOWN, - author = null, - state = null, - ) - } - } - - private fun parseMangaList(listElement: Element?): List { - listElement ?: return emptyList() - - return listElement.select("span.tiptip[data-tiptip]").mapNotNull { - val mangaInfo = listElement.getElementById(it.attr("data-tiptip")) ?: return@mapNotNull null - val a = it.selectFirst("a") ?: return@mapNotNull null - val relativeUrl = a.attrAsRelativeUrl("href") - Manga( - id = generateUid(relativeUrl), - title = a.text(), - altTitle = null, - description = mangaInfo.select("div.al-j.fs-12").text(), - url = relativeUrl, - publicUrl = relativeUrl.toAbsoluteUrl(domain), - coverUrl = mangaInfo.selectFirst("div > img.img")?.imageUrl().orEmpty(), - isNsfw = false, - rating = RATING_UNKNOWN, - tags = emptySet(), - author = null, - state = null, - source = source, - ) - } - } - - override suspend fun getPages(chapter: MangaChapter): List { - fun generateImageId(index: Int) = generateUid("${chapter.url}/$index") - - val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() - val pages = ArrayList() - val referer = chapter.url.toAbsoluteUrl(domain) - doc.select("#content > img").forEach { img -> - pages.add( - MangaPage( - id = generateImageId(pages.size), - url = img.imageUrl(), - referer = referer, - preview = null, - source = source, - ), - ) - } - - // Some chapters use js script to render images - val script = doc.selectLast("#content > script") - if (script != null && script.data().contains("listImageCaption")) { - val imagesStr = script.data().substringBefore(';').substringAfterLast('=').trim() - val imageArr = JSONArray(imagesStr) - for (i in 0 until imageArr.length()) { - val imageUrl = imageArr.getJSONObject(i).getString("url") - pages.add( - MangaPage( - id = generateImageId(pages.size), - url = imageUrl, - referer = referer, - preview = null, - source = source, - ), - ) - } - } - - return pages - } - - override suspend fun getTags(): Set { - val map = getOrCreateTagMap() - val tags = HashSet(map.size) - for (entry in map) { - tags.add(entry.value) - } - - return tags - } - - - private suspend fun getOrCreateTagMap(): ArrayMap = mutex.withLock { - cacheTags?.let { return@withLock it } - val doc = webClient.httpGet("/timkiem/nangcao".toAbsoluteUrl(domain)).parseHtml() - val tagItems = doc.select("li[data-id]") - val tagMap = ArrayMap(tagItems.size) - for (tag in tagItems) { - val title = tag.text().trim() - tagMap[tag.text().trim()] = MangaTag( - title = title, - key = tag.attr("data-id"), - source = source, - ) - } - - cacheTags = tagMap - tagMap - } - - private fun Element.imageUrl(): String { - return attrAsAbsoluteUrlOrNull("src") - ?: attrAsAbsoluteUrlOrNull("data-cfsrc") - ?: "" - } + PagedMangaParser(context, MangaSource.BLOGTRUYEN, pageSize = 20) { + + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("blogtruyen.vn", null) + + override val sortOrders: Set + get() = EnumSet.of(SortOrder.UPDATED) + + private val mutex = Mutex() + private val dateFormat = SimpleDateFormat("dd/MM/yyyy HH:mm", Locale.US) + private var cacheTags: ArrayMap? = null + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val descriptionElement = doc.selectFirstOrThrow("div.description") + val statusText = descriptionElement + .selectFirst("p:contains(Trạng thái) > span.color-red") + ?.text() + + val state = when (statusText) { + "Đang tiến hành" -> MangaState.ONGOING + "Đã hoàn thành" -> MangaState.FINISHED + else -> null + } + + val rating = doc.selectFirst("span.total-vote")?.attr("ng-init")?.let { text -> + val like = text.substringAfter("TotalLike=") + .substringBefore(';') + .toIntOrNull() ?: return@let RATING_UNKNOWN + val dislike = text.substringAfter("TotalDisLike=") + .toIntOrNull() ?: return@let RATING_UNKNOWN + + when { + like == 0 && dislike == 0 -> RATING_UNKNOWN + else -> like.toFloat() / (like + dislike) + } + } + + val tagMap = getOrCreateTagMap() + val tags = descriptionElement.select("p > span.category").mapNotNullToSet { + val tagName = it.selectFirst("a")?.text()?.trim() ?: return@mapNotNullToSet null + tagMap[tagName] + } + + return manga.copy( + tags = tags, + author = descriptionElement.selectFirst("p:contains(Tác giả) > a")?.text(), + description = doc.selectFirst(".detail .content")?.html(), + chapters = parseChapterList(doc), + largeCoverUrl = doc.selectLast("div.thumbnail > img")?.imageUrl(), + state = state, + rating = rating ?: RATING_UNKNOWN, + isNsfw = doc.getElementById("warningCategory") != null, + ) + } + + private fun parseChapterList(doc: Document): List { + val chapterList = doc.select("#list-chapters > p") + return chapterList.asReversed().mapChapters { index, element -> + val titleElement = element.selectFirst("span.title > a") ?: return@mapChapters null + val name = titleElement.text() + val relativeUrl = titleElement.attrAsRelativeUrl("href") + val id = relativeUrl.substringAfter('/').substringBefore('/') + val uploadDate = dateFormat.tryParse(element.select("span.publishedDate").text()) + MangaChapter( + id = generateUid(id), + name = name, + number = index + 1, + url = relativeUrl, + scanlator = null, + uploadDate = uploadDate, + branch = null, + source = source, + ) + } + } + + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + return when { + !query.isNullOrEmpty() -> { + val searchUrl = "https://${domain}/timkiem/nangcao/1/0/-1/-1?txt=${query.urlEncoded()}&p=$page" + val searchContent = webClient.httpGet(searchUrl).parseHtml() + .selectFirst("section.list-manga-bycate > div.list") + parseMangaList(searchContent) + } + + !tags.isNullOrEmpty() -> { + val tag = tags.oneOrThrowIfMany()!! + val categoryAjax = + "https://${domain}/ajax/Category/AjaxLoadMangaByCategory?id=${tag.key}&orderBy=5&p=$page" + val listContent = webClient.httpGet(categoryAjax).parseHtml().selectFirst("div.list") + parseMangaList(listContent) + } + + else -> getNormalList(page) + } + } + + private suspend fun getNormalList(page: Int): List { + val pageLink = "https://${domain}/page-$page" + val doc = webClient.httpGet(pageLink).parseHtml() + val listElements = doc.selectFirstOrThrow("section.list-mainpage.listview") + .select("div.bg-white.storyitem") + + return listElements.mapNotNull { + val linkTag = it.selectFirst("div.fl-l > a") ?: return@mapNotNull null + val relativeUrl = linkTag.attrAsRelativeUrl("href") + val tagMap = getOrCreateTagMap() + val tags = it.select("footer > div.category > a").mapNotNullToSet { a -> + tagMap[a.text()] + } + + Manga( + id = generateUid(relativeUrl), + title = linkTag.attr("title"), + altTitle = null, + description = it.selectFirst("p.al-j.break.line-height-15")?.text(), + url = relativeUrl, + publicUrl = relativeUrl.toAbsoluteUrl(domain), + coverUrl = linkTag.selectLast("img")?.imageUrl().orEmpty(), + source = source, + tags = tags, + isNsfw = false, + rating = RATING_UNKNOWN, + author = null, + state = null, + ) + } + } + + private fun parseMangaList(listElement: Element?): List { + listElement ?: return emptyList() + + return listElement.select("span.tiptip[data-tiptip]").mapNotNull { + val mangaInfo = listElement.getElementById(it.attr("data-tiptip")) ?: return@mapNotNull null + val a = it.selectFirst("a") ?: return@mapNotNull null + val relativeUrl = a.attrAsRelativeUrl("href") + Manga( + id = generateUid(relativeUrl), + title = a.text(), + altTitle = null, + description = mangaInfo.select("div.al-j.fs-12").text(), + url = relativeUrl, + publicUrl = relativeUrl.toAbsoluteUrl(domain), + coverUrl = mangaInfo.selectFirst("div > img.img")?.imageUrl().orEmpty(), + isNsfw = false, + rating = RATING_UNKNOWN, + tags = emptySet(), + author = null, + state = null, + source = source, + ) + } + } + + override suspend fun getPages(chapter: MangaChapter): List { + fun generateImageId(index: Int) = generateUid("${chapter.url}/$index") + + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + val pages = ArrayList() + val referer = chapter.url.toAbsoluteUrl(domain) + doc.select("#content > img").forEach { img -> + pages.add( + MangaPage( + id = generateImageId(pages.size), + url = img.imageUrl(), + preview = null, + source = source, + ), + ) + } + + // Some chapters use js script to render images + val script = doc.selectLast("#content > script") + if (script != null && script.data().contains("listImageCaption")) { + val imagesStr = script.data().substringBefore(';').substringAfterLast('=').trim() + val imageArr = JSONArray(imagesStr) + for (i in 0 until imageArr.length()) { + val imageUrl = imageArr.getJSONObject(i).getString("url") + pages.add( + MangaPage( + id = generateImageId(pages.size), + url = imageUrl, + preview = null, + source = source, + ), + ) + } + } + + return pages + } + + override suspend fun getTags(): Set { + val map = getOrCreateTagMap() + val tags = HashSet(map.size) + for (entry in map) { + tags.add(entry.value) + } + + return tags + } + + + private suspend fun getOrCreateTagMap(): ArrayMap = mutex.withLock { + cacheTags?.let { return@withLock it } + val doc = webClient.httpGet("/timkiem/nangcao".toAbsoluteUrl(domain)).parseHtml() + val tagItems = doc.select("li[data-id]") + val tagMap = ArrayMap(tagItems.size) + for (tag in tagItems) { + val title = tag.text().trim() + tagMap[tag.text().trim()] = MangaTag( + title = title, + key = tag.attr("data-id"), + source = source, + ) + } + + cacheTags = tagMap + tagMap + } + + private fun Element.imageUrl(): String { + return attrAsAbsoluteUrlOrNull("src") + ?: attrAsAbsoluteUrlOrNull("data-cfsrc") + ?: "" + } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/CloneMangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/CloneMangaParser.kt index 56fd66e0..aa42dfd0 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/CloneMangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/CloneMangaParser.kt @@ -90,7 +90,6 @@ internal class CloneMangaParser(context: MangaLoaderContext) : PagedMangaParser( MangaPage( id = generateUid(imgUrl), url = imgUrl, - referer = imgUrl, preview = null, source = source, ), diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ComickFunParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ComickFunParser.kt index fdb94512..ad3b87b2 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ComickFunParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ComickFunParser.kt @@ -24,193 +24,192 @@ private const val CHAPTERS_LIMIT = 99999 @MangaSourceParser("COMICK_FUN", "ComicK") internal class ComickFunParser(context: MangaLoaderContext) : MangaParser(context, MangaSource.COMICK_FUN) { - override val configKeyDomain = ConfigKey.Domain("comick.app", null) + override val configKeyDomain = ConfigKey.Domain("comick.app", null) - override val sortOrders: Set = EnumSet.of( - SortOrder.POPULARITY, - SortOrder.UPDATED, - SortOrder.RATING, - ) + override val sortOrders: Set = EnumSet.of( + SortOrder.POPULARITY, + SortOrder.UPDATED, + SortOrder.RATING, + ) - @Volatile - private var cachedTags: SparseArrayCompat? = null + @Volatile + private var cachedTags: SparseArrayCompat? = null - override suspend fun getList( - offset: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - val domain = domain - val url = buildString { - append("https://api.") - append(domain) - append("/search?tachiyomi=true") - if (!query.isNullOrEmpty()) { - if (offset > 0) { - return emptyList() - } - append("&q=") - append(query.urlEncoded()) - } else { - append("&limit=") - append(PAGE_SIZE) - append("&page=") - append((offset / PAGE_SIZE) + 1) - if (!tags.isNullOrEmpty()) { - append("&genres=") - appendAll(tags, "&genres=", MangaTag::key) - } - append("&sort=") // view, uploaded, rating, follow, user_follow_count - append( - when (sortOrder) { - SortOrder.POPULARITY -> "view" - SortOrder.RATING -> "rating" - else -> "uploaded" - }, - ) - } - } - val ja = webClient.httpGet(url).parseJsonArray() - val tagsMap = cachedTags ?: loadTags() - return ja.mapJSON { jo -> - val slug = jo.getString("slug") - Manga( - id = generateUid(slug), - title = jo.getString("title"), - altTitle = null, - url = slug, - publicUrl = "https://$domain/comic/$slug", - rating = jo.getDoubleOrDefault("rating", -10.0).toFloat() / 10f, - isNsfw = false, - coverUrl = jo.getString("cover_url"), - largeCoverUrl = null, - description = jo.getStringOrNull("desc"), - tags = jo.selectGenres("genres", tagsMap), - state = runCatching { - if (jo.getBoolean("translation_completed")) { - MangaState.FINISHED - } else { - MangaState.ONGOING - } - }.getOrNull(), - author = null, - source = source, - ) - } - } + override suspend fun getList( + offset: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + val domain = domain + val url = buildString { + append("https://api.") + append(domain) + append("/search?tachiyomi=true") + if (!query.isNullOrEmpty()) { + if (offset > 0) { + return emptyList() + } + append("&q=") + append(query.urlEncoded()) + } else { + append("&limit=") + append(PAGE_SIZE) + append("&page=") + append((offset / PAGE_SIZE) + 1) + if (!tags.isNullOrEmpty()) { + append("&genres=") + appendAll(tags, "&genres=", MangaTag::key) + } + append("&sort=") // view, uploaded, rating, follow, user_follow_count + append( + when (sortOrder) { + SortOrder.POPULARITY -> "view" + SortOrder.RATING -> "rating" + else -> "uploaded" + }, + ) + } + } + val ja = webClient.httpGet(url).parseJsonArray() + val tagsMap = cachedTags ?: loadTags() + return ja.mapJSON { jo -> + val slug = jo.getString("slug") + Manga( + id = generateUid(slug), + title = jo.getString("title"), + altTitle = null, + url = slug, + publicUrl = "https://$domain/comic/$slug", + rating = jo.getDoubleOrDefault("rating", -10.0).toFloat() / 10f, + isNsfw = false, + coverUrl = jo.getString("cover_url"), + largeCoverUrl = null, + description = jo.getStringOrNull("desc"), + tags = jo.selectGenres("genres", tagsMap), + state = runCatching { + if (jo.getBoolean("translation_completed")) { + MangaState.FINISHED + } else { + MangaState.ONGOING + } + }.getOrNull(), + author = null, + source = source, + ) + } + } - override suspend fun getDetails(manga: Manga): Manga { - val domain = domain - val url = "https://api.$domain/comic/${manga.url}?tachiyomi=true" - val jo = webClient.httpGet(url).parseJson() - val comic = jo.getJSONObject("comic") - return manga.copy( - title = comic.getString("title"), - altTitle = null, // TODO - isNsfw = jo.getBoolean("matureContent") || comic.getBoolean("hentai"), - description = comic.getStringOrNull("parsed") ?: comic.getString("desc"), - tags = manga.tags + jo.getJSONArray("genres").mapJSONToSet { - MangaTag( - title = it.getString("name"), - key = it.getString("slug"), - source = source, - ) - }, - author = jo.getJSONArray("artists").optJSONObject(0)?.getString("name"), - chapters = getChapters(comic.getLong("id")), - ) - } + override suspend fun getDetails(manga: Manga): Manga { + val domain = domain + val url = "https://api.$domain/comic/${manga.url}?tachiyomi=true" + val jo = webClient.httpGet(url).parseJson() + val comic = jo.getJSONObject("comic") + return manga.copy( + title = comic.getString("title"), + altTitle = null, // TODO + isNsfw = jo.getBoolean("matureContent") || comic.getBoolean("hentai"), + description = comic.getStringOrNull("parsed") ?: comic.getString("desc"), + tags = manga.tags + jo.getJSONArray("genres").mapJSONToSet { + MangaTag( + title = it.getString("name"), + key = it.getString("slug"), + source = source, + ) + }, + author = jo.getJSONArray("artists").optJSONObject(0)?.getString("name"), + chapters = getChapters(comic.getLong("id")), + ) + } - override suspend fun getPages(chapter: MangaChapter): List { - val jo = webClient.httpGet( - "https://api.${domain}/chapter/${chapter.url}?tachiyomi=true", - ).parseJson().getJSONObject("chapter") - val referer = "https://${domain}/" - return jo.getJSONArray("images").mapJSON { - val url = it.getString("url") - MangaPage( - id = generateUid(url), - url = url, - referer = referer, - preview = null, - source = source, - ) - } - } + override suspend fun getPages(chapter: MangaChapter): List { + val jo = webClient.httpGet( + "https://api.${domain}/chapter/${chapter.url}?tachiyomi=true", + ).parseJson().getJSONObject("chapter") + val referer = "https://${domain}/" + return jo.getJSONArray("images").mapJSON { + val url = it.getString("url") + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } - override suspend fun getTags(): Set { - val sparseArray = cachedTags ?: loadTags() - val set = ArraySet(sparseArray.size()) - for (i in 0 until sparseArray.size()) { - set.add(sparseArray.valueAt(i)) - } - return set - } + override suspend fun getTags(): Set { + val sparseArray = cachedTags ?: loadTags() + val set = ArraySet(sparseArray.size()) + for (i in 0 until sparseArray.size()) { + set.add(sparseArray.valueAt(i)) + } + return set + } - private suspend fun loadTags(): SparseArrayCompat { - val ja = webClient.httpGet("https://api.${domain}/genre").parseJsonArray() - val tags = SparseArrayCompat(ja.length()) - for (jo in ja.JSONIterator()) { - tags.append( - jo.getInt("id"), - MangaTag( - title = jo.getString("name"), - key = jo.getString("slug"), - source = source, - ), - ) - } - cachedTags = tags - return tags - } + private suspend fun loadTags(): SparseArrayCompat { + val ja = webClient.httpGet("https://api.${domain}/genre").parseJsonArray() + val tags = SparseArrayCompat(ja.length()) + for (jo in ja.JSONIterator()) { + tags.append( + jo.getInt("id"), + MangaTag( + title = jo.getString("name"), + key = jo.getString("slug"), + source = source, + ), + ) + } + cachedTags = tags + return tags + } - private suspend fun getChapters(id: Long): List { - val ja = webClient.httpGet( - url = "https://api.${domain}/comic/$id/chapter?tachiyomi=true&limit=$CHAPTERS_LIMIT", - ).parseJson().getJSONArray("chapters") - val dateFormat = SimpleDateFormat("yyyy-MM-dd") - val counters = HashMap() - return ja.mapReversed { jo -> - val locale = Locale.forLanguageTag(jo.getString("lang")) - var number = counters[locale] ?: 0 - number++ - counters[locale] = number - MangaChapter( - id = generateUid(jo.getLong("id")), - name = buildString { - jo.getStringOrNull("vol")?.let { append("Vol ").append(it).append(' ') } - jo.getStringOrNull("chap")?.let { append("Chap ").append(it) } - jo.getStringOrNull("title")?.let { append(": ").append(it) } - }, - number = number, - url = jo.getString("hid"), - scanlator = jo.optJSONArray("group_name")?.optString(0), - uploadDate = dateFormat.tryParse(jo.getString("created_at").substringBefore('T')), - branch = locale.getDisplayName(locale).toTitleCase(locale), - source = source, - ) - } - } + private suspend fun getChapters(id: Long): List { + val ja = webClient.httpGet( + url = "https://api.${domain}/comic/$id/chapter?tachiyomi=true&limit=$CHAPTERS_LIMIT", + ).parseJson().getJSONArray("chapters") + val dateFormat = SimpleDateFormat("yyyy-MM-dd") + val counters = HashMap() + return ja.mapReversed { jo -> + val locale = Locale.forLanguageTag(jo.getString("lang")) + var number = counters[locale] ?: 0 + number++ + counters[locale] = number + MangaChapter( + id = generateUid(jo.getLong("id")), + name = buildString { + jo.getStringOrNull("vol")?.let { append("Vol ").append(it).append(' ') } + jo.getStringOrNull("chap")?.let { append("Chap ").append(it) } + jo.getStringOrNull("title")?.let { append(": ").append(it) } + }, + number = number, + url = jo.getString("hid"), + scanlator = jo.optJSONArray("group_name")?.optString(0), + uploadDate = dateFormat.tryParse(jo.getString("created_at").substringBefore('T')), + branch = locale.getDisplayName(locale).toTitleCase(locale), + source = source, + ) + } + } - private inline fun JSONArray.mapReversed(block: (JSONObject) -> R): List { - val len = length() - val destination = ArrayList(len) - for (i in (0 until len).reversed()) { - val jo = getJSONObject(i) - destination.add(block(jo)) - } - return destination - } + private inline fun JSONArray.mapReversed(block: (JSONObject) -> R): List { + val len = length() + val destination = ArrayList(len) + for (i in (0 until len).reversed()) { + val jo = getJSONObject(i) + destination.add(block(jo)) + } + return destination + } - private fun JSONObject.selectGenres(name: String, tags: SparseArrayCompat): Set { - val array = optJSONArray(name) ?: return emptySet() - val res = ArraySet(array.length()) - for (i in 0 until array.length()) { - val id = array.getInt(i) - val tag = tags.get(id) ?: continue - res.add(tag) - } - return res - } + private fun JSONObject.selectGenres(name: String, tags: SparseArrayCompat): Set { + val array = optJSONArray(name) ?: return emptySet() + val res = ArraySet(array.length()) + for (i in 0 until array.length()) { + val id = array.getInt(i) + val tag = tags.get(id) ?: continue + res.add(tag) + } + return res + } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/DesuMeParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/DesuMeParser.kt index 3004dcc4..d974ffed 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/DesuMeParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/DesuMeParser.kt @@ -135,7 +135,6 @@ internal class DesuMeParser(context: MangaLoaderContext) : PagedMangaParser(cont return json.getJSONObject("pages").getJSONArray("list").mapJSON { jo -> MangaPage( id = generateUid(jo.getLong("id")), - referer = fullUrl, preview = null, source = chapter.source, url = jo.getString("img"), diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ExHentaiParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ExHentaiParser.kt index 86be7938..0f0664bd 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ExHentaiParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ExHentaiParser.kt @@ -20,289 +20,288 @@ private const val DOMAIN_AUTHORIZED = "exhentai.org" @MangaSourceParser("EXHENTAI", "ExHentai") internal class ExHentaiParser( - context: MangaLoaderContext, + context: MangaLoaderContext, ) : PagedMangaParser(context, MangaSource.EXHENTAI, pageSize = 25), MangaParserAuthProvider { - override val sortOrders: Set = Collections.singleton( - SortOrder.NEWEST, - ) + override val sortOrders: Set = Collections.singleton( + SortOrder.NEWEST, + ) - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain(if (isAuthorized) DOMAIN_AUTHORIZED else DOMAIN_UNAUTHORIZED, null) + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain(if (isAuthorized) DOMAIN_AUTHORIZED else DOMAIN_UNAUTHORIZED, null) - override val authUrl: String - get() = "https://${domain}/bounce_login.php" + override val authUrl: String + get() = "https://${domain}/bounce_login.php" - private val ratingPattern = Regex("-?[0-9]+px") - private val authCookies = arrayOf("ipb_member_id", "ipb_pass_hash") - private var updateDm = false - private val nextPages = SparseArrayCompat() - private val suspiciousContentKey = ConfigKey.ShowSuspiciousContent(true) + private val ratingPattern = Regex("-?[0-9]+px") + private val authCookies = arrayOf("ipb_member_id", "ipb_pass_hash") + private var updateDm = false + private val nextPages = SparseArrayCompat() + private val suspiciousContentKey = ConfigKey.ShowSuspiciousContent(true) - override val isAuthorized: Boolean - get() { - val authorized = isAuthorized(DOMAIN_UNAUTHORIZED) - if (authorized) { - if (!isAuthorized(DOMAIN_AUTHORIZED)) { - context.cookieJar.copyCookies( - DOMAIN_UNAUTHORIZED, - DOMAIN_AUTHORIZED, - authCookies, - ) - context.cookieJar.insertCookies(DOMAIN_AUTHORIZED, "yay=louder") - } - return true - } - return false - } + override val isAuthorized: Boolean + get() { + val authorized = isAuthorized(DOMAIN_UNAUTHORIZED) + if (authorized) { + if (!isAuthorized(DOMAIN_AUTHORIZED)) { + context.cookieJar.copyCookies( + DOMAIN_UNAUTHORIZED, + DOMAIN_AUTHORIZED, + authCookies, + ) + context.cookieJar.insertCookies(DOMAIN_AUTHORIZED, "yay=louder") + } + return true + } + return false + } - init { - context.cookieJar.insertCookies(DOMAIN_AUTHORIZED, "nw=1", "sl=dm_2") - context.cookieJar.insertCookies(DOMAIN_UNAUTHORIZED, "nw=1", "sl=dm_2") - paginator.firstPage = 0 - } + init { + context.cookieJar.insertCookies(DOMAIN_AUTHORIZED, "nw=1", "sl=dm_2") + context.cookieJar.insertCookies(DOMAIN_UNAUTHORIZED, "nw=1", "sl=dm_2") + paginator.firstPage = 0 + } - override suspend fun getListPage( - page: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - var search = query?.urlEncoded().orEmpty() - val next = nextPages.get(page, 0L) - if (page > 0 && next == 0L) { - assert(false) { "Page timestamp not found" } - return emptyList() - } - val url = buildString { - append("https://") - append(domain) - append("/?next=") - append(next) - if (!tags.isNullOrEmpty()) { - var fCats = 0 - for (tag in tags) { - tag.key.toIntOrNull()?.let { fCats = fCats or it } ?: run { - search += tag.key + " " - } - } - if (fCats != 0) { - append("&f_cats=") - append(1023 - fCats) - } - } - if (search.isNotEmpty()) { - append("&f_search=") - append(search.trim().replace(' ', '+')) - } - // by unknown reason cookie "sl=dm_2" is ignored, so, we should request it again - if (updateDm) { - append("&inline_set=dm_e") - } - append("&advsearch=1") - if (config[suspiciousContentKey]) { - append("&f_sh=on") - } - } - val body = webClient.httpGet(url).parseHtml().body() - val root = body.selectFirst("table.itg") - ?.selectFirst("tbody") - ?: if (updateDm) { - body.parseFailed("Cannot find root") - } else { - updateDm = true - return getListPage(page, query, tags, sortOrder) - } - updateDm = false - nextPages[page + 1] = getNextTimestamp(body) - return root.children().mapNotNull { tr -> - if (tr.childrenSize() != 2) return@mapNotNull null - val (td1, td2) = tr.children() - val glink = td2.selectFirstOrThrow("div.glink") - val a = glink.parents().select("a").first() ?: glink.parseFailed("link not found") - val href = a.attrAsRelativeUrl("href") - val tagsDiv = glink.nextElementSibling() ?: glink.parseFailed("tags div not found") - val mainTag = td2.selectFirst("div.cn")?.let { div -> - MangaTag( - title = div.text().toTitleCase(), - key = tagIdByClass(div.classNames()) ?: return@let null, - source = source, - ) - } - Manga( - id = generateUid(href), - title = glink.text().cleanupTitle(), - altTitle = null, - url = href, - publicUrl = a.absUrl("href"), - rating = td2.selectFirst("div.ir")?.parseRating() ?: RATING_UNKNOWN, - isNsfw = true, - coverUrl = td1.selectFirst("img")?.absUrl("src").orEmpty(), - tags = setOfNotNull(mainTag), - state = null, - author = tagsDiv.getElementsContainingOwnText("artist:").first() - ?.nextElementSibling()?.text(), - source = source, - ) - } - } + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + var search = query?.urlEncoded().orEmpty() + val next = nextPages.get(page, 0L) + if (page > 0 && next == 0L) { + assert(false) { "Page timestamp not found" } + return emptyList() + } + val url = buildString { + append("https://") + append(domain) + append("/?next=") + append(next) + if (!tags.isNullOrEmpty()) { + var fCats = 0 + for (tag in tags) { + tag.key.toIntOrNull()?.let { fCats = fCats or it } ?: run { + search += tag.key + " " + } + } + if (fCats != 0) { + append("&f_cats=") + append(1023 - fCats) + } + } + if (search.isNotEmpty()) { + append("&f_search=") + append(search.trim().replace(' ', '+')) + } + // by unknown reason cookie "sl=dm_2" is ignored, so, we should request it again + if (updateDm) { + append("&inline_set=dm_e") + } + append("&advsearch=1") + if (config[suspiciousContentKey]) { + append("&f_sh=on") + } + } + val body = webClient.httpGet(url).parseHtml().body() + val root = body.selectFirst("table.itg") + ?.selectFirst("tbody") + ?: if (updateDm) { + body.parseFailed("Cannot find root") + } else { + updateDm = true + return getListPage(page, query, tags, sortOrder) + } + updateDm = false + nextPages[page + 1] = getNextTimestamp(body) + return root.children().mapNotNull { tr -> + if (tr.childrenSize() != 2) return@mapNotNull null + val (td1, td2) = tr.children() + val glink = td2.selectFirstOrThrow("div.glink") + val a = glink.parents().select("a").first() ?: glink.parseFailed("link not found") + val href = a.attrAsRelativeUrl("href") + val tagsDiv = glink.nextElementSibling() ?: glink.parseFailed("tags div not found") + val mainTag = td2.selectFirst("div.cn")?.let { div -> + MangaTag( + title = div.text().toTitleCase(), + key = tagIdByClass(div.classNames()) ?: return@let null, + source = source, + ) + } + Manga( + id = generateUid(href), + title = glink.text().cleanupTitle(), + altTitle = null, + url = href, + publicUrl = a.absUrl("href"), + rating = td2.selectFirst("div.ir")?.parseRating() ?: RATING_UNKNOWN, + isNsfw = true, + coverUrl = td1.selectFirst("img")?.absUrl("src").orEmpty(), + tags = setOfNotNull(mainTag), + state = null, + author = tagsDiv.getElementsContainingOwnText("artist:").first() + ?.nextElementSibling()?.text(), + source = source, + ) + } + } - override suspend fun getDetails(manga: Manga): Manga { - val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - val root = doc.body().selectFirstOrThrow("div.gm") - val cover = root.getElementById("gd1")?.children()?.first() - val title = root.getElementById("gd2") - val taglist = root.getElementById("taglist") - val tabs = doc.body().selectFirst("table.ptt")?.selectFirst("tr") - return manga.copy( - title = title?.getElementById("gn")?.text()?.cleanupTitle() ?: manga.title, - altTitle = title?.getElementById("gj")?.text()?.cleanupTitle() ?: manga.altTitle, - publicUrl = doc.baseUri().ifEmpty { manga.publicUrl }, - rating = root.getElementById("rating_label")?.text() - ?.substringAfterLast(' ') - ?.toFloatOrNull() - ?.div(5f) ?: manga.rating, - largeCoverUrl = cover?.styleValueOrNull("background")?.cssUrl(), - description = taglist?.select("tr")?.joinToString("
") { tr -> - val (tc, td) = tr.children() - val subtags = td.select("a").joinToString { it.html() } - "${tc.html()} $subtags" - }, - chapters = tabs?.select("a")?.findLast { a -> - a.text().toIntOrNull() != null - }?.let { a -> - val count = a.text().toInt() - val chapters = ChaptersListBuilder(count) - for (i in 1..count) { - val url = "${manga.url}?p=${i - 1}" - chapters += MangaChapter( - id = generateUid(url), - name = "${manga.title} #$i", - number = i, - url = url, - uploadDate = 0L, - source = source, - scanlator = null, - branch = null, - ) - } - chapters.toList() - }, - ) - } + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val root = doc.body().selectFirstOrThrow("div.gm") + val cover = root.getElementById("gd1")?.children()?.first() + val title = root.getElementById("gd2") + val taglist = root.getElementById("taglist") + val tabs = doc.body().selectFirst("table.ptt")?.selectFirst("tr") + return manga.copy( + title = title?.getElementById("gn")?.text()?.cleanupTitle() ?: manga.title, + altTitle = title?.getElementById("gj")?.text()?.cleanupTitle() ?: manga.altTitle, + publicUrl = doc.baseUri().ifEmpty { manga.publicUrl }, + rating = root.getElementById("rating_label")?.text() + ?.substringAfterLast(' ') + ?.toFloatOrNull() + ?.div(5f) ?: manga.rating, + largeCoverUrl = cover?.styleValueOrNull("background")?.cssUrl(), + description = taglist?.select("tr")?.joinToString("
") { tr -> + val (tc, td) = tr.children() + val subtags = td.select("a").joinToString { it.html() } + "${tc.html()} $subtags" + }, + chapters = tabs?.select("a")?.findLast { a -> + a.text().toIntOrNull() != null + }?.let { a -> + val count = a.text().toInt() + val chapters = ChaptersListBuilder(count) + for (i in 1..count) { + val url = "${manga.url}?p=${i - 1}" + chapters += MangaChapter( + id = generateUid(url), + name = "${manga.title} #$i", + number = i, + url = url, + uploadDate = 0L, + source = source, + scanlator = null, + branch = null, + ) + } + chapters.toList() + }, + ) + } - override suspend fun getPages(chapter: MangaChapter): List { - val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() - val root = doc.body().requireElementById("gdt") - return root.select("a").map { a -> - val url = a.attrAsRelativeUrl("href") - MangaPage( - id = generateUid(url), - url = url, - referer = a.absUrl("href"), - preview = null, - source = source, - ) - } - } + override suspend fun getPages(chapter: MangaChapter): List { + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + val root = doc.body().requireElementById("gdt") + return root.select("a").map { a -> + val url = a.attrAsRelativeUrl("href") + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } - override suspend fun getPageUrl(page: MangaPage): String { - val doc = webClient.httpGet(page.url.toAbsoluteUrl(domain)).parseHtml() - return doc.body().requireElementById("img").attrAsAbsoluteUrl("src") - } + override suspend fun getPageUrl(page: MangaPage): String { + val doc = webClient.httpGet(page.url.toAbsoluteUrl(domain)).parseHtml() + return doc.body().requireElementById("img").attrAsAbsoluteUrl("src") + } - override suspend fun getTags(): Set { - val doc = webClient.httpGet("https://${domain}").parseHtml() - val root = doc.body().requireElementById("searchbox").selectFirstOrThrow("table") - return root.select("div.cs").mapNotNullToSet { div -> - val id = div.id().substringAfterLast('_').toIntOrNull() - ?: return@mapNotNullToSet null - MangaTag( - title = div.text().toTitleCase(), - key = id.toString(), - source = source, - ) - } - } + override suspend fun getTags(): Set { + val doc = webClient.httpGet("https://${domain}").parseHtml() + val root = doc.body().requireElementById("searchbox").selectFirstOrThrow("table") + return root.select("div.cs").mapNotNullToSet { div -> + val id = div.id().substringAfterLast('_').toIntOrNull() + ?: return@mapNotNullToSet null + MangaTag( + title = div.text().toTitleCase(), + key = id.toString(), + source = source, + ) + } + } - override suspend fun getUsername(): String { - val doc = webClient.httpGet("https://forums.$DOMAIN_UNAUTHORIZED/").parseHtml().body() - val username = doc.getElementById("userlinks") - ?.getElementsByAttributeValueContaining("href", "showuser=") - ?.firstOrNull() - ?.ownText() - ?: if (doc.getElementById("userlinksguest") != null) { - throw AuthRequiredException(source) - } else { - doc.parseFailed() - } - return username - } + override suspend fun getUsername(): String { + val doc = webClient.httpGet("https://forums.$DOMAIN_UNAUTHORIZED/").parseHtml().body() + val username = doc.getElementById("userlinks") + ?.getElementsByAttributeValueContaining("href", "showuser=") + ?.firstOrNull() + ?.ownText() + ?: if (doc.getElementById("userlinksguest") != null) { + throw AuthRequiredException(source) + } else { + doc.parseFailed() + } + return username + } - override fun onCreateConfig(keys: MutableCollection>) { - super.onCreateConfig(keys) - keys.add(suspiciousContentKey) - } + override fun onCreateConfig(keys: MutableCollection>) { + super.onCreateConfig(keys) + keys.add(suspiciousContentKey) + } - private fun isAuthorized(domain: String): Boolean { - val cookies = context.cookieJar.getCookies(domain).mapToSet { x -> x.name } - return authCookies.all { it in cookies } - } + private fun isAuthorized(domain: String): Boolean { + val cookies = context.cookieJar.getCookies(domain).mapToSet { x -> x.name } + return authCookies.all { it in cookies } + } - private fun Element.parseRating(): Float { - return runCatching { - val style = requireNotNull(attr("style")) - val (v1, v2) = ratingPattern.find(style)!!.destructured - var p1 = v1.dropLast(2).toInt() - val p2 = v2.dropLast(2).toInt() - if (p2 != -1) { - p1 += 8 - } - (80 - p1) / 80f - }.getOrDefault(RATING_UNKNOWN) - } + private fun Element.parseRating(): Float { + return runCatching { + val style = requireNotNull(attr("style")) + val (v1, v2) = ratingPattern.find(style)!!.destructured + var p1 = v1.dropLast(2).toInt() + val p2 = v2.dropLast(2).toInt() + if (p2 != -1) { + p1 += 8 + } + (80 - p1) / 80f + }.getOrDefault(RATING_UNKNOWN) + } - private fun String.cleanupTitle(): String { - val result = StringBuilder(length) - var skip = false - for (c in this) { - when { - c == '[' -> skip = true - c == ']' -> skip = false - c.isWhitespace() && result.isEmpty() -> continue - !skip -> result.append(c) - } - } - while (result.lastOrNull()?.isWhitespace() == true) { - result.deleteCharAt(result.lastIndex) - } - return result.toString() - } + private fun String.cleanupTitle(): String { + val result = StringBuilder(length) + var skip = false + for (c in this) { + when { + c == '[' -> skip = true + c == ']' -> skip = false + c.isWhitespace() && result.isEmpty() -> continue + !skip -> result.append(c) + } + } + while (result.lastOrNull()?.isWhitespace() == true) { + result.deleteCharAt(result.lastIndex) + } + return result.toString() + } - private fun String.cssUrl(): String? { - val fromIndex = indexOf("url(") - if (fromIndex == -1) { - return null - } - val toIndex = indexOf(')', startIndex = fromIndex) - return if (toIndex == -1) { - null - } else { - substring(fromIndex + 4, toIndex).trim() - } - } + private fun String.cssUrl(): String? { + val fromIndex = indexOf("url(") + if (fromIndex == -1) { + return null + } + val toIndex = indexOf(')', startIndex = fromIndex) + return if (toIndex == -1) { + null + } else { + substring(fromIndex + 4, toIndex).trim() + } + } - private fun tagIdByClass(classNames: Collection): String? { - val className = classNames.find { x -> x.startsWith("ct") } ?: return null - val num = className.drop(2).toIntOrNull(16) ?: return null - return 2.0.pow(num).toInt().toString() - } + private fun tagIdByClass(classNames: Collection): String? { + val className = classNames.find { x -> x.startsWith("ct") } ?: return null + val num = className.drop(2).toIntOrNull(16) ?: return null + return 2.0.pow(num).toInt().toString() + } - private fun getNextTimestamp(root: Element): Long { - return root.getElementById("unext") - ?.attrAsAbsoluteUrlOrNull("href") - ?.toHttpUrlOrNull() - ?.queryParameter("next") - ?.toLongOrNull() ?: 1 - } + private fun getNextTimestamp(root: Element): Long { + return root.getElementById("unext") + ?.attrAsAbsoluteUrlOrNull("href") + ?.toHttpUrlOrNull() + ?.queryParameter("next") + ?.toLongOrNull() ?: 1 + } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaDexParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaDexParser.kt index ed236b15..96b795ef 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaDexParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaDexParser.kt @@ -22,254 +22,252 @@ private const val CHAPTERS_MAX_PAGE_SIZE = 500 private const val CHAPTERS_PARALLELISM = 3 private const val CHAPTERS_MAX_COUNT = 10_000 // strange api behavior, looks like a bug private const val CONTENT_RATING = - "contentRating[]=safe&contentRating[]=suggestive&contentRating[]=erotica&contentRating[]=pornographic" + "contentRating[]=safe&contentRating[]=suggestive&contentRating[]=erotica&contentRating[]=pornographic" private const val LOCALE_FALLBACK = "en" @MangaSourceParser("MANGADEX", "MangaDex") internal class MangaDexParser(context: MangaLoaderContext) : MangaParser(context, MangaSource.MANGADEX) { - override val configKeyDomain = ConfigKey.Domain("mangadex.org", null) + override val configKeyDomain = ConfigKey.Domain("mangadex.org", null) - override val sortOrders: EnumSet = EnumSet.of( - SortOrder.UPDATED, - SortOrder.ALPHABETICAL, - SortOrder.NEWEST, - SortOrder.POPULARITY, - ) + override val sortOrders: EnumSet = EnumSet.of( + SortOrder.UPDATED, + SortOrder.ALPHABETICAL, + SortOrder.NEWEST, + SortOrder.POPULARITY, + ) - override suspend fun getList( - offset: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - val domain = domain - val url = buildString { - append("https://api.") - append(domain) - append("/manga?limit=") - append(PAGE_SIZE) - append("&offset=") - append(offset) - append("&includes[]=cover_art&includes[]=author&includes[]=artist&") - tags?.forEach { tag -> - append("includedTags[]=") - append(tag.key) - append('&') - } - if (!query.isNullOrEmpty()) { - append("title=") - append(query.urlEncoded()) - append('&') - } - append(CONTENT_RATING) - append("&order") - append( - when (sortOrder) { - SortOrder.UPDATED, - -> "[latestUploadedChapter]=desc" + override suspend fun getList( + offset: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + val domain = domain + val url = buildString { + append("https://api.") + append(domain) + append("/manga?limit=") + append(PAGE_SIZE) + append("&offset=") + append(offset) + append("&includes[]=cover_art&includes[]=author&includes[]=artist&") + tags?.forEach { tag -> + append("includedTags[]=") + append(tag.key) + append('&') + } + if (!query.isNullOrEmpty()) { + append("title=") + append(query.urlEncoded()) + append('&') + } + append(CONTENT_RATING) + append("&order") + append( + when (sortOrder) { + SortOrder.UPDATED, + -> "[latestUploadedChapter]=desc" - SortOrder.ALPHABETICAL -> "[title]=asc" - SortOrder.NEWEST -> "[createdAt]=desc" - SortOrder.POPULARITY -> "[followedCount]=desc" - else -> "[followedCount]=desc" - }, - ) - } - val json = webClient.httpGet(url).parseJson().getJSONArray("data") - return json.mapJSON { jo -> - val id = jo.getString("id") - val attrs = jo.getJSONObject("attributes") - val relations = jo.getJSONArray("relationships").associateByKey("type") - val cover = relations["cover_art"] - ?.getJSONObject("attributes") - ?.getString("fileName") - ?.let { - "https://uploads.$domain/covers/$id/$it" - } - Manga( - id = generateUid(id), - title = requireNotNull(attrs.getJSONObject("title").selectByLocale()) { - "Title should not be null" - }, - altTitle = attrs.optJSONObject("altTitles")?.selectByLocale(), - url = id, - publicUrl = "https://$domain/title/$id", - rating = RATING_UNKNOWN, - isNsfw = attrs.getStringOrNull("contentRating") == "erotica", - coverUrl = cover?.plus(".256.jpg").orEmpty(), - largeCoverUrl = cover, - description = attrs.optJSONObject("description")?.selectByLocale(), - tags = attrs.getJSONArray("tags").mapJSONToSet { tag -> - MangaTag( - title = tag.getJSONObject("attributes") - .getJSONObject("name") - .firstStringValue() - .toTitleCase(), - key = tag.getString("id"), - source = source, - ) - }, - state = when (jo.getStringOrNull("status")) { - "ongoing" -> MangaState.ONGOING - "completed" -> MangaState.FINISHED - else -> null - }, - author = (relations["author"] ?: relations["artist"]) - ?.getJSONObject("attributes") - ?.getStringOrNull("name"), - source = source, - ) - } - } + SortOrder.ALPHABETICAL -> "[title]=asc" + SortOrder.NEWEST -> "[createdAt]=desc" + SortOrder.POPULARITY -> "[followedCount]=desc" + else -> "[followedCount]=desc" + }, + ) + } + val json = webClient.httpGet(url).parseJson().getJSONArray("data") + return json.mapJSON { jo -> + val id = jo.getString("id") + val attrs = jo.getJSONObject("attributes") + val relations = jo.getJSONArray("relationships").associateByKey("type") + val cover = relations["cover_art"] + ?.getJSONObject("attributes") + ?.getString("fileName") + ?.let { + "https://uploads.$domain/covers/$id/$it" + } + Manga( + id = generateUid(id), + title = requireNotNull(attrs.getJSONObject("title").selectByLocale()) { + "Title should not be null" + }, + altTitle = attrs.optJSONObject("altTitles")?.selectByLocale(), + url = id, + publicUrl = "https://$domain/title/$id", + rating = RATING_UNKNOWN, + isNsfw = attrs.getStringOrNull("contentRating") == "erotica", + coverUrl = cover?.plus(".256.jpg").orEmpty(), + largeCoverUrl = cover, + description = attrs.optJSONObject("description")?.selectByLocale(), + tags = attrs.getJSONArray("tags").mapJSONToSet { tag -> + MangaTag( + title = tag.getJSONObject("attributes") + .getJSONObject("name") + .firstStringValue() + .toTitleCase(), + key = tag.getString("id"), + source = source, + ) + }, + state = when (jo.getStringOrNull("status")) { + "ongoing" -> MangaState.ONGOING + "completed" -> MangaState.FINISHED + else -> null + }, + author = (relations["author"] ?: relations["artist"]) + ?.getJSONObject("attributes") + ?.getStringOrNull("name"), + source = source, + ) + } + } - override suspend fun getDetails(manga: Manga): Manga = coroutineScope { - val domain = domain - val mangaId = manga.url.removePrefix("/") - val attrsDeferred = async { - webClient.httpGet( - "https://api.$domain/manga/${mangaId}?includes[]=artist&includes[]=author&includes[]=cover_art", - ).parseJson().getJSONObject("data").getJSONObject("attributes") - } - val feedDeferred = async { loadChapters(mangaId) } - val mangaAttrs = attrsDeferred.await() - val feed = feedDeferred.await() - // 2022-01-02T00:27:11+00:00 - val dateFormat = SimpleDateFormat( - "yyyy-MM-dd'T'HH:mm:ss'+00:00'", - Locale.ROOT, - ) - manga.copy( - description = mangaAttrs.optJSONObject("description")?.selectByLocale() - ?: manga.description, - chapters = feed.mapChapters { _, jo -> - val id = jo.getString("id") - val attrs = jo.getJSONObject("attributes") - if (!attrs.isNull("externalUrl")) { - return@mapChapters null - } - val locale = attrs.getStringOrNull("translatedLanguage")?.let { Locale.forLanguageTag(it) } - val relations = jo.getJSONArray("relationships").associateByKey("type") - val number = attrs.getIntOrDefault("chapter", 0) - MangaChapter( - id = generateUid(id), - name = attrs.getStringOrNull("title")?.takeUnless(String::isEmpty) - ?: "Chapter #$number", - number = number, - url = id, - scanlator = relations["scanlation_group"]?.getStringOrNull("name"), - uploadDate = dateFormat.tryParse(attrs.getString("publishAt")), - branch = locale?.getDisplayName(locale)?.toTitleCase(locale), - source = source, - ) - }, - ) - } + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { + val domain = domain + val mangaId = manga.url.removePrefix("/") + val attrsDeferred = async { + webClient.httpGet( + "https://api.$domain/manga/${mangaId}?includes[]=artist&includes[]=author&includes[]=cover_art", + ).parseJson().getJSONObject("data").getJSONObject("attributes") + } + val feedDeferred = async { loadChapters(mangaId) } + val mangaAttrs = attrsDeferred.await() + val feed = feedDeferred.await() + // 2022-01-02T00:27:11+00:00 + val dateFormat = SimpleDateFormat( + "yyyy-MM-dd'T'HH:mm:ss'+00:00'", + Locale.ROOT, + ) + manga.copy( + description = mangaAttrs.optJSONObject("description")?.selectByLocale() + ?: manga.description, + chapters = feed.mapChapters { _, jo -> + val id = jo.getString("id") + val attrs = jo.getJSONObject("attributes") + if (!attrs.isNull("externalUrl")) { + return@mapChapters null + } + val locale = attrs.getStringOrNull("translatedLanguage")?.let { Locale.forLanguageTag(it) } + val relations = jo.getJSONArray("relationships").associateByKey("type") + val number = attrs.getIntOrDefault("chapter", 0) + MangaChapter( + id = generateUid(id), + name = attrs.getStringOrNull("title")?.takeUnless(String::isEmpty) + ?: "Chapter #$number", + number = number, + url = id, + scanlator = relations["scanlation_group"]?.getStringOrNull("name"), + uploadDate = dateFormat.tryParse(attrs.getString("publishAt")), + branch = locale?.getDisplayName(locale)?.toTitleCase(locale), + source = source, + ) + }, + ) + } - override suspend fun getPages(chapter: MangaChapter): List { - val domain = domain - val chapterJson = webClient.httpGet("https://api.$domain/at-home/server/${chapter.url}?forcePort443=false") - .parseJson() - .getJSONObject("chapter") - val pages = chapterJson.getJSONArray("data") - val prefix = "https://uploads.$domain/data/${chapterJson.getString("hash")}/" - val referer = "https://$domain/" - return List(pages.length()) { i -> - val url = prefix + pages.getString(i) - MangaPage( - id = generateUid(url), - url = url, - referer = referer, - preview = null, // TODO prefix + dataSaver.getString(i), - source = source, - ) - } - } + override suspend fun getPages(chapter: MangaChapter): List { + val domain = domain + val chapterJson = webClient.httpGet("https://api.$domain/at-home/server/${chapter.url}?forcePort443=false") + .parseJson() + .getJSONObject("chapter") + val pages = chapterJson.getJSONArray("data") + val prefix = "https://uploads.$domain/data/${chapterJson.getString("hash")}/" + return List(pages.length()) { i -> + val url = prefix + pages.getString(i) + MangaPage( + id = generateUid(url), + url = url, + preview = null, // TODO prefix + dataSaver.getString(i), + source = source, + ) + } + } - override suspend fun getTags(): Set { - val tags = webClient.httpGet("https://api.${domain}/manga/tag").parseJson() - .getJSONArray("data") - return tags.mapJSONToSet { jo -> - MangaTag( - title = jo.getJSONObject("attributes").getJSONObject("name").firstStringValue().toTitleCase(), - key = jo.getString("id"), - source = source, - ) - } - } + override suspend fun getTags(): Set { + val tags = webClient.httpGet("https://api.${domain}/manga/tag").parseJson() + .getJSONArray("data") + return tags.mapJSONToSet { jo -> + MangaTag( + title = jo.getJSONObject("attributes").getJSONObject("name").firstStringValue().toTitleCase(), + key = jo.getString("id"), + source = source, + ) + } + } - private fun JSONObject.firstStringValue() = values().next() as String + private fun JSONObject.firstStringValue() = values().next() as String - private fun JSONObject.selectByLocale(): String? { - val preferredLocales = context.getPreferredLocales() - for (locale in preferredLocales) { - getStringOrNull(locale.language)?.let { return it } - getStringOrNull(locale.toLanguageTag())?.let { return it } - } - return getStringOrNull(LOCALE_FALLBACK) ?: values().nextOrNull() as? String - } + private fun JSONObject.selectByLocale(): String? { + val preferredLocales = context.getPreferredLocales() + for (locale in preferredLocales) { + getStringOrNull(locale.language)?.let { return it } + getStringOrNull(locale.toLanguageTag())?.let { return it } + } + return getStringOrNull(LOCALE_FALLBACK) ?: values().nextOrNull() as? String + } - private suspend fun loadChapters(mangaId: String): List { - val firstPage = loadChapters(mangaId, offset = 0, limit = CHAPTERS_FIRST_PAGE_SIZE) - if (firstPage.size >= firstPage.total) { - return firstPage.data - } - val tail = coroutineScope { - val leftCount = firstPage.total.coerceAtMost(CHAPTERS_MAX_COUNT) - firstPage.size - val pages = (leftCount / CHAPTERS_MAX_PAGE_SIZE.toFloat()).toIntUp() - val dispatcher = Dispatchers.Default.limitedParallelism(CHAPTERS_PARALLELISM) - List(pages) { page -> - val offset = page * CHAPTERS_MAX_PAGE_SIZE + firstPage.size - async(dispatcher) { - loadChapters(mangaId, offset, CHAPTERS_MAX_PAGE_SIZE) - } - }.awaitAll() - } - val result = ArrayList(firstPage.total) - result += firstPage.data - tail.flatMapTo(result) { it.data } - return result - } + private suspend fun loadChapters(mangaId: String): List { + val firstPage = loadChapters(mangaId, offset = 0, limit = CHAPTERS_FIRST_PAGE_SIZE) + if (firstPage.size >= firstPage.total) { + return firstPage.data + } + val tail = coroutineScope { + val leftCount = firstPage.total.coerceAtMost(CHAPTERS_MAX_COUNT) - firstPage.size + val pages = (leftCount / CHAPTERS_MAX_PAGE_SIZE.toFloat()).toIntUp() + val dispatcher = Dispatchers.Default.limitedParallelism(CHAPTERS_PARALLELISM) + List(pages) { page -> + val offset = page * CHAPTERS_MAX_PAGE_SIZE + firstPage.size + async(dispatcher) { + loadChapters(mangaId, offset, CHAPTERS_MAX_PAGE_SIZE) + } + }.awaitAll() + } + val result = ArrayList(firstPage.total) + result += firstPage.data + tail.flatMapTo(result) { it.data } + return result + } - private suspend fun loadChapters(mangaId: String, offset: Int, limit: Int): Chapters { - val limitedLimit = when { - offset >= CHAPTERS_MAX_COUNT -> return Chapters(emptyList(), CHAPTERS_MAX_COUNT) - offset + limit > CHAPTERS_MAX_COUNT -> CHAPTERS_MAX_COUNT - offset - else -> limit - } - val url = buildString { - append("https://api.") - append(domain) - append("/manga/") - append(mangaId) - append("/feed") - append("?limit=") - append(limitedLimit) - append("&includes[]=scanlation_group&order[volume]=asc&order[chapter]=asc&offset=") - append(offset) - append('&') - append(CONTENT_RATING) - } - val json = webClient.httpGet(url).parseJson() - if (json.getString("result") == "ok") { - return Chapters( - data = json.optJSONArray("data")?.toJSONList().orEmpty(), - total = json.getInt("total"), - ) - } else { - val error = json.optJSONArray("errors").mapJSON { jo -> - jo.getString("detail") - }.joinToString("\n") - throw ParseException(error, url) - } - } + private suspend fun loadChapters(mangaId: String, offset: Int, limit: Int): Chapters { + val limitedLimit = when { + offset >= CHAPTERS_MAX_COUNT -> return Chapters(emptyList(), CHAPTERS_MAX_COUNT) + offset + limit > CHAPTERS_MAX_COUNT -> CHAPTERS_MAX_COUNT - offset + else -> limit + } + val url = buildString { + append("https://api.") + append(domain) + append("/manga/") + append(mangaId) + append("/feed") + append("?limit=") + append(limitedLimit) + append("&includes[]=scanlation_group&order[volume]=asc&order[chapter]=asc&offset=") + append(offset) + append('&') + append(CONTENT_RATING) + } + val json = webClient.httpGet(url).parseJson() + if (json.getString("result") == "ok") { + return Chapters( + data = json.optJSONArray("data")?.toJSONList().orEmpty(), + total = json.getInt("total"), + ) + } else { + val error = json.optJSONArray("errors").mapJSON { jo -> + jo.getString("detail") + }.joinToString("\n") + throw ParseException(error, url) + } + } - private class Chapters( - val data: List, - val total: Int, - ) { + private class Chapters( + val data: List, + val total: Int, + ) { - val size: Int - get() = data.size - } + val size: Int + get() = data.size + } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaInUaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaInUaParser.kt index 68726425..549fa3f1 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaInUaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaInUaParser.kt @@ -13,141 +13,140 @@ private const val DEF_BRANCH_NAME = "Основний переклад" @MangaSourceParser("MANGAINUA", "MANGA/in/UA", "uk") class MangaInUaParser(context: MangaLoaderContext) : PagedMangaParser( - context = context, - source = MangaSource.MANGAINUA, - pageSize = 24, - searchPageSize = 10, + context = context, + source = MangaSource.MANGAINUA, + pageSize = 24, + searchPageSize = 10, ) { - override val sortOrders: Set - get() = Collections.singleton(SortOrder.UPDATED) + override val sortOrders: Set + get() = Collections.singleton(SortOrder.UPDATED) - override val configKeyDomain: ConfigKey.Domain = ConfigKey.Domain("manga.in.ua", null) + override val configKeyDomain: ConfigKey.Domain = ConfigKey.Domain("manga.in.ua", null) - override suspend fun getListPage( - page: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - val url = when { - !query.isNullOrEmpty() -> ( - "/index.php?do=search" + - "&subaction=search" + - "&search_start=$page" + - "&full_search=1" + - "&story=$query" + - "&titleonly=3" - ).toAbsoluteUrl(domain) + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + val url = when { + !query.isNullOrEmpty() -> ( + "/index.php?do=search" + + "&subaction=search" + + "&search_start=$page" + + "&full_search=1" + + "&story=$query" + + "&titleonly=3" + ).toAbsoluteUrl(domain) - tags.isNullOrEmpty() -> "/mangas/page/$page".toAbsoluteUrl(domain) - tags.size == 1 -> "${tags.first().key}/page/$page" - tags.size > 1 -> throw IllegalArgumentException("This source supports only 1 genre") - else -> "/mangas/page/$page".toAbsoluteUrl(domain) - } - val doc = webClient.httpGet(url).parseHtml() - val container = doc.body().requireElementById("site-content") - val items = container.select("div.col-6") - return items.mapNotNull { item -> - val href = item.selectFirst("a")?.attrAsRelativeUrl("href") ?: return@mapNotNull null - Manga( - id = generateUid(href), - title = item.selectFirst("h3.card__title")?.text() ?: return@mapNotNull null, - coverUrl = item.selectFirst("header.card__cover")?.selectFirst("img")?.run { - attrAsAbsoluteUrlOrNull("data-src") ?: attrAsAbsoluteUrlOrNull("src") - }.orEmpty(), - altTitle = null, - author = null, - rating = item.selectFirst("div.card__short-rate--num") - ?.text() - ?.toFloatOrNull() - ?.div(10F) ?: RATING_UNKNOWN, - url = href, - isNsfw = item.selectFirst("ul.card__list")?.select("li")?.lastOrNull()?.text() == "18+", - tags = runCatching { - item.selectFirst("div.card__category")?.select("a")?.mapToSet { - MangaTag( - title = it.ownText(), - key = it.attr("href").removeSuffix("/"), - source = source, - ) - } - }.getOrNull().orEmpty(), - state = null, - publicUrl = href.toAbsoluteUrl(container.host ?: domain), - source = source, - ) - } - } + tags.isNullOrEmpty() -> "/mangas/page/$page".toAbsoluteUrl(domain) + tags.size == 1 -> "${tags.first().key}/page/$page" + tags.size > 1 -> throw IllegalArgumentException("This source supports only 1 genre") + else -> "/mangas/page/$page".toAbsoluteUrl(domain) + } + val doc = webClient.httpGet(url).parseHtml() + val container = doc.body().requireElementById("site-content") + val items = container.select("div.col-6") + return items.mapNotNull { item -> + val href = item.selectFirst("a")?.attrAsRelativeUrl("href") ?: return@mapNotNull null + Manga( + id = generateUid(href), + title = item.selectFirst("h3.card__title")?.text() ?: return@mapNotNull null, + coverUrl = item.selectFirst("header.card__cover")?.selectFirst("img")?.run { + attrAsAbsoluteUrlOrNull("data-src") ?: attrAsAbsoluteUrlOrNull("src") + }.orEmpty(), + altTitle = null, + author = null, + rating = item.selectFirst("div.card__short-rate--num") + ?.text() + ?.toFloatOrNull() + ?.div(10F) ?: RATING_UNKNOWN, + url = href, + isNsfw = item.selectFirst("ul.card__list")?.select("li")?.lastOrNull()?.text() == "18+", + tags = runCatching { + item.selectFirst("div.card__category")?.select("a")?.mapToSet { + MangaTag( + title = it.ownText(), + key = it.attr("href").removeSuffix("/"), + source = source, + ) + } + }.getOrNull().orEmpty(), + state = null, + publicUrl = href.toAbsoluteUrl(container.host ?: domain), + source = source, + ) + } + } - override suspend fun getDetails(manga: Manga): Manga { - val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - val root = doc.body().requireElementById("site-content") - val dateFormat = SimpleDateFormat("dd.MM.yyyy", Locale.US) - val chapterNodes = root.selectFirstOrThrow(".linkstocomics").select(".ltcitems") - var prevChapterName: String? = null - var i = 0 - return manga.copy( - description = root.selectFirst("div.item__full-description")?.text(), - largeCoverUrl = root.selectFirst("div.item__full-sidebar--poster")?.selectFirst("img") - ?.attrAsAbsoluteUrlOrNull("src"), - chapters = chapterNodes.mapChapters { _, item -> - val href = item?.selectFirst("a")?.attrAsRelativeUrlOrNull("href") - ?: return@mapChapters null - val isAlternative = item.styleValueOrNull("background") != null - val name = item.selectFirst("a")?.text().orEmpty() - if (!isAlternative) i++ - MangaChapter( - id = generateUid(href), - name = if (isAlternative) { - prevChapterName ?: return@mapChapters null - } else { - prevChapterName = name - name - }, - number = i, - url = href, - scanlator = null, - branch = if (isAlternative) { - name.substringAfterLast(':').trim() - } else { - DEF_BRANCH_NAME - }, - uploadDate = dateFormat.tryParse(item.selectFirst("div.ltcright")?.text()), - source = source, - ) - }, - ) - } + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val root = doc.body().requireElementById("site-content") + val dateFormat = SimpleDateFormat("dd.MM.yyyy", Locale.US) + val chapterNodes = root.selectFirstOrThrow(".linkstocomics").select(".ltcitems") + var prevChapterName: String? = null + var i = 0 + return manga.copy( + description = root.selectFirst("div.item__full-description")?.text(), + largeCoverUrl = root.selectFirst("div.item__full-sidebar--poster")?.selectFirst("img") + ?.attrAsAbsoluteUrlOrNull("src"), + chapters = chapterNodes.mapChapters { _, item -> + val href = item?.selectFirst("a")?.attrAsRelativeUrlOrNull("href") + ?: return@mapChapters null + val isAlternative = item.styleValueOrNull("background") != null + val name = item.selectFirst("a")?.text().orEmpty() + if (!isAlternative) i++ + MangaChapter( + id = generateUid(href), + name = if (isAlternative) { + prevChapterName ?: return@mapChapters null + } else { + prevChapterName = name + name + }, + number = i, + url = href, + scanlator = null, + branch = if (isAlternative) { + name.substringAfterLast(':').trim() + } else { + DEF_BRANCH_NAME + }, + uploadDate = dateFormat.tryParse(item.selectFirst("div.ltcright")?.text()), + source = source, + ) + }, + ) + } - override suspend fun getPages(chapter: MangaChapter): List { - val fullUrl = chapter.url.toAbsoluteUrl(domain) - val doc = webClient.httpGet(fullUrl).parseHtml() - val root = doc.body().requireElementById("comics").selectFirstOrThrow("ul.xfieldimagegallery") - return root.select("li").map { ul -> - val img = ul.selectFirstOrThrow("img") - val url = img.attrAsAbsoluteUrl("data-src") - MangaPage( - id = generateUid(url), - url = url, - preview = null, - referer = fullUrl, - source = source, - ) - } - } + override suspend fun getPages(chapter: MangaChapter): List { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + val root = doc.body().requireElementById("comics").selectFirstOrThrow("ul.xfieldimagegallery") + return root.select("li").map { ul -> + val img = ul.selectFirstOrThrow("img") + val url = img.attrAsAbsoluteUrl("data-src") + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } - override suspend fun getTags(): Set { - val domain = domain - val doc = webClient.httpGet("https://$domain/mangas").parseHtml() - val root = doc.body().requireElementById("menu_1").selectFirstOrThrow("div.menu__wrapper") - return root.select("li").mapNotNullToSet { li -> - val a = li.selectFirst("a") ?: return@mapNotNullToSet null - MangaTag( - title = a.ownText(), - key = a.attr("href").removeSuffix("/"), - source = source, - ) - } - } + override suspend fun getTags(): Set { + val domain = domain + val doc = webClient.httpGet("https://$domain/mangas").parseHtml() + val root = doc.body().requireElementById("menu_1").selectFirstOrThrow("div.menu__wrapper") + return root.select("li").mapNotNullToSet { li -> + val a = li.selectFirst("a") ?: return@mapNotNullToSet null + MangaTag( + title = a.ownText(), + key = a.attr("href").removeSuffix("/"), + source = source, + ) + } + } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaTownParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaTownParser.kt index dbc5ee62..60f8de5e 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaTownParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaTownParser.kt @@ -13,205 +13,204 @@ import java.util.* @MangaSourceParser("MANGATOWN", "MangaTown", "en") internal class MangaTownParser(context: MangaLoaderContext) : MangaParser(context, MangaSource.MANGATOWN) { - override val configKeyDomain = ConfigKey.Domain("www.mangatown.com", null) + override val configKeyDomain = ConfigKey.Domain("www.mangatown.com", null) - override val sortOrders: Set = EnumSet.of( - SortOrder.ALPHABETICAL, - SortOrder.RATING, - SortOrder.POPULARITY, - SortOrder.UPDATED, - ) + override val sortOrders: Set = EnumSet.of( + SortOrder.ALPHABETICAL, + SortOrder.RATING, + SortOrder.POPULARITY, + SortOrder.UPDATED, + ) - private val regexTag = Regex("[^\\-]+-[^\\-]+-[^\\-]+-[^\\-]+-[^\\-]+-[^\\-]+") + private val regexTag = Regex("[^\\-]+-[^\\-]+-[^\\-]+-[^\\-]+-[^\\-]+-[^\\-]+") - override suspend fun getList( - offset: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - val sortKey = when (sortOrder) { - SortOrder.ALPHABETICAL -> "?name.az" - SortOrder.RATING -> "?rating.za" - SortOrder.UPDATED -> "?last_chapter_time.za" - else -> "" - } - val page = (offset / 30) + 1 - val url = when { - !query.isNullOrEmpty() -> { - if (offset != 0) { - return emptyList() - } - "/search?name=${query.urlEncoded()}".toAbsoluteUrl(domain) - } + override suspend fun getList( + offset: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + val sortKey = when (sortOrder) { + SortOrder.ALPHABETICAL -> "?name.az" + SortOrder.RATING -> "?rating.za" + SortOrder.UPDATED -> "?last_chapter_time.za" + else -> "" + } + val page = (offset / 30) + 1 + val url = when { + !query.isNullOrEmpty() -> { + if (offset != 0) { + return emptyList() + } + "/search?name=${query.urlEncoded()}".toAbsoluteUrl(domain) + } - tags.isNullOrEmpty() -> "/directory/$page.htm$sortKey".toAbsoluteUrl(domain) - tags.size == 1 -> "/directory/${tags.first().key}/$page.htm$sortKey".toAbsoluteUrl(domain) - else -> tags.joinToString( - prefix = "/search?page=$page".toAbsoluteUrl(domain), - ) { tag -> - "&genres[${tag.key}]=1" - } - } - val doc = webClient.httpGet(url).parseHtml() - val root = doc.body().selectFirstOrThrow("ul.manga_pic_list") - return root.select("li").mapNotNull { li -> - val a = li.selectFirst("a.manga_cover") - val href = a?.attrAsRelativeUrlOrNull("href") - ?: return@mapNotNull null - val views = li.select("p.view") - val status = views.firstNotNullOfOrNull { it.ownText().takeIf { x -> x.startsWith("Status:") } } - ?.substringAfter(':')?.trim()?.lowercase(Locale.ROOT) - Manga( - id = generateUid(href), - title = a.attr("title"), - coverUrl = a.selectFirst("img")?.absUrl("src").orEmpty(), - source = MangaSource.MANGATOWN, - altTitle = null, - rating = li.selectFirst("p.score")?.selectFirst("b") - ?.ownText()?.toFloatOrNull()?.div(5f) ?: RATING_UNKNOWN, - author = views.firstNotNullOfOrNull { it.text().takeIf { x -> x.startsWith("Author:") } } - ?.substringAfter(':') - ?.trim(), - state = when (status) { - "ongoing" -> MangaState.ONGOING - "completed" -> MangaState.FINISHED - else -> null - }, - tags = li.selectFirst("p.keyWord")?.select("a")?.mapNotNullToSet tags@{ x -> - MangaTag( - title = x.attr("title").toTitleCase(), - key = x.attr("href").parseTagKey() ?: return@tags null, - source = MangaSource.MANGATOWN, - ) - }.orEmpty(), - url = href, - isNsfw = false, - publicUrl = href.toAbsoluteUrl(a.host ?: domain), - ) - } - } + tags.isNullOrEmpty() -> "/directory/$page.htm$sortKey".toAbsoluteUrl(domain) + tags.size == 1 -> "/directory/${tags.first().key}/$page.htm$sortKey".toAbsoluteUrl(domain) + else -> tags.joinToString( + prefix = "/search?page=$page".toAbsoluteUrl(domain), + ) { tag -> + "&genres[${tag.key}]=1" + } + } + val doc = webClient.httpGet(url).parseHtml() + val root = doc.body().selectFirstOrThrow("ul.manga_pic_list") + return root.select("li").mapNotNull { li -> + val a = li.selectFirst("a.manga_cover") + val href = a?.attrAsRelativeUrlOrNull("href") + ?: return@mapNotNull null + val views = li.select("p.view") + val status = views.firstNotNullOfOrNull { it.ownText().takeIf { x -> x.startsWith("Status:") } } + ?.substringAfter(':')?.trim()?.lowercase(Locale.ROOT) + Manga( + id = generateUid(href), + title = a.attr("title"), + coverUrl = a.selectFirst("img")?.absUrl("src").orEmpty(), + source = MangaSource.MANGATOWN, + altTitle = null, + rating = li.selectFirst("p.score")?.selectFirst("b") + ?.ownText()?.toFloatOrNull()?.div(5f) ?: RATING_UNKNOWN, + author = views.firstNotNullOfOrNull { it.text().takeIf { x -> x.startsWith("Author:") } } + ?.substringAfter(':') + ?.trim(), + state = when (status) { + "ongoing" -> MangaState.ONGOING + "completed" -> MangaState.FINISHED + else -> null + }, + tags = li.selectFirst("p.keyWord")?.select("a")?.mapNotNullToSet tags@{ x -> + MangaTag( + title = x.attr("title").toTitleCase(), + key = x.attr("href").parseTagKey() ?: return@tags null, + source = MangaSource.MANGATOWN, + ) + }.orEmpty(), + url = href, + isNsfw = false, + publicUrl = href.toAbsoluteUrl(a.host ?: domain), + ) + } + } - override suspend fun getDetails(manga: Manga): Manga { - val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - val root = doc.body().selectFirstOrThrow("section.main") - .selectFirstOrThrow("div.article_content") - val info = root.selectFirst("div.detail_info")?.selectFirst("ul") - val chaptersList = root.selectFirst("div.chapter_content") - ?.selectFirst("ul.chapter_list")?.select("li")?.asReversed() - val dateFormat = SimpleDateFormat("MMM dd,yyyy", Locale.US) - return manga.copy( - tags = manga.tags + info?.select("li")?.find { x -> - x.selectFirst("b")?.ownText() == "Genre(s):" - }?.select("a")?.mapNotNull { a -> - MangaTag( - title = a.attr("title").toTitleCase(), - key = a.attr("href").parseTagKey() ?: return@mapNotNull null, - source = MangaSource.MANGATOWN, - ) - }.orEmpty(), - description = info?.getElementById("show")?.ownText(), - chapters = chaptersList?.mapChapters { i, li -> - val href = li.selectFirst("a")?.attrAsRelativeUrlOrNull("href") - ?: return@mapChapters null - val name = li.select("span") - .filter { x -> x.className().isEmpty() } - .joinToString(" - ") { it.text() }.trim() - MangaChapter( - id = generateUid(href), - url = href, - source = MangaSource.MANGATOWN, - number = i + 1, - uploadDate = parseChapterDate( - dateFormat, - li.selectFirst("span.time")?.text(), - ), - name = name.ifEmpty { "${manga.title} - ${i + 1}" }, - scanlator = null, - branch = null, - ) - } ?: bypassLicensedChapters(manga), - ) - } + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val root = doc.body().selectFirstOrThrow("section.main") + .selectFirstOrThrow("div.article_content") + val info = root.selectFirst("div.detail_info")?.selectFirst("ul") + val chaptersList = root.selectFirst("div.chapter_content") + ?.selectFirst("ul.chapter_list")?.select("li")?.asReversed() + val dateFormat = SimpleDateFormat("MMM dd,yyyy", Locale.US) + return manga.copy( + tags = manga.tags + info?.select("li")?.find { x -> + x.selectFirst("b")?.ownText() == "Genre(s):" + }?.select("a")?.mapNotNull { a -> + MangaTag( + title = a.attr("title").toTitleCase(), + key = a.attr("href").parseTagKey() ?: return@mapNotNull null, + source = MangaSource.MANGATOWN, + ) + }.orEmpty(), + description = info?.getElementById("show")?.ownText(), + chapters = chaptersList?.mapChapters { i, li -> + val href = li.selectFirst("a")?.attrAsRelativeUrlOrNull("href") + ?: return@mapChapters null + val name = li.select("span") + .filter { x -> x.className().isEmpty() } + .joinToString(" - ") { it.text() }.trim() + MangaChapter( + id = generateUid(href), + url = href, + source = MangaSource.MANGATOWN, + number = i + 1, + uploadDate = parseChapterDate( + dateFormat, + li.selectFirst("span.time")?.text(), + ), + name = name.ifEmpty { "${manga.title} - ${i + 1}" }, + scanlator = null, + branch = null, + ) + } ?: bypassLicensedChapters(manga), + ) + } - override suspend fun getPages(chapter: MangaChapter): List { - val fullUrl = chapter.url.toAbsoluteUrl(domain) - val doc = webClient.httpGet(fullUrl).parseHtml() - val root = doc.body().selectFirstOrThrow("div.page_select") - return root.selectFirstOrThrow("select").selectOrThrow("option").mapNotNull { - val href = it.attrAsRelativeUrlOrNull("value") - if (href == null || href.endsWith("featured.html")) { - return@mapNotNull null - } - MangaPage( - id = generateUid(href), - url = href, - preview = null, - referer = fullUrl, - source = MangaSource.MANGATOWN, - ) - } - } + override suspend fun getPages(chapter: MangaChapter): List { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + val root = doc.body().selectFirstOrThrow("div.page_select") + return root.selectFirstOrThrow("select").selectOrThrow("option").mapNotNull { + val href = it.attrAsRelativeUrlOrNull("value") + if (href == null || href.endsWith("featured.html")) { + return@mapNotNull null + } + MangaPage( + id = generateUid(href), + url = href, + preview = null, + source = MangaSource.MANGATOWN, + ) + } + } - override suspend fun getPageUrl(page: MangaPage): String { - val doc = webClient.httpGet(page.url.toAbsoluteUrl(domain)).parseHtml() - return doc.requireElementById("image").attrAsAbsoluteUrl("src") - } + override suspend fun getPageUrl(page: MangaPage): String { + val doc = webClient.httpGet(page.url.toAbsoluteUrl(domain)).parseHtml() + return doc.requireElementById("image").attrAsAbsoluteUrl("src") + } - override suspend fun getTags(): Set { - val doc = webClient.httpGet("/directory/".toAbsoluteUrl(domain)).parseHtml() - val root = doc.body().selectFirst("aside.right") - ?.getElementsContainingOwnText("Genres") - ?.first() - ?.nextElementSibling() ?: doc.parseFailed("Root not found") - return root.select("li").mapNotNullToSet { li -> - val a = li.selectFirst("a") ?: return@mapNotNullToSet null - val key = a.attr("href").parseTagKey() - if (key.isNullOrEmpty()) { - return@mapNotNullToSet null - } - MangaTag( - source = MangaSource.MANGATOWN, - key = key, - title = a.text().toTitleCase(), - ) - } - } + override suspend fun getTags(): Set { + val doc = webClient.httpGet("/directory/".toAbsoluteUrl(domain)).parseHtml() + val root = doc.body().selectFirst("aside.right") + ?.getElementsContainingOwnText("Genres") + ?.first() + ?.nextElementSibling() ?: doc.parseFailed("Root not found") + return root.select("li").mapNotNullToSet { li -> + val a = li.selectFirst("a") ?: return@mapNotNullToSet null + val key = a.attr("href").parseTagKey() + if (key.isNullOrEmpty()) { + return@mapNotNullToSet null + } + MangaTag( + source = MangaSource.MANGATOWN, + key = key, + title = a.text().toTitleCase(), + ) + } + } - private fun parseChapterDate(dateFormat: DateFormat, date: String?): Long { - return when { - date.isNullOrEmpty() -> 0L - date.contains("Today") -> Calendar.getInstance().timeInMillis - date.contains("Yesterday") -> Calendar.getInstance().apply { add(Calendar.DAY_OF_MONTH, -1) }.timeInMillis - else -> dateFormat.tryParse(date) - } - } + private fun parseChapterDate(dateFormat: DateFormat, date: String?): Long { + return when { + date.isNullOrEmpty() -> 0L + date.contains("Today") -> Calendar.getInstance().timeInMillis + date.contains("Yesterday") -> Calendar.getInstance().apply { add(Calendar.DAY_OF_MONTH, -1) }.timeInMillis + else -> dateFormat.tryParse(date) + } + } - private suspend fun bypassLicensedChapters(manga: Manga): List { - val doc = webClient.httpGet(manga.url.toAbsoluteUrl(getDomain("m"))).parseHtml() - val list = doc.body().selectFirst("ul.detail-ch-list") ?: return emptyList() - val dateFormat = SimpleDateFormat("MMM dd,yyyy", Locale.US) - return list.select("li").asReversed().mapIndexedNotNull { i, li -> - val a = li.selectFirst("a") ?: return@mapIndexedNotNull null - val href = a.attrAsRelativeUrl("href") - val name = a.selectFirst("span.vol")?.text().orEmpty().ifEmpty { - a.ownText() - } - MangaChapter( - id = generateUid(href), - url = href, - source = MangaSource.MANGATOWN, - number = i + 1, - uploadDate = parseChapterDate( - dateFormat, - li.selectFirst("span.time")?.text(), - ), - name = name.ifEmpty { "${manga.title} - ${i + 1}" }, - scanlator = null, - branch = null, - ) - } - } + private suspend fun bypassLicensedChapters(manga: Manga): List { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(getDomain("m"))).parseHtml() + val list = doc.body().selectFirst("ul.detail-ch-list") ?: return emptyList() + val dateFormat = SimpleDateFormat("MMM dd,yyyy", Locale.US) + return list.select("li").asReversed().mapIndexedNotNull { i, li -> + val a = li.selectFirst("a") ?: return@mapIndexedNotNull null + val href = a.attrAsRelativeUrl("href") + val name = a.selectFirst("span.vol")?.text().orEmpty().ifEmpty { + a.ownText() + } + MangaChapter( + id = generateUid(href), + url = href, + source = MangaSource.MANGATOWN, + number = i + 1, + uploadDate = parseChapterDate( + dateFormat, + li.selectFirst("span.time")?.text(), + ), + name = name.ifEmpty { "${manga.title} - ${i + 1}" }, + scanlator = null, + branch = null, + ) + } + } - private fun String.parseTagKey() = split('/').findLast { regexTag matches it } + private fun String.parseTagKey() = split('/').findLast { regexTag matches it } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/Manhwa18Parser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/Manhwa18Parser.kt index 55bc37cd..93d1e214 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/Manhwa18Parser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/Manhwa18Parser.kt @@ -165,7 +165,6 @@ class Manhwa18Parser(context: MangaLoaderContext) : MangaPage( id = generateUid(url), url = url, - referer = chapterUrl, preview = null, source = MangaSource.MANHWA18, ) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NHentaiParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NHentaiParser.kt index 5ca2f515..8c1c53be 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NHentaiParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NHentaiParser.kt @@ -17,172 +17,171 @@ import java.util.* @MangaSourceParser("NHENTAI", "N-Hentai") class NHentaiParser(context: MangaLoaderContext) : PagedMangaParser(context, MangaSource.NHENTAI, pageSize = 25) { - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("nhentai.net", null) + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("nhentai.net", null) - override val sortOrders: Set - get() = EnumSet.of(SortOrder.NEWEST, SortOrder.POPULARITY) + override val sortOrders: Set + get() = EnumSet.of(SortOrder.NEWEST, SortOrder.POPULARITY) - override suspend fun getListPage( - page: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - if (query.isNullOrEmpty() && tags != null && tags.size > 1) { - return getListPage(page, buildQuery(tags), emptySet(), sortOrder) - } - val domain = domain - val url = buildString { - append("https://") - append(domain) - if (!query.isNullOrEmpty()) { - append("/search/?q=") - append(query.urlEncoded()) - append("&page=") - append(page) - if (sortOrder == SortOrder.POPULARITY) { - append("&sort=popular") - } - } else { - append('/') - if (!tags.isNullOrEmpty()) { - val tag = tags.single() - append("tag/") - append(tag.key) - append('/') - if (sortOrder == SortOrder.POPULARITY) { - append("popular") - } - append("?page=") - append(page) - } else { - if (sortOrder == SortOrder.POPULARITY) { - append("?sort=popular&page=") - } else { - append("?page=") - } - append(page) - } - } - } - val root = webClient.httpGet(url).parseHtml().body().requireElementById("content") - .selectLastOrThrow("div.index-container") - val regexBrackets = Regex("\\[[^]]+]|\\([^)]+\\)") - val regexSpaces = Regex("\\s+") - return root.select(".gallery").map { div -> - val a = div.selectFirstOrThrow("a.cover") - val href = a.attrAsRelativeUrl("href") - val img = div.selectFirstOrThrow("img") - val title = div.selectFirstOrThrow(".caption").text() - Manga( - id = generateUid(href), - title = title.replace(regexBrackets, "") - .replace(regexSpaces, " ") - .trim(), - altTitle = null, - url = href, - publicUrl = href.toAbsoluteUrl(domain), - rating = RATING_UNKNOWN, - isNsfw = true, - coverUrl = img.attrAsAbsoluteUrlOrNull("data-src") - ?: img.attrAsAbsoluteUrl("src"), - tags = setOf(), - state = null, - author = null, - largeCoverUrl = null, - description = null, - chapters = listOf(), - source = source, - ) - } - } + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + if (query.isNullOrEmpty() && tags != null && tags.size > 1) { + return getListPage(page, buildQuery(tags), emptySet(), sortOrder) + } + val domain = domain + val url = buildString { + append("https://") + append(domain) + if (!query.isNullOrEmpty()) { + append("/search/?q=") + append(query.urlEncoded()) + append("&page=") + append(page) + if (sortOrder == SortOrder.POPULARITY) { + append("&sort=popular") + } + } else { + append('/') + if (!tags.isNullOrEmpty()) { + val tag = tags.single() + append("tag/") + append(tag.key) + append('/') + if (sortOrder == SortOrder.POPULARITY) { + append("popular") + } + append("?page=") + append(page) + } else { + if (sortOrder == SortOrder.POPULARITY) { + append("?sort=popular&page=") + } else { + append("?page=") + } + append(page) + } + } + } + val root = webClient.httpGet(url).parseHtml().body().requireElementById("content") + .selectLastOrThrow("div.index-container") + val regexBrackets = Regex("\\[[^]]+]|\\([^)]+\\)") + val regexSpaces = Regex("\\s+") + return root.select(".gallery").map { div -> + val a = div.selectFirstOrThrow("a.cover") + val href = a.attrAsRelativeUrl("href") + val img = div.selectFirstOrThrow("img") + val title = div.selectFirstOrThrow(".caption").text() + Manga( + id = generateUid(href), + title = title.replace(regexBrackets, "") + .replace(regexSpaces, " ") + .trim(), + altTitle = null, + url = href, + publicUrl = href.toAbsoluteUrl(domain), + rating = RATING_UNKNOWN, + isNsfw = true, + coverUrl = img.attrAsAbsoluteUrlOrNull("data-src") + ?: img.attrAsAbsoluteUrl("src"), + tags = setOf(), + state = null, + author = null, + largeCoverUrl = null, + description = null, + chapters = listOf(), + source = source, + ) + } + } - override suspend fun getDetails(manga: Manga): Manga { - val root = webClient.httpGet( - url = manga.url.toAbsoluteUrl(domain), - ).parseHtml().body().requireElementById("bigcontainer") - val img = root.requireElementById("cover").selectFirstOrThrow("img") - val tagContainers = root.requireElementById("tags").select(".tag-container") - val dateFormat = SimpleDateFormat( - "yyyy-MM-dd'T'HH:mm:ss.SSSSSS'+00:00'", - Locale.ROOT, - ) - return manga.copy( - tags = tagContainers.find { x -> x.ownText() == "Tags:" }?.parseTags() ?: manga.tags, - author = tagContainers.find { x -> x.ownText() == "Artists:" } - ?.selectFirst("span.name")?.text()?.toCamelCase(), - largeCoverUrl = img.attrAsAbsoluteUrlOrNull("data-src") - ?: img.attrAsAbsoluteUrl("src"), - description = null, - chapters = listOf( - MangaChapter( - id = manga.id, - name = manga.title, - number = 1, - url = manga.url, - scanlator = null, - uploadDate = dateFormat.tryParse( - tagContainers.find { x -> x.ownText() == "Uploaded:" } - ?.selectFirst("time") - ?.attr("datetime"), - ), - branch = null, - source = source, - ), - ), - ) - } + override suspend fun getDetails(manga: Manga): Manga { + val root = webClient.httpGet( + url = manga.url.toAbsoluteUrl(domain), + ).parseHtml().body().requireElementById("bigcontainer") + val img = root.requireElementById("cover").selectFirstOrThrow("img") + val tagContainers = root.requireElementById("tags").select(".tag-container") + val dateFormat = SimpleDateFormat( + "yyyy-MM-dd'T'HH:mm:ss.SSSSSS'+00:00'", + Locale.ROOT, + ) + return manga.copy( + tags = tagContainers.find { x -> x.ownText() == "Tags:" }?.parseTags() ?: manga.tags, + author = tagContainers.find { x -> x.ownText() == "Artists:" } + ?.selectFirst("span.name")?.text()?.toCamelCase(), + largeCoverUrl = img.attrAsAbsoluteUrlOrNull("data-src") + ?: img.attrAsAbsoluteUrl("src"), + description = null, + chapters = listOf( + MangaChapter( + id = manga.id, + name = manga.title, + number = 1, + url = manga.url, + scanlator = null, + uploadDate = dateFormat.tryParse( + tagContainers.find { x -> x.ownText() == "Uploaded:" } + ?.selectFirst("time") + ?.attr("datetime"), + ), + branch = null, + source = source, + ), + ), + ) + } - override suspend fun getPages(chapter: MangaChapter): List { - val url = chapter.url.toAbsoluteUrl(domain) - val root = webClient.httpGet(url).parseHtml().requireElementById("thumbnail-container") - return root.select(".thumb-container").map { div -> - val a = div.selectFirstOrThrow("a") - val img = div.selectFirstOrThrow("img") - val href = a.attrAsRelativeUrl("href") - MangaPage( - id = generateUid(href), - url = href, - referer = url, - preview = img.attrAsAbsoluteUrlOrNull("data-src") - ?: img.attrAsAbsoluteUrl("src"), - source = source, - ) - } - } + override suspend fun getPages(chapter: MangaChapter): List { + val url = chapter.url.toAbsoluteUrl(domain) + val root = webClient.httpGet(url).parseHtml().requireElementById("thumbnail-container") + return root.select(".thumb-container").map { div -> + val a = div.selectFirstOrThrow("a") + val img = div.selectFirstOrThrow("img") + val href = a.attrAsRelativeUrl("href") + MangaPage( + id = generateUid(href), + url = href, + preview = img.attrAsAbsoluteUrlOrNull("data-src") + ?: img.attrAsAbsoluteUrl("src"), + source = source, + ) + } + } - override suspend fun getPageUrl(page: MangaPage): String { - val root = webClient.httpGet(page.url.toAbsoluteUrl(domain)).parseHtml().body() - .requireElementById("image-container") - return root.selectFirstOrThrow("img").attrAsAbsoluteUrl("src") - } + override suspend fun getPageUrl(page: MangaPage): String { + val root = webClient.httpGet(page.url.toAbsoluteUrl(domain)).parseHtml().body() + .requireElementById("image-container") + return root.selectFirstOrThrow("img").attrAsAbsoluteUrl("src") + } - override suspend fun getTags(): Set { - return coroutineScope { - // parse first 3 pages of tags - (1..3).map { page -> - async { getTags(page) } - } - }.awaitAll().flattenTo(ArraySet(360)) - } + override suspend fun getTags(): Set { + return coroutineScope { + // parse first 3 pages of tags + (1..3).map { page -> + async { getTags(page) } + } + }.awaitAll().flattenTo(ArraySet(360)) + } - private suspend fun getTags(page: Int): Set { - val root = webClient.httpGet("https://${domain}/tags/popular?page=$page").parseHtml().body() - .getElementById("tag-container") - return root?.parseTags().orEmpty() - } + private suspend fun getTags(page: Int): Set { + val root = webClient.httpGet("https://${domain}/tags/popular?page=$page").parseHtml().body() + .getElementById("tag-container") + return root?.parseTags().orEmpty() + } - private fun Element.parseTags() = select("a.tag").mapToSet { a -> - val href = a.attr("href").removeSuffix('/') - MangaTag( - title = a.selectFirstOrThrow(".name").text().toTitleCase(), - key = href.substringAfterLast('/'), - source = source, - ) - } + private fun Element.parseTags() = select("a.tag").mapToSet { a -> + val href = a.attr("href").removeSuffix('/') + MangaTag( + title = a.selectFirstOrThrow(".name").text().toTitleCase(), + key = href.substringAfterLast('/'), + source = source, + ) + } - private fun buildQuery(tags: Collection) = tags.joinToString(separator = " ") { tag -> - "tag:\"${tag.key}\"" - } + private fun buildQuery(tags: Collection) = tags.joinToString(separator = " ") { tag -> + "tag:\"${tag.key}\"" + } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NetTruyenParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NetTruyenParser.kt index aaf4c956..6b89233e 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NetTruyenParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NetTruyenParser.kt @@ -16,209 +16,208 @@ import java.util.* @MangaSourceParser("NETTRUYEN", "NetTruyen", "vi") class NetTruyenParser(context: MangaLoaderContext) : - PagedMangaParser(context, MangaSource.NETTRUYEN, pageSize = 36) { - - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("nettruyenin.com", null) - - override val sortOrders: Set - get() = EnumSet.of(SortOrder.UPDATED, SortOrder.POPULARITY, SortOrder.NEWEST, SortOrder.RATING) - - private val mutex = Mutex() - private val dateFormat = SimpleDateFormat("dd/MM/yy", Locale.US) - private var tagCache: ArrayMap? = null - - override suspend fun getDetails(manga: Manga): Manga { - val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - val rating = doc.selectFirst("span[itemprop=ratingValue]") - ?.ownText() - ?.toFloatOrNull() ?: 0f - - val chapterElements = doc.getElementById("nt_listchapter")?.select("ul > li") ?: doc.parseFailed() - val chapters = chapterElements.asReversed().mapChapters { index, element -> - val a = element.selectFirst("div.chapter > a") ?: return@mapChapters null - val relativeUrl = a.attrAsRelativeUrlOrNull("href") ?: return@mapChapters null - val timeText = element.selectFirst("div.col-xs-4.text-center.no-wrap.small")?.text() - - MangaChapter( - id = generateUid(relativeUrl), - name = a.text(), - number = index + 1, - url = relativeUrl, - scanlator = null, - uploadDate = parseChapterTime(timeText), - branch = null, - source = source, - ) - } - - return manga.copy( - rating = rating / 5, - chapters = chapters, - description = doc.selectFirst("div.detail-content > p")?.html(), - isNsfw = doc.selectFirst("div.alert.alert-danger > strong:contains(Cảnh báo độ tuổi)") != null, - ) - } - - // 20 giây trước - // 52 phút trước - // 6 giờ trước - // 2 ngày trước - // 19:09 30/07 - // 23/12/21 - private fun parseChapterTime(timeText: String?): Long { - if (timeText.isNullOrEmpty()) { - return 0L - } - - val timeWords = arrayOf("giây", "phút", "giờ", "ngày") - val calendar = Calendar.getInstance() - val timeArr = timeText.split(' ') - if (WordSet(*timeWords).anyWordIn(timeText)) { - val timeSuffix = timeArr.getOrNull(1) - val timeDiff = timeArr.getOrNull(0)?.toIntOrNull() ?: return 0L - when (timeSuffix) { - timeWords[0] -> calendar.add(Calendar.SECOND, -timeDiff) - timeWords[1] -> calendar.add(Calendar.MINUTE, -timeDiff) - timeWords[2] -> calendar.add(Calendar.HOUR, -timeDiff) - timeWords[3] -> calendar.add(Calendar.DATE, -timeDiff) - else -> return 0L - } - } else { - val relativeDate = timeArr.lastOrNull() ?: return 0L - val dateString = when (relativeDate.split('/').size) { - 2 -> { - val currentYear = calendar.get(Calendar.YEAR).toString().takeLast(2) - "$relativeDate/$currentYear" - } - - 3 -> relativeDate - else -> return 0L - } - - calendar.timeInMillis = dateFormat.tryParse(dateString) - } - - - return calendar.time.time - } - - override suspend fun getListPage( - page: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - val isSearching = !query.isNullOrEmpty() - val url = buildString { - append("https://") - append(domain) - if (isSearching) { - append("/tim-truyen?keyword=") - append(query!!.urlEncoded()) - append("&page=") - append(page) - } else { - val tagQuery = tags.orEmpty().joinToString(",") { it.key } - append("/tim-truyen-nang-cao?genres=$tagQuery") - append("¬genres=&gender=-1&status=-1&minchapter=1&sort=${getSortOrderKey(sortOrder)}") - append("&page=$page") - } - } - - val response = if (isSearching) { - val result = runCatchingCancellable { webClient.httpGet(url) } - val exception = result.exceptionOrNull() - if (exception is NotFoundException) { - return emptyList() - } - - result.getOrThrow() - } else { - webClient.httpGet(url) - } - - val itemsElements = response.parseHtml() - .select("div.ModuleContent > div.items") - .select("div.item") - return itemsElements.mapNotNull { item -> - val tooltipElement = item.selectFirst("div.box_tootip") ?: return@mapNotNull null - val absUrl = item.selectFirst("div.image > a")?.attrAsAbsoluteUrlOrNull("href") ?: return@mapNotNull null - val slug = absUrl.substringAfterLast('/') - val mangaState = when (tooltipElement.selectFirst("div.message_main > p:contains(Tình trạng)")?.ownText()) { - "Đang tiến hành" -> MangaState.ONGOING - "Hoàn thành" -> MangaState.FINISHED - else -> null - } - - val tagMap = getOrCreateTagMap() - val tagsElement = tooltipElement.selectFirst("div.message_main > p:contains(Thể loại)")?.ownText().orEmpty() - val mangaTags = tagsElement.split(',').mapNotNullToSet { tagMap[it.trim()] } - Manga( - id = generateUid(slug), - title = tooltipElement.selectFirst("div.title")?.text().orEmpty(), - altTitle = null, - url = absUrl.toRelativeUrl(domain), - publicUrl = absUrl, - rating = RATING_UNKNOWN, - isNsfw = false, - coverUrl = item.selectFirst("div.image a img")?.absUrl("data-original").orEmpty(), - largeCoverUrl = null, - tags = mangaTags, - state = mangaState, - author = tooltipElement.selectFirst("div.message_main > p:contains(Tác giả)")?.ownText(), - description = tooltipElement.selectFirst("div.box_text")?.text(), - chapters = null, - source = source, - ) - } - } - - - override suspend fun getPages(chapter: MangaChapter): List { - val pageElements = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() - .select("div.reading-detail.box_doc > div img") - return pageElements.map { element -> - val url = element.attrAsAbsoluteUrl("data-original") - MangaPage( - id = generateUid(url), - url = url, - referer = domain, - preview = null, - source = source, - ) - } - } - - override suspend fun getTags(): Set { - val map = getOrCreateTagMap() - val tagSet = ArraySet(map.size) - for (entry in map) { - tagSet.add(entry.value) - } - - return tagSet - } - - private suspend fun getOrCreateTagMap(): ArrayMap = mutex.withLock { - tagCache?.let { return@withLock it } - val doc = webClient.httpGet("/tim-truyen-nang-cao".toAbsoluteUrl(domain)).parseHtml() - val tagItems = doc.select("div.genre-item") - val result = ArrayMap(tagItems.size) - for (item in tagItems) { - val title = item.text().trim() - val key = item.select("span[data-id]").attr("data-id") - result[title] = MangaTag(title = title, key = key, source = source) - } - tagCache = result - result - } - - private fun getSortOrderKey(sortOrder: SortOrder) = when (sortOrder) { - SortOrder.UPDATED -> 0 - SortOrder.POPULARITY -> 10 - SortOrder.NEWEST -> 15 - SortOrder.RATING -> 20 - else -> throw IllegalArgumentException("Sort order ${sortOrder.name} not supported") - } + PagedMangaParser(context, MangaSource.NETTRUYEN, pageSize = 36) { + + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("nettruyenin.com", null) + + override val sortOrders: Set + get() = EnumSet.of(SortOrder.UPDATED, SortOrder.POPULARITY, SortOrder.NEWEST, SortOrder.RATING) + + private val mutex = Mutex() + private val dateFormat = SimpleDateFormat("dd/MM/yy", Locale.US) + private var tagCache: ArrayMap? = null + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val rating = doc.selectFirst("span[itemprop=ratingValue]") + ?.ownText() + ?.toFloatOrNull() ?: 0f + + val chapterElements = doc.getElementById("nt_listchapter")?.select("ul > li") ?: doc.parseFailed() + val chapters = chapterElements.asReversed().mapChapters { index, element -> + val a = element.selectFirst("div.chapter > a") ?: return@mapChapters null + val relativeUrl = a.attrAsRelativeUrlOrNull("href") ?: return@mapChapters null + val timeText = element.selectFirst("div.col-xs-4.text-center.no-wrap.small")?.text() + + MangaChapter( + id = generateUid(relativeUrl), + name = a.text(), + number = index + 1, + url = relativeUrl, + scanlator = null, + uploadDate = parseChapterTime(timeText), + branch = null, + source = source, + ) + } + + return manga.copy( + rating = rating / 5, + chapters = chapters, + description = doc.selectFirst("div.detail-content > p")?.html(), + isNsfw = doc.selectFirst("div.alert.alert-danger > strong:contains(Cảnh báo độ tuổi)") != null, + ) + } + + // 20 giây trước + // 52 phút trước + // 6 giờ trước + // 2 ngày trước + // 19:09 30/07 + // 23/12/21 + private fun parseChapterTime(timeText: String?): Long { + if (timeText.isNullOrEmpty()) { + return 0L + } + + val timeWords = arrayOf("giây", "phút", "giờ", "ngày") + val calendar = Calendar.getInstance() + val timeArr = timeText.split(' ') + if (WordSet(*timeWords).anyWordIn(timeText)) { + val timeSuffix = timeArr.getOrNull(1) + val timeDiff = timeArr.getOrNull(0)?.toIntOrNull() ?: return 0L + when (timeSuffix) { + timeWords[0] -> calendar.add(Calendar.SECOND, -timeDiff) + timeWords[1] -> calendar.add(Calendar.MINUTE, -timeDiff) + timeWords[2] -> calendar.add(Calendar.HOUR, -timeDiff) + timeWords[3] -> calendar.add(Calendar.DATE, -timeDiff) + else -> return 0L + } + } else { + val relativeDate = timeArr.lastOrNull() ?: return 0L + val dateString = when (relativeDate.split('/').size) { + 2 -> { + val currentYear = calendar.get(Calendar.YEAR).toString().takeLast(2) + "$relativeDate/$currentYear" + } + + 3 -> relativeDate + else -> return 0L + } + + calendar.timeInMillis = dateFormat.tryParse(dateString) + } + + + return calendar.time.time + } + + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + val isSearching = !query.isNullOrEmpty() + val url = buildString { + append("https://") + append(domain) + if (isSearching) { + append("/tim-truyen?keyword=") + append(query!!.urlEncoded()) + append("&page=") + append(page) + } else { + val tagQuery = tags.orEmpty().joinToString(",") { it.key } + append("/tim-truyen-nang-cao?genres=$tagQuery") + append("¬genres=&gender=-1&status=-1&minchapter=1&sort=${getSortOrderKey(sortOrder)}") + append("&page=$page") + } + } + + val response = if (isSearching) { + val result = runCatchingCancellable { webClient.httpGet(url) } + val exception = result.exceptionOrNull() + if (exception is NotFoundException) { + return emptyList() + } + + result.getOrThrow() + } else { + webClient.httpGet(url) + } + + val itemsElements = response.parseHtml() + .select("div.ModuleContent > div.items") + .select("div.item") + return itemsElements.mapNotNull { item -> + val tooltipElement = item.selectFirst("div.box_tootip") ?: return@mapNotNull null + val absUrl = item.selectFirst("div.image > a")?.attrAsAbsoluteUrlOrNull("href") ?: return@mapNotNull null + val slug = absUrl.substringAfterLast('/') + val mangaState = when (tooltipElement.selectFirst("div.message_main > p:contains(Tình trạng)")?.ownText()) { + "Đang tiến hành" -> MangaState.ONGOING + "Hoàn thành" -> MangaState.FINISHED + else -> null + } + + val tagMap = getOrCreateTagMap() + val tagsElement = tooltipElement.selectFirst("div.message_main > p:contains(Thể loại)")?.ownText().orEmpty() + val mangaTags = tagsElement.split(',').mapNotNullToSet { tagMap[it.trim()] } + Manga( + id = generateUid(slug), + title = tooltipElement.selectFirst("div.title")?.text().orEmpty(), + altTitle = null, + url = absUrl.toRelativeUrl(domain), + publicUrl = absUrl, + rating = RATING_UNKNOWN, + isNsfw = false, + coverUrl = item.selectFirst("div.image a img")?.absUrl("data-original").orEmpty(), + largeCoverUrl = null, + tags = mangaTags, + state = mangaState, + author = tooltipElement.selectFirst("div.message_main > p:contains(Tác giả)")?.ownText(), + description = tooltipElement.selectFirst("div.box_text")?.text(), + chapters = null, + source = source, + ) + } + } + + + override suspend fun getPages(chapter: MangaChapter): List { + val pageElements = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + .select("div.reading-detail.box_doc > div img") + return pageElements.map { element -> + val url = element.attrAsAbsoluteUrl("data-original") + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } + + override suspend fun getTags(): Set { + val map = getOrCreateTagMap() + val tagSet = ArraySet(map.size) + for (entry in map) { + tagSet.add(entry.value) + } + + return tagSet + } + + private suspend fun getOrCreateTagMap(): ArrayMap = mutex.withLock { + tagCache?.let { return@withLock it } + val doc = webClient.httpGet("/tim-truyen-nang-cao".toAbsoluteUrl(domain)).parseHtml() + val tagItems = doc.select("div.genre-item") + val result = ArrayMap(tagItems.size) + for (item in tagItems) { + val title = item.text().trim() + val key = item.select("span[data-id]").attr("data-id") + result[title] = MangaTag(title = title, key = key, source = source) + } + tagCache = result + result + } + + private fun getSortOrderKey(sortOrder: SortOrder) = when (sortOrder) { + SortOrder.UPDATED -> 0 + SortOrder.POPULARITY -> 10 + SortOrder.NEWEST -> 15 + SortOrder.RATING -> 20 + else -> throw IllegalArgumentException("Sort order ${sortOrder.name} not supported") + } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NicovideoSeigaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NicovideoSeigaParser.kt index 2434199b..7be9b20a 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NicovideoSeigaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NicovideoSeigaParser.kt @@ -12,174 +12,173 @@ private const val STATUS_FINISHED = "完結" @MangaSourceParser("NICOVIDEO_SEIGA", "Nicovideo Seiga", "ja") class NicovideoSeigaParser(context: MangaLoaderContext) : - MangaParser(context, MangaSource.NICOVIDEO_SEIGA), - MangaParserAuthProvider { + MangaParser(context, MangaSource.NICOVIDEO_SEIGA), + MangaParserAuthProvider { - override val authUrl: String - get() = "https://${getDomain("account")}/login?site=seiga" + override val authUrl: String + get() = "https://${getDomain("account")}/login?site=seiga" - override val isAuthorized: Boolean - get() = context.cookieJar.getCookies(getDomain("seiga")).any { - it.name == "user_session" - } + override val isAuthorized: Boolean + get() = context.cookieJar.getCookies(getDomain("seiga")).any { + it.name == "user_session" + } - override suspend fun getUsername(): String { - val body = webClient.httpGet("https://${getDomain("app")}/my/apps").parseHtml().body() - return body.selectFirst("#userinfo > div > div > strong")?.text() ?: throw AuthRequiredException(source) - } + override suspend fun getUsername(): String { + val body = webClient.httpGet("https://${getDomain("app")}/my/apps").parseHtml().body() + return body.selectFirst("#userinfo > div > div > strong")?.text() ?: throw AuthRequiredException(source) + } - override val sortOrders: Set = EnumSet.of( - SortOrder.UPDATED, - SortOrder.POPULARITY, - ) + override val sortOrders: Set = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + ) - override val configKeyDomain: ConfigKey.Domain = ConfigKey.Domain("nicovideo.jp", null) + override val configKeyDomain: ConfigKey.Domain = ConfigKey.Domain("nicovideo.jp", null) - @InternalParsersApi - override suspend fun getList( - offset: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - val page = (offset / 20f).toIntUp().inc() - val domain = getDomain("seiga") - val url = when { - !query.isNullOrEmpty() -> return if (offset == 0) getSearchList(query, page) else emptyList() - tags.isNullOrEmpty() -> "https://$domain/manga/list?page=$page&sort=${getSortKey(sortOrder)}" - tags.size == 1 -> "https://$domain/manga/list?category=${tags.first().key}&page=$page" + - "&sort=${getSortKey(sortOrder)}" + @InternalParsersApi + override suspend fun getList( + offset: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + val page = (offset / 20f).toIntUp().inc() + val domain = getDomain("seiga") + val url = when { + !query.isNullOrEmpty() -> return if (offset == 0) getSearchList(query, page) else emptyList() + tags.isNullOrEmpty() -> "https://$domain/manga/list?page=$page&sort=${getSortKey(sortOrder)}" + tags.size == 1 -> "https://$domain/manga/list?category=${tags.first().key}&page=$page" + + "&sort=${getSortKey(sortOrder)}" - tags.size > 1 -> throw IllegalArgumentException("This source supports only 1 category") - else -> "https://$domain/manga/list?page=$page&sort=${getSortKey(sortOrder)}" - } - val doc = webClient.httpGet(url).parseHtml() - val comicList = doc.body().select("#comic_list > ul > li") ?: doc.parseFailed("Container not found") - val items = comicList.select("div > .description > div > div") - return items.mapNotNull { item -> - val href = - item.selectFirst(".comic_icon > div > a")?.attrAsRelativeUrlOrNull("href") ?: return@mapNotNull null - val statusText = item.selectFirst(".mg_description_header > .mg_icon > .content_status > span")?.text() - Manga( - id = generateUid(href), - title = item.selectFirst(".mg_body > .title > a")?.text() ?: return@mapNotNull null, - coverUrl = item.selectFirst(".comic_icon > div > a > img")?.attrAsAbsoluteUrl("src").orEmpty(), - altTitle = null, - author = item.selectFirst(".mg_description_header > .mg_author > a")?.text(), - rating = RATING_UNKNOWN, - url = href, - isNsfw = false, - tags = item.getElementsByAttributeValueContaining("href", "?category=").mapToSet { a -> - MangaTag( - key = a.attr("href").substringAfterLast('='), - title = a.ownText().trim(), - source = source, - ) - }, - state = when (statusText) { - STATUS_ONGOING -> MangaState.ONGOING - STATUS_FINISHED -> MangaState.FINISHED - else -> null - }, - publicUrl = href.toAbsoluteUrl(item.host ?: getDomain("seiga")), - source = source, - ) - } - } + tags.size > 1 -> throw IllegalArgumentException("This source supports only 1 category") + else -> "https://$domain/manga/list?page=$page&sort=${getSortKey(sortOrder)}" + } + val doc = webClient.httpGet(url).parseHtml() + val comicList = doc.body().select("#comic_list > ul > li") ?: doc.parseFailed("Container not found") + val items = comicList.select("div > .description > div > div") + return items.mapNotNull { item -> + val href = + item.selectFirst(".comic_icon > div > a")?.attrAsRelativeUrlOrNull("href") ?: return@mapNotNull null + val statusText = item.selectFirst(".mg_description_header > .mg_icon > .content_status > span")?.text() + Manga( + id = generateUid(href), + title = item.selectFirst(".mg_body > .title > a")?.text() ?: return@mapNotNull null, + coverUrl = item.selectFirst(".comic_icon > div > a > img")?.attrAsAbsoluteUrl("src").orEmpty(), + altTitle = null, + author = item.selectFirst(".mg_description_header > .mg_author > a")?.text(), + rating = RATING_UNKNOWN, + url = href, + isNsfw = false, + tags = item.getElementsByAttributeValueContaining("href", "?category=").mapToSet { a -> + MangaTag( + key = a.attr("href").substringAfterLast('='), + title = a.ownText().trim(), + source = source, + ) + }, + state = when (statusText) { + STATUS_ONGOING -> MangaState.ONGOING + STATUS_FINISHED -> MangaState.FINISHED + else -> null + }, + publicUrl = href.toAbsoluteUrl(item.host ?: getDomain("seiga")), + source = source, + ) + } + } - override suspend fun getDetails(manga: Manga): Manga { - val doc = webClient.httpGet(manga.url.toAbsoluteUrl(getDomain("seiga"))).parseHtml() - val contents = doc.body().selectFirstOrThrow("#contents") - val statusText = contents - .select("div.mg_work_detail > div > div:nth-child(2) > div.tip.content_status.status_series > span") - .text() - return manga.copy( - description = contents.selectFirst("div.mg_work_detail > div > div.row > div.description_text")?.html(), - largeCoverUrl = contents.selectFirst("div.primaries > div.main_visual > a > img") - ?.attrAsAbsoluteUrlOrNull("src"), - state = when (statusText) { - STATUS_ONGOING -> MangaState.ONGOING - STATUS_FINISHED -> MangaState.FINISHED - else -> null - }, - isNsfw = contents.select(".icon_adult").isNotEmpty(), - chapters = contents.select("#episode_list > ul > li").mapChapters { i, li -> - val href = li.selectFirst("div > div.description > div.title > a") - ?.attrAsRelativeUrl("href") ?: li.parseFailed() - MangaChapter( - id = generateUid(href), - name = li.select("div > div.description > div.title > a").text(), - number = i + 1, - url = href, - scanlator = null, - branch = null, - uploadDate = 0, - source = source, - ) - }, - ) - } + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(getDomain("seiga"))).parseHtml() + val contents = doc.body().selectFirstOrThrow("#contents") + val statusText = contents + .select("div.mg_work_detail > div > div:nth-child(2) > div.tip.content_status.status_series > span") + .text() + return manga.copy( + description = contents.selectFirst("div.mg_work_detail > div > div.row > div.description_text")?.html(), + largeCoverUrl = contents.selectFirst("div.primaries > div.main_visual > a > img") + ?.attrAsAbsoluteUrlOrNull("src"), + state = when (statusText) { + STATUS_ONGOING -> MangaState.ONGOING + STATUS_FINISHED -> MangaState.FINISHED + else -> null + }, + isNsfw = contents.select(".icon_adult").isNotEmpty(), + chapters = contents.select("#episode_list > ul > li").mapChapters { i, li -> + val href = li.selectFirst("div > div.description > div.title > a") + ?.attrAsRelativeUrl("href") ?: li.parseFailed() + MangaChapter( + id = generateUid(href), + name = li.select("div > div.description > div.title > a").text(), + number = i + 1, + url = href, + scanlator = null, + branch = null, + uploadDate = 0, + source = source, + ) + }, + ) + } - override suspend fun getPages(chapter: MangaChapter): List { - val fullUrl = chapter.url.toAbsoluteUrl(getDomain("seiga")) - val doc = webClient.httpGet(fullUrl).parseHtml() - if (!doc.select("#login_manga").isEmpty()) - throw AuthRequiredException(source) - val root = doc.body().select("#page_contents > li") - return root.map { li -> - val url = li.select("div > img").attr("data-original") - MangaPage( - id = generateUid(url), - url = url, - preview = null, - referer = fullUrl, - source = source, - ) - } - } + override suspend fun getPages(chapter: MangaChapter): List { + val fullUrl = chapter.url.toAbsoluteUrl(getDomain("seiga")) + val doc = webClient.httpGet(fullUrl).parseHtml() + if (!doc.select("#login_manga").isEmpty()) + throw AuthRequiredException(source) + val root = doc.body().select("#page_contents > li") + return root.map { li -> + val url = li.select("div > img").attr("data-original") + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } - override suspend fun getTags(): Set { - val doc = webClient.httpGet("https://${getDomain("seiga")}/manga/list").parseHtml() - val root = doc.body().selectOrThrow("#mg_category_list > ul > li") - return root.mapToSet { li -> - val a = li.selectFirstOrThrow("a") - MangaTag( - title = a.text(), - key = a.attrAsRelativeUrlOrNull("href").orEmpty(), - source = source, - ) - } - } + override suspend fun getTags(): Set { + val doc = webClient.httpGet("https://${getDomain("seiga")}/manga/list").parseHtml() + val root = doc.body().selectOrThrow("#mg_category_list > ul > li") + return root.mapToSet { li -> + val a = li.selectFirstOrThrow("a") + MangaTag( + title = a.text(), + key = a.attrAsRelativeUrlOrNull("href").orEmpty(), + source = source, + ) + } + } - private suspend fun getSearchList(query: String, page: Int): List { - val domain = getDomain("seiga") - val doc = webClient.httpGet("https://$domain/manga/search/?q=$query&page=$page&sort=score").parseHtml() - val root = doc.body().select(".search_result__item") - return root.mapNotNull { item -> - val href = item.selectFirst(".search_result__item__thumbnail > a") - ?.attrAsRelativeUrl("href") ?: doc.parseFailed() - Manga( - id = generateUid(href), - url = href, - publicUrl = href.toAbsoluteUrl(item.host ?: domain), - title = item.selectFirst(".search_result__item__info > .search_result__item__info--title > a") - ?.text()?.trim() ?: return@mapNotNull null, - altTitle = null, - author = null, - tags = emptySet(), - rating = RATING_UNKNOWN, - state = null, - isNsfw = false, - source = source, - coverUrl = item.selectFirst(".search_result__item__thumbnail > a > img") - ?.attrAsAbsoluteUrl("data-original").orEmpty(), - ) - } - } + private suspend fun getSearchList(query: String, page: Int): List { + val domain = getDomain("seiga") + val doc = webClient.httpGet("https://$domain/manga/search/?q=$query&page=$page&sort=score").parseHtml() + val root = doc.body().select(".search_result__item") + return root.mapNotNull { item -> + val href = item.selectFirst(".search_result__item__thumbnail > a") + ?.attrAsRelativeUrl("href") ?: doc.parseFailed() + Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(item.host ?: domain), + title = item.selectFirst(".search_result__item__info > .search_result__item__info--title > a") + ?.text()?.trim() ?: return@mapNotNull null, + altTitle = null, + author = null, + tags = emptySet(), + rating = RATING_UNKNOWN, + state = null, + isNsfw = false, + source = source, + coverUrl = item.selectFirst(".search_result__item__thumbnail > a > img") + ?.attrAsAbsoluteUrl("data-original").orEmpty(), + ) + } + } - private fun getSortKey(sortOrder: SortOrder) = when (sortOrder) { - SortOrder.POPULARITY -> "manga_view" - SortOrder.UPDATED -> "manga_updated" - else -> "manga_view" - } + private fun getSortKey(sortOrder: SortOrder) = when (sortOrder) { + SortOrder.POPULARITY -> "manga_view" + SortOrder.UPDATED -> "manga_updated" + else -> "manga_view" + } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NineMangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NineMangaParser.kt index f3d6d2ce..f3b2f6b8 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NineMangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NineMangaParser.kt @@ -12,248 +12,247 @@ import java.text.SimpleDateFormat import java.util.* internal abstract class NineMangaParser( - context: MangaLoaderContext, - source: MangaSource, - defaultDomain: String, + context: MangaLoaderContext, + source: MangaSource, + defaultDomain: String, ) : PagedMangaParser(context, source, pageSize = 26) { - override val configKeyDomain = ConfigKey.Domain(defaultDomain, null) + override val configKeyDomain = ConfigKey.Domain(defaultDomain, null) - init { - context.cookieJar.insertCookies(domain, "ninemanga_template_desk=yes") - } + init { + context.cookieJar.insertCookies(domain, "ninemanga_template_desk=yes") + } - override val headers = Headers.Builder() - .add("Accept-Language", "en-US;q=0.7,en;q=0.3") - .build() + override val headers = Headers.Builder() + .add("Accept-Language", "en-US;q=0.7,en;q=0.3") + .build() - override val sortOrders: Set = Collections.singleton( - SortOrder.POPULARITY, - ) + override val sortOrders: Set = Collections.singleton( + SortOrder.POPULARITY, + ) - override suspend fun getListPage( - page: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - val url = buildString { - append("https://") - append(domain) - when { - !query.isNullOrEmpty() -> { - append("/search/?name_sel=&wd=") - append(query.urlEncoded()) - append("&page=") - } + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + val url = buildString { + append("https://") + append(domain) + when { + !query.isNullOrEmpty() -> { + append("/search/?name_sel=&wd=") + append(query.urlEncoded()) + append("&page=") + } - !tags.isNullOrEmpty() -> { - append("/search/?category_id=") - for (tag in tags) { - append(tag.key) - append(',') - } - append("&page=") - } + !tags.isNullOrEmpty() -> { + append("/search/?category_id=") + for (tag in tags) { + append(tag.key) + append(',') + } + append("&page=") + } - else -> { - append("/category/index_") - } - } - append(page) - append(".html") - } - val doc = webClient.httpGet(url).parseHtml() - val root = doc.body().selectFirst("ul.direlist") - ?: doc.parseFailed("Cannot find root") - val baseHost = root.baseUri().toHttpUrl().host - return root.select("li").map { node -> - val href = node.selectFirst("a")?.absUrl("href") - ?: node.parseFailed("Link not found") - val relUrl = href.toRelativeUrl(baseHost) - val dd = node.selectFirst("dd") - Manga( - id = generateUid(relUrl), - url = relUrl, - publicUrl = href, - title = dd?.selectFirst("a.bookname")?.text()?.toCamelCase().orEmpty(), - altTitle = null, - coverUrl = node.selectFirst("img")?.absUrl("src").orEmpty(), - rating = RATING_UNKNOWN, - author = null, - isNsfw = false, - tags = emptySet(), - state = null, - source = source, - description = dd?.selectFirst("p")?.html(), - ) - } - } + else -> { + append("/category/index_") + } + } + append(page) + append(".html") + } + val doc = webClient.httpGet(url).parseHtml() + val root = doc.body().selectFirst("ul.direlist") + ?: doc.parseFailed("Cannot find root") + val baseHost = root.baseUri().toHttpUrl().host + return root.select("li").map { node -> + val href = node.selectFirst("a")?.absUrl("href") + ?: node.parseFailed("Link not found") + val relUrl = href.toRelativeUrl(baseHost) + val dd = node.selectFirst("dd") + Manga( + id = generateUid(relUrl), + url = relUrl, + publicUrl = href, + title = dd?.selectFirst("a.bookname")?.text()?.toCamelCase().orEmpty(), + altTitle = null, + coverUrl = node.selectFirst("img")?.absUrl("src").orEmpty(), + rating = RATING_UNKNOWN, + author = null, + isNsfw = false, + tags = emptySet(), + state = null, + source = source, + description = dd?.selectFirst("p")?.html(), + ) + } + } - override suspend fun getDetails(manga: Manga): Manga { - val doc = webClient.httpGet( - manga.url.toAbsoluteUrl(domain) + "?waring=1", - ).parseHtml() - val root = doc.body().selectFirstOrThrow("div.manga") - val infoRoot = root.selectFirstOrThrow("div.bookintro") - return manga.copy( - tags = infoRoot.getElementsByAttributeValue("itemprop", "genre").first() - ?.select("a")?.mapToSet { a -> - MangaTag( - title = a.text().toTitleCase(), - key = a.attr("href").substringBetween("/", "."), - source = source, - ) - }.orEmpty(), - author = infoRoot.getElementsByAttributeValue("itemprop", "author").first()?.text(), - state = parseStatus(infoRoot.select("li a.red").text()), - description = infoRoot.getElementsByAttributeValue("itemprop", "description").first() - ?.html()?.substringAfter(""), - chapters = root.selectFirst("div.chapterbox")?.select("ul.sub_vol_ul > li") - ?.asReversed()?.mapChapters { i, li -> - val a = li.selectFirst("a.chapter_list_a") - val href = a?.attrAsRelativeUrlOrNull("href") - ?.replace("%20", " ") ?: li.parseFailed("Link not found") - MangaChapter( - id = generateUid(href), - name = a.text(), - number = i + 1, - url = href, - uploadDate = parseChapterDateByLang(li.selectFirst("span")?.text().orEmpty()), - source = source, - scanlator = null, - branch = null, - ) - }, - ) - } + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet( + manga.url.toAbsoluteUrl(domain) + "?waring=1", + ).parseHtml() + val root = doc.body().selectFirstOrThrow("div.manga") + val infoRoot = root.selectFirstOrThrow("div.bookintro") + return manga.copy( + tags = infoRoot.getElementsByAttributeValue("itemprop", "genre").first() + ?.select("a")?.mapToSet { a -> + MangaTag( + title = a.text().toTitleCase(), + key = a.attr("href").substringBetween("/", "."), + source = source, + ) + }.orEmpty(), + author = infoRoot.getElementsByAttributeValue("itemprop", "author").first()?.text(), + state = parseStatus(infoRoot.select("li a.red").text()), + description = infoRoot.getElementsByAttributeValue("itemprop", "description").first() + ?.html()?.substringAfter(""), + chapters = root.selectFirst("div.chapterbox")?.select("ul.sub_vol_ul > li") + ?.asReversed()?.mapChapters { i, li -> + val a = li.selectFirst("a.chapter_list_a") + val href = a?.attrAsRelativeUrlOrNull("href") + ?.replace("%20", " ") ?: li.parseFailed("Link not found") + MangaChapter( + id = generateUid(href), + name = a.text(), + number = i + 1, + url = href, + uploadDate = parseChapterDateByLang(li.selectFirst("span")?.text().orEmpty()), + source = source, + scanlator = null, + branch = null, + ) + }, + ) + } - override suspend fun getPages(chapter: MangaChapter): List { - val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() - return doc.body().getElementById("page")?.select("option")?.map { option -> - val url = option.attr("value") - MangaPage( - id = generateUid(url), - url = url, - referer = chapter.url.toAbsoluteUrl(domain), - preview = null, - source = source, - ) - } ?: doc.parseFailed("Pages list not found") - } + override suspend fun getPages(chapter: MangaChapter): List { + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + return doc.body().getElementById("page")?.select("option")?.map { option -> + val url = option.attr("value") + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } ?: doc.parseFailed("Pages list not found") + } - override suspend fun getPageUrl(page: MangaPage): String { - val doc = webClient.httpGet(page.url.toAbsoluteUrl(domain)).parseHtml() - val root = doc.body() - return root.selectFirst("a.pic_download")?.absUrl("href") - ?: doc.parseFailed("Page image not found") - } + override suspend fun getPageUrl(page: MangaPage): String { + val doc = webClient.httpGet(page.url.toAbsoluteUrl(domain)).parseHtml() + val root = doc.body() + return root.selectFirst("a.pic_download")?.absUrl("href") + ?: doc.parseFailed("Page image not found") + } - override suspend fun getTags(): Set { - val doc = webClient.httpGet("https://${domain}/search/?type=high") - .parseHtml() - val root = doc.body().getElementById("search_form") - return root?.select("li.cate_list")?.mapNotNullToSet { li -> - val cateId = li.attr("cate_id") ?: return@mapNotNullToSet null - val a = li.selectFirst("a") ?: return@mapNotNullToSet null - MangaTag( - title = a.text().toTitleCase(), - key = cateId, - source = source, - ) - } ?: doc.parseFailed("Root not found") - } + override suspend fun getTags(): Set { + val doc = webClient.httpGet("https://${domain}/search/?type=high") + .parseHtml() + val root = doc.body().getElementById("search_form") + return root?.select("li.cate_list")?.mapNotNullToSet { li -> + val cateId = li.attr("cate_id") ?: return@mapNotNullToSet null + val a = li.selectFirst("a") ?: return@mapNotNullToSet null + MangaTag( + title = a.text().toTitleCase(), + key = cateId, + source = source, + ) + } ?: doc.parseFailed("Root not found") + } - private fun parseStatus(status: String) = when { - status.contains("Ongoing") -> MangaState.ONGOING - status.contains("Completed") -> MangaState.FINISHED - else -> null - } + private fun parseStatus(status: String) = when { + status.contains("Ongoing") -> MangaState.ONGOING + status.contains("Completed") -> MangaState.FINISHED + else -> null + } - private fun parseChapterDateByLang(date: String): Long { - val dateWords = date.split(" ") + private fun parseChapterDateByLang(date: String): Long { + val dateWords = date.split(" ") - if (dateWords.size == 3) { - if (dateWords[1].contains(",")) { - SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH).tryParse(date) - } else { - val timeAgo = Integer.parseInt(dateWords[0]) - return Calendar.getInstance().apply { - when (dateWords[1]) { - "minutes" -> Calendar.MINUTE // EN-FR - "hours" -> Calendar.HOUR // EN + if (dateWords.size == 3) { + if (dateWords[1].contains(",")) { + SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH).tryParse(date) + } else { + val timeAgo = Integer.parseInt(dateWords[0]) + return Calendar.getInstance().apply { + when (dateWords[1]) { + "minutes" -> Calendar.MINUTE // EN-FR + "hours" -> Calendar.HOUR // EN - "minutos" -> Calendar.MINUTE // ES - "horas" -> Calendar.HOUR + "minutos" -> Calendar.MINUTE // ES + "horas" -> Calendar.HOUR - // "minutos" -> Calendar.MINUTE // BR - "hora" -> Calendar.HOUR + // "minutos" -> Calendar.MINUTE // BR + "hora" -> Calendar.HOUR - "минут" -> Calendar.MINUTE // RU - "часа" -> Calendar.HOUR + "минут" -> Calendar.MINUTE // RU + "часа" -> Calendar.HOUR - "Stunden" -> Calendar.HOUR // DE + "Stunden" -> Calendar.HOUR // DE - "minuti" -> Calendar.MINUTE // IT - "ore" -> Calendar.HOUR + "minuti" -> Calendar.MINUTE // IT + "ore" -> Calendar.HOUR - "heures" -> Calendar.HOUR // FR ("minutes" also French word) - else -> null - }?.let { - add(it, -timeAgo) - } - }.timeInMillis - } - } - return 0L - } + "heures" -> Calendar.HOUR // FR ("minutes" also French word) + else -> null + }?.let { + add(it, -timeAgo) + } + }.timeInMillis + } + } + return 0L + } - @MangaSourceParser("NINEMANGA_EN", "NineManga English", "en") - class English(context: MangaLoaderContext) : NineMangaParser( - context, - MangaSource.NINEMANGA_EN, - "www.ninemanga.com", - ) + @MangaSourceParser("NINEMANGA_EN", "NineManga English", "en") + class English(context: MangaLoaderContext) : NineMangaParser( + context, + MangaSource.NINEMANGA_EN, + "www.ninemanga.com", + ) - @MangaSourceParser("NINEMANGA_ES", "NineManga Español", "es") - class Spanish(context: MangaLoaderContext) : NineMangaParser( - context, - MangaSource.NINEMANGA_ES, - "es.ninemanga.com", - ) + @MangaSourceParser("NINEMANGA_ES", "NineManga Español", "es") + class Spanish(context: MangaLoaderContext) : NineMangaParser( + context, + MangaSource.NINEMANGA_ES, + "es.ninemanga.com", + ) - @MangaSourceParser("NINEMANGA_RU", "NineManga Русский", "ru") - class Russian(context: MangaLoaderContext) : NineMangaParser( - context, - MangaSource.NINEMANGA_RU, - "ru.ninemanga.com", - ) + @MangaSourceParser("NINEMANGA_RU", "NineManga Русский", "ru") + class Russian(context: MangaLoaderContext) : NineMangaParser( + context, + MangaSource.NINEMANGA_RU, + "ru.ninemanga.com", + ) - @MangaSourceParser("NINEMANGA_DE", "NineManga Deutsch", "de") - class Deutsch(context: MangaLoaderContext) : NineMangaParser( - context, - MangaSource.NINEMANGA_DE, - "de.ninemanga.com", - ) + @MangaSourceParser("NINEMANGA_DE", "NineManga Deutsch", "de") + class Deutsch(context: MangaLoaderContext) : NineMangaParser( + context, + MangaSource.NINEMANGA_DE, + "de.ninemanga.com", + ) - @MangaSourceParser("NINEMANGA_BR", "NineManga Brasil", "pt") - class Brazil(context: MangaLoaderContext) : NineMangaParser( - context, - MangaSource.NINEMANGA_BR, - "br.ninemanga.com", - ) + @MangaSourceParser("NINEMANGA_BR", "NineManga Brasil", "pt") + class Brazil(context: MangaLoaderContext) : NineMangaParser( + context, + MangaSource.NINEMANGA_BR, + "br.ninemanga.com", + ) - @MangaSourceParser("NINEMANGA_IT", "NineManga Italiano", "it") - class Italiano(context: MangaLoaderContext) : NineMangaParser( - context, - MangaSource.NINEMANGA_IT, - "it.ninemanga.com", - ) + @MangaSourceParser("NINEMANGA_IT", "NineManga Italiano", "it") + class Italiano(context: MangaLoaderContext) : NineMangaParser( + context, + MangaSource.NINEMANGA_IT, + "it.ninemanga.com", + ) - @MangaSourceParser("NINEMANGA_FR", "NineManga Français", "fr") - class Francais(context: MangaLoaderContext) : NineMangaParser( - context, - MangaSource.NINEMANGA_FR, - "fr.ninemanga.com", - ) + @MangaSourceParser("NINEMANGA_FR", "NineManga Français", "fr") + class Francais(context: MangaLoaderContext) : NineMangaParser( + context, + MangaSource.NINEMANGA_FR, + "fr.ninemanga.com", + ) } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NudeMoonParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NudeMoonParser.kt index 823cd37b..3b8a38d3 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NudeMoonParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NudeMoonParser.kt @@ -15,213 +15,212 @@ private const val MAX_THUMB_INDEX = 19 @MangaSourceParser("NUDEMOON", "Nude-Moon", "ru") internal class NudeMoonParser( - context: MangaLoaderContext, + context: MangaLoaderContext, ) : MangaParser(context, MangaSource.NUDEMOON), MangaParserAuthProvider { - override val configKeyDomain = ConfigKey.Domain( - defaultValue = "nude-moon.org", - presetValues = arrayOf("nude-moon.org", "nude-moon.net"), - ) - override val authUrl: String - get() = "https://${domain}/index.php" - - override val isAuthorized: Boolean - get() { - return context.cookieJar.getCookies(domain).any { - it.name == "fusion_user" - } - } - - override val sortOrders: Set = EnumSet.of( - SortOrder.NEWEST, - SortOrder.POPULARITY, - SortOrder.RATING, - ) - - init { - context.cookieJar.insertCookies( - domain, - "NMfYa=1;", - "nm_mobile=0;", - ) - } - - override suspend fun getList( - offset: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - val domain = domain - val url = when { - !query.isNullOrEmpty() -> "https://$domain/search?stext=${query.urlEncoded()}&rowstart=$offset" - !tags.isNullOrEmpty() -> tags.joinToString( - separator = "_", - prefix = "https://$domain/tags/", - postfix = "&rowstart=$offset", - transform = { it.key.urlEncoded() }, - ) - - else -> "https://$domain/all_manga?${getSortKey(sortOrder)}&rowstart=$offset" - } - val doc = webClient.httpGet(url).parseHtml() - val root = doc.body().run { - selectFirst("td.main-bg") ?: selectFirst("td.main-body") - } ?: doc.parseFailed("Cannot find root") - return root.select("table.news_pic2").mapNotNull { row -> - val a = row.selectFirst("td.bg_style1")?.selectFirst("a") - ?: return@mapNotNull null - val href = a.attrAsRelativeUrl("href") - val title = a.selectFirst("h2")?.text().orEmpty() - val info = row.selectFirst("td[width=100%]") ?: return@mapNotNull null - Manga( - id = generateUid(href), - url = href, - title = title.substringAfter(" / "), - altTitle = title.substringBefore(" / ", "") - .takeUnless { it.isBlank() }, - author = info.getElementsContainingOwnText("Автор:").firstOrNull() - ?.nextElementSibling()?.ownText(), - coverUrl = row.selectFirst("img.news_pic2")?.absUrl("data-src") - .orEmpty(), - tags = row.selectFirst("span.tag-links")?.select("a") - ?.mapToSet { - MangaTag( - title = it.text().toTitleCase(), - key = it.attr("href").substringAfterLast('/'), - source = source, - ) - }.orEmpty(), - source = source, - publicUrl = a.absUrl("href"), - rating = RATING_UNKNOWN, - isNsfw = true, - description = row.selectFirst("div.description")?.html(), - state = null, - ) - } - } - - override suspend fun getDetails(manga: Manga): Manga { - val body = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml().body() - val root = body.selectFirst("table.shoutbox") - ?: body.parseFailed("Cannot find root") - val info = root.select("div.tbl2") - val lastInfo = info.last() - return manga.copy( - largeCoverUrl = body.selectFirst("img.news_pic2")?.absUrl("src"), - description = info.select("div.blockquote").lastOrNull()?.html() ?: manga.description, - tags = info.select("span.tag-links").firstOrNull()?.select("a")?.mapToSet { - MangaTag( - title = it.text().toTitleCase(), - key = it.attr("href").substringAfterLast('/'), - source = source, - ) - }?.plus(manga.tags) ?: manga.tags, - author = lastInfo?.getElementsByAttributeValueContaining("href", "mangaka/")?.text() - ?: manga.author, - chapters = listOf( - MangaChapter( - id = manga.id, - url = getReadLink(manga.url), - source = source, - number = 1, - name = manga.title, - scanlator = lastInfo?.getElementsByAttributeValueContaining("href", "perevod/")?.text(), - uploadDate = lastInfo?.getElementsContainingOwnText("Дата:") - ?.firstOrNull() - ?.html() - ?.parseDate() ?: 0L, - branch = null, - ), - ), - ) - } - - override suspend fun getPages(chapter: MangaChapter): List { - val fullUrl = chapter.url.toAbsoluteUrl(domain) - val doc = webClient.httpGet(fullUrl).parseHtml() - val mangaId = chapter.url.substringAfterLast('/').substringBefore('-').toIntOrNull() - - val script = doc.select("script").firstNotNullOfOrNull { - it.html().takeIf { x -> x.contains(" images = new ") } - } ?: if (isAuthorized) { - doc.parseFailed("Cannot find pages list") - } else { - throw AuthRequiredException(source) - } - val pagesRegex = Regex("images\\[(\\d+)].src\\s*=\\s*'([^']+)'", RegexOption.MULTILINE) - return pagesRegex.findAll(script).map { match -> - val i = match.groupValues[1].toInt() - val url = match.groupValues[2] - MangaPage( - id = generateUid(url), - url = url, - referer = fullUrl, - preview = if (i <= MAX_THUMB_INDEX && mangaId != null) { - val part2 = url.substringBeforeLast('/') - val part3 = url.substringAfterLast('/') - val part1 = part2.substringBeforeLast('/') - "$part1/thumb/$mangaId/thumb_$part3" - } else { - null - }, - source = source, - ) - }.toList() - } - - override suspend fun getTags(): Set { - val domain = domain - val doc = webClient.httpGet("https://$domain/all_manga").parseHtml() - val root = doc.body().getElementsContainingOwnText("Поиск манги по тегам") - .firstOrNull()?.parents()?.find { it.tag().normalName() == "tbody" } - ?.selectFirst("td.textbox")?.selectFirst("td.small") - ?: doc.parseFailed("Tags root not found") - return root.select("a").mapToSet { - MangaTag( - title = it.text().toTitleCase(), - key = it.attr("href").substringAfterLast('/') - .removeSuffix("+"), - source = source, - ) - } - } - - override suspend fun getUsername(): String { - val body = webClient.httpGet("https://${domain}/").parseHtml() - .body() - return body - .getElementsContainingOwnText("Профиль") - .firstOrNull() - ?.attr("href") - ?.substringAfterLast('/') - ?: run { - throw if (body.selectFirst("form[name=\"loginform\"]") != null) { - AuthRequiredException(source) - } else { - body.parseFailed("Cannot find username") - } - } - } - - private fun getSortKey(sortOrder: SortOrder) = - when (sortOrder) { - SortOrder.POPULARITY -> "views" - SortOrder.NEWEST -> "date" - SortOrder.RATING -> "like" - else -> "like" - } - - private fun String.parseDate(): Long { - val dateString = substringBetweenFirst("Дата:", "<")?.trim() ?: return 0 - val dateFormat = SimpleDateFormat("d MMMM yyyy", Locale("ru")) - return dateFormat.tryParse(dateString) - } - - private fun getReadLink(url: String): String { - val prefix = url.substringBefore('-', "") - val suffix = url.substringAfter('-').trimStart('-') - return "$prefix-online-$suffix" - } + override val configKeyDomain = ConfigKey.Domain( + defaultValue = "nude-moon.org", + presetValues = arrayOf("nude-moon.org", "nude-moon.net"), + ) + override val authUrl: String + get() = "https://${domain}/index.php" + + override val isAuthorized: Boolean + get() { + return context.cookieJar.getCookies(domain).any { + it.name == "fusion_user" + } + } + + override val sortOrders: Set = EnumSet.of( + SortOrder.NEWEST, + SortOrder.POPULARITY, + SortOrder.RATING, + ) + + init { + context.cookieJar.insertCookies( + domain, + "NMfYa=1;", + "nm_mobile=0;", + ) + } + + override suspend fun getList( + offset: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + val domain = domain + val url = when { + !query.isNullOrEmpty() -> "https://$domain/search?stext=${query.urlEncoded()}&rowstart=$offset" + !tags.isNullOrEmpty() -> tags.joinToString( + separator = "_", + prefix = "https://$domain/tags/", + postfix = "&rowstart=$offset", + transform = { it.key.urlEncoded() }, + ) + + else -> "https://$domain/all_manga?${getSortKey(sortOrder)}&rowstart=$offset" + } + val doc = webClient.httpGet(url).parseHtml() + val root = doc.body().run { + selectFirst("td.main-bg") ?: selectFirst("td.main-body") + } ?: doc.parseFailed("Cannot find root") + return root.select("table.news_pic2").mapNotNull { row -> + val a = row.selectFirst("td.bg_style1")?.selectFirst("a") + ?: return@mapNotNull null + val href = a.attrAsRelativeUrl("href") + val title = a.selectFirst("h2")?.text().orEmpty() + val info = row.selectFirst("td[width=100%]") ?: return@mapNotNull null + Manga( + id = generateUid(href), + url = href, + title = title.substringAfter(" / "), + altTitle = title.substringBefore(" / ", "") + .takeUnless { it.isBlank() }, + author = info.getElementsContainingOwnText("Автор:").firstOrNull() + ?.nextElementSibling()?.ownText(), + coverUrl = row.selectFirst("img.news_pic2")?.absUrl("data-src") + .orEmpty(), + tags = row.selectFirst("span.tag-links")?.select("a") + ?.mapToSet { + MangaTag( + title = it.text().toTitleCase(), + key = it.attr("href").substringAfterLast('/'), + source = source, + ) + }.orEmpty(), + source = source, + publicUrl = a.absUrl("href"), + rating = RATING_UNKNOWN, + isNsfw = true, + description = row.selectFirst("div.description")?.html(), + state = null, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val body = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml().body() + val root = body.selectFirst("table.shoutbox") + ?: body.parseFailed("Cannot find root") + val info = root.select("div.tbl2") + val lastInfo = info.last() + return manga.copy( + largeCoverUrl = body.selectFirst("img.news_pic2")?.absUrl("src"), + description = info.select("div.blockquote").lastOrNull()?.html() ?: manga.description, + tags = info.select("span.tag-links").firstOrNull()?.select("a")?.mapToSet { + MangaTag( + title = it.text().toTitleCase(), + key = it.attr("href").substringAfterLast('/'), + source = source, + ) + }?.plus(manga.tags) ?: manga.tags, + author = lastInfo?.getElementsByAttributeValueContaining("href", "mangaka/")?.text() + ?: manga.author, + chapters = listOf( + MangaChapter( + id = manga.id, + url = getReadLink(manga.url), + source = source, + number = 1, + name = manga.title, + scanlator = lastInfo?.getElementsByAttributeValueContaining("href", "perevod/")?.text(), + uploadDate = lastInfo?.getElementsContainingOwnText("Дата:") + ?.firstOrNull() + ?.html() + ?.parseDate() ?: 0L, + branch = null, + ), + ), + ) + } + + override suspend fun getPages(chapter: MangaChapter): List { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + val mangaId = chapter.url.substringAfterLast('/').substringBefore('-').toIntOrNull() + + val script = doc.select("script").firstNotNullOfOrNull { + it.html().takeIf { x -> x.contains(" images = new ") } + } ?: if (isAuthorized) { + doc.parseFailed("Cannot find pages list") + } else { + throw AuthRequiredException(source) + } + val pagesRegex = Regex("images\\[(\\d+)].src\\s*=\\s*'([^']+)'", RegexOption.MULTILINE) + return pagesRegex.findAll(script).map { match -> + val i = match.groupValues[1].toInt() + val url = match.groupValues[2] + MangaPage( + id = generateUid(url), + url = url, + preview = if (i <= MAX_THUMB_INDEX && mangaId != null) { + val part2 = url.substringBeforeLast('/') + val part3 = url.substringAfterLast('/') + val part1 = part2.substringBeforeLast('/') + "$part1/thumb/$mangaId/thumb_$part3" + } else { + null + }, + source = source, + ) + }.toList() + } + + override suspend fun getTags(): Set { + val domain = domain + val doc = webClient.httpGet("https://$domain/all_manga").parseHtml() + val root = doc.body().getElementsContainingOwnText("Поиск манги по тегам") + .firstOrNull()?.parents()?.find { it.tag().normalName() == "tbody" } + ?.selectFirst("td.textbox")?.selectFirst("td.small") + ?: doc.parseFailed("Tags root not found") + return root.select("a").mapToSet { + MangaTag( + title = it.text().toTitleCase(), + key = it.attr("href").substringAfterLast('/') + .removeSuffix("+"), + source = source, + ) + } + } + + override suspend fun getUsername(): String { + val body = webClient.httpGet("https://${domain}/").parseHtml() + .body() + return body + .getElementsContainingOwnText("Профиль") + .firstOrNull() + ?.attr("href") + ?.substringAfterLast('/') + ?: run { + throw if (body.selectFirst("form[name=\"loginform\"]") != null) { + AuthRequiredException(source) + } else { + body.parseFailed("Cannot find username") + } + } + } + + private fun getSortKey(sortOrder: SortOrder) = + when (sortOrder) { + SortOrder.POPULARITY -> "views" + SortOrder.NEWEST -> "date" + SortOrder.RATING -> "like" + else -> "like" + } + + private fun String.parseDate(): Long { + val dateString = substringBetweenFirst("Дата:", "<")?.trim() ?: return 0 + val dateFormat = SimpleDateFormat("d MMMM yyyy", Locale("ru")) + return dateFormat.tryParse(dateString) + } + + private fun getReadLink(url: String): String { + val prefix = url.substringBefore('-', "") + val suffix = url.substringAfter('-').trimStart('-') + return "$prefix-online-$suffix" + } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/RemangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/RemangaParser.kt index fa3a24dc..4ae288d5 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/RemangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/RemangaParser.kt @@ -28,259 +28,258 @@ private const val STATUS_FINISHED = 0 @MangaSourceParser("REMANGA", "Remanga", "ru") internal class RemangaParser( - context: MangaLoaderContext, + context: MangaLoaderContext, ) : PagedMangaParser(context, MangaSource.REMANGA, PAGE_SIZE), MangaParserAuthProvider { - private val baseHeaders = Headers.Builder() - .add("User-Agent", "Mozilla/5.0 (Android 13; Mobile; rv:68.0) Gecko/68.0 Firefox/109.0") - .build() + private val baseHeaders = Headers.Builder() + .add("User-Agent", "Mozilla/5.0 (Android 13; Mobile; rv:68.0) Gecko/68.0 Firefox/109.0") + .build() - override val headers - get() = getApiHeaders() + override val headers + get() = getApiHeaders() - override val configKeyDomain = ConfigKey.Domain("remanga.org", null) - override val authUrl: String - get() = "https://${domain}/user/login" + override val configKeyDomain = ConfigKey.Domain("remanga.org", null) + override val authUrl: String + get() = "https://${domain}/user/login" - override val sortOrders: Set = EnumSet.of( - SortOrder.UPDATED, - SortOrder.POPULARITY, - SortOrder.RATING, - SortOrder.NEWEST, - ) + override val sortOrders: Set = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.RATING, + SortOrder.NEWEST, + ) - override val isAuthorized: Boolean - get() { - return context.cookieJar.getCookies(domain).any { - it.name == "user" - } - } + override val isAuthorized: Boolean + get() { + return context.cookieJar.getCookies(domain).any { + it.name == "user" + } + } - private val regexLastUrlPath = Regex("/[^/]+/?$") + private val regexLastUrlPath = Regex("/[^/]+/?$") - override suspend fun getListPage( - page: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - copyCookies() - val domain = domain - val urlBuilder = StringBuilder() - .append("https://api.") - .append(domain) - if (query != null) { - urlBuilder.append("/api/search/?query=") - .append(query.urlEncoded()) - } else { - urlBuilder.append("/api/search/catalog/?ordering=") - .append(getSortKey(sortOrder)) - tags?.forEach { tag -> - urlBuilder.append("&genres=") - urlBuilder.append(tag.key) - } - } - urlBuilder - .append("&page=") - .append(page) - .append("&count=") - .append(PAGE_SIZE) - val content = webClient.httpGet(urlBuilder.toString()).parseJson() - .getJSONArray("content") - return content.mapJSON { jo -> - val url = "/manga/${jo.getString("dir")}" - val img = jo.getJSONObject("img") - Manga( - id = generateUid(url), - url = url, - publicUrl = "https://$domain$url", - title = jo.getString("rus_name"), - altTitle = jo.getString("en_name"), - rating = jo.getString("avg_rating").toFloatOrNull()?.div(10f) ?: RATING_UNKNOWN, - coverUrl = "https://api.$domain${img.getString("mid")}", - largeCoverUrl = "https://api.$domain${img.getString("high")}", - author = null, - isNsfw = false, - state = null, - tags = jo.optJSONArray("genres")?.mapJSONToSet { g -> - MangaTag( - title = g.getString("name").toTitleCase(), - key = g.getInt("id").toString(), - source = MangaSource.REMANGA, - ) - }.orEmpty(), - source = MangaSource.REMANGA, - ) - } - } + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + copyCookies() + val domain = domain + val urlBuilder = StringBuilder() + .append("https://api.") + .append(domain) + if (query != null) { + urlBuilder.append("/api/search/?query=") + .append(query.urlEncoded()) + } else { + urlBuilder.append("/api/search/catalog/?ordering=") + .append(getSortKey(sortOrder)) + tags?.forEach { tag -> + urlBuilder.append("&genres=") + urlBuilder.append(tag.key) + } + } + urlBuilder + .append("&page=") + .append(page) + .append("&count=") + .append(PAGE_SIZE) + val content = webClient.httpGet(urlBuilder.toString()).parseJson() + .getJSONArray("content") + return content.mapJSON { jo -> + val url = "/manga/${jo.getString("dir")}" + val img = jo.getJSONObject("img") + Manga( + id = generateUid(url), + url = url, + publicUrl = "https://$domain$url", + title = jo.getString("rus_name"), + altTitle = jo.getString("en_name"), + rating = jo.getString("avg_rating").toFloatOrNull()?.div(10f) ?: RATING_UNKNOWN, + coverUrl = "https://api.$domain${img.getString("mid")}", + largeCoverUrl = "https://api.$domain${img.getString("high")}", + author = null, + isNsfw = false, + state = null, + tags = jo.optJSONArray("genres")?.mapJSONToSet { g -> + MangaTag( + title = g.getString("name").toTitleCase(), + key = g.getInt("id").toString(), + source = MangaSource.REMANGA, + ) + }.orEmpty(), + source = MangaSource.REMANGA, + ) + } + } - override suspend fun getDetails(manga: Manga): Manga { - copyCookies() - val domain = domain - val slug = manga.url.find(regexLastUrlPath) - ?: throw ParseException("Cannot obtain slug from ${manga.url}", manga.publicUrl) - val data = webClient.httpGet( - url = "https://api.$domain/api/titles$slug/", - ).handle401().parseJson() - val content = try { - data.getJSONObject("content") - } catch (e: JSONException) { - throw ParseException(data.optString("msg"), manga.publicUrl, e) - } - val branchId = content.getJSONArray("branches").optJSONObject(0) - ?.getLong("id") ?: throw ParseException("No branches found", manga.publicUrl) - val chapters = grabChapters(domain, branchId) - val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.US) - return manga.copy( - description = content.getString("description"), - state = when (content.optJSONObject("status")?.getInt("id")) { - STATUS_ONGOING -> MangaState.ONGOING - STATUS_FINISHED -> MangaState.FINISHED - else -> null - }, - tags = content.getJSONArray("genres").mapJSONToSet { g -> - MangaTag( - title = g.getString("name").toTitleCase(), - key = g.getInt("id").toString(), - source = MangaSource.REMANGA, - ) - }, - chapters = chapters.mapChapters { i, jo -> - if ( - jo.getBooleanOrDefault("is_paid", false) && - !jo.getBooleanOrDefault("is_bought", false) - ) { - return@mapChapters null - } - val id = jo.getLong("id") - val name = jo.getString("name").toTitleCase(Locale.ROOT) - val publishers = jo.optJSONArray("publishers") - MangaChapter( - id = generateUid(id), - url = "/api/titles/chapters/$id/", - number = chapters.size - i, - name = buildString { - append("Том ") - append(jo.optString("tome", "0")) - append(". ") - append("Глава ") - append(jo.optString("chapter", "0")) - if (name.isNotEmpty()) { - append(" - ") - append(name) - } - }, - uploadDate = dateFormat.tryParse(jo.getString("upload_date")), - scanlator = publishers?.optJSONObject(0)?.getStringOrNull("name"), - source = MangaSource.REMANGA, - branch = null, - ) - }.asReversed(), - ) - } + override suspend fun getDetails(manga: Manga): Manga { + copyCookies() + val domain = domain + val slug = manga.url.find(regexLastUrlPath) + ?: throw ParseException("Cannot obtain slug from ${manga.url}", manga.publicUrl) + val data = webClient.httpGet( + url = "https://api.$domain/api/titles$slug/", + ).handle401().parseJson() + val content = try { + data.getJSONObject("content") + } catch (e: JSONException) { + throw ParseException(data.optString("msg"), manga.publicUrl, e) + } + val branchId = content.getJSONArray("branches").optJSONObject(0) + ?.getLong("id") ?: throw ParseException("No branches found", manga.publicUrl) + val chapters = grabChapters(domain, branchId) + val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.US) + return manga.copy( + description = content.getString("description"), + state = when (content.optJSONObject("status")?.getInt("id")) { + STATUS_ONGOING -> MangaState.ONGOING + STATUS_FINISHED -> MangaState.FINISHED + else -> null + }, + tags = content.getJSONArray("genres").mapJSONToSet { g -> + MangaTag( + title = g.getString("name").toTitleCase(), + key = g.getInt("id").toString(), + source = MangaSource.REMANGA, + ) + }, + chapters = chapters.mapChapters { i, jo -> + if ( + jo.getBooleanOrDefault("is_paid", false) && + !jo.getBooleanOrDefault("is_bought", false) + ) { + return@mapChapters null + } + val id = jo.getLong("id") + val name = jo.getString("name").toTitleCase(Locale.ROOT) + val publishers = jo.optJSONArray("publishers") + MangaChapter( + id = generateUid(id), + url = "/api/titles/chapters/$id/", + number = chapters.size - i, + name = buildString { + append("Том ") + append(jo.optString("tome", "0")) + append(". ") + append("Глава ") + append(jo.optString("chapter", "0")) + if (name.isNotEmpty()) { + append(" - ") + append(name) + } + }, + uploadDate = dateFormat.tryParse(jo.getString("upload_date")), + scanlator = publishers?.optJSONObject(0)?.getStringOrNull("name"), + source = MangaSource.REMANGA, + branch = null, + ) + }.asReversed(), + ) + } - override suspend fun getPages(chapter: MangaChapter): List { - val referer = "https://${domain}/" - val content = webClient.httpGet(chapter.url.toAbsoluteUrl(getDomain("api"))) - .handle401() - .parseJson() - .getJSONObject("content") - val pages = content.optJSONArray("pages") - if (pages == null) { - val pubDate = content.getStringOrNull("pub_date")?.let { - SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.US).tryParse(it) - } - if (pubDate != null && pubDate > System.currentTimeMillis()) { - val at = SimpleDateFormat.getDateInstance(DateFormat.LONG).format(Date(pubDate)) - throw ContentUnavailableException("Глава станет доступной $at") - } else { - throw ContentUnavailableException("Глава недоступна") - } - } - val result = ArrayList(pages.length()) - for (i in 0 until pages.length()) { - when (val item = pages.get(i)) { - is JSONObject -> result += parsePage(item, referer) - is JSONArray -> item.mapJSONTo(result) { parsePage(it, referer) } - else -> throw ParseException("Unknown json item $item", chapter.url) - } - } - return result - } + override suspend fun getPages(chapter: MangaChapter): List { + val referer = "https://${domain}/" + val content = webClient.httpGet(chapter.url.toAbsoluteUrl(getDomain("api"))) + .handle401() + .parseJson() + .getJSONObject("content") + val pages = content.optJSONArray("pages") + if (pages == null) { + val pubDate = content.getStringOrNull("pub_date")?.let { + SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.US).tryParse(it) + } + if (pubDate != null && pubDate > System.currentTimeMillis()) { + val at = SimpleDateFormat.getDateInstance(DateFormat.LONG).format(Date(pubDate)) + throw ContentUnavailableException("Глава станет доступной $at") + } else { + throw ContentUnavailableException("Глава недоступна") + } + } + val result = ArrayList(pages.length()) + for (i in 0 until pages.length()) { + when (val item = pages.get(i)) { + is JSONObject -> result += parsePage(item, referer) + is JSONArray -> item.mapJSONTo(result) { parsePage(it, referer) } + else -> throw ParseException("Unknown json item $item", chapter.url) + } + } + return result + } - override suspend fun getTags(): Set { - val domain = domain - val content = webClient.httpGet("https://api.$domain/api/forms/titles/?get=genres") - .parseJson().getJSONObject("content").getJSONArray("genres") - return content.mapJSONToSet { jo -> - MangaTag( - title = jo.getString("name").toTitleCase(), - key = jo.getInt("id").toString(), - source = source, - ) - } - } + override suspend fun getTags(): Set { + val domain = domain + val content = webClient.httpGet("https://api.$domain/api/forms/titles/?get=genres") + .parseJson().getJSONObject("content").getJSONArray("genres") + return content.mapJSONToSet { jo -> + MangaTag( + title = jo.getString("name").toTitleCase(), + key = jo.getInt("id").toString(), + source = source, + ) + } + } - override suspend fun getUsername(): String { - val jo = webClient.httpGet( - url = "https://api.${domain}/api/users/current/", - ).handle401().parseJson() - return jo.getJSONObject("content").getString("username") - } + override suspend fun getUsername(): String { + val jo = webClient.httpGet( + url = "https://api.${domain}/api/users/current/", + ).handle401().parseJson() + return jo.getJSONObject("content").getString("username") + } - private fun getApiHeaders(): Headers { - val userCookie = context.cookieJar.getCookies(domain).find { - it.name == "user" - } ?: return baseHeaders - val jo = JSONObject(URLDecoder.decode(userCookie.value, Charsets.UTF_8.name())) - val accessToken = jo.getStringOrNull("access_token") ?: return baseHeaders - return baseHeaders.newBuilder().add("authorization", "bearer $accessToken").build() - } + private fun getApiHeaders(): Headers { + val userCookie = context.cookieJar.getCookies(domain).find { + it.name == "user" + } ?: return baseHeaders + val jo = JSONObject(URLDecoder.decode(userCookie.value, Charsets.UTF_8.name())) + val accessToken = jo.getStringOrNull("access_token") ?: return baseHeaders + return baseHeaders.newBuilder().add("authorization", "bearer $accessToken").build() + } - private fun copyCookies() { - val domain = domain - context.cookieJar.copyCookies(domain, "api.$domain") - } + private fun copyCookies() { + val domain = domain + context.cookieJar.copyCookies(domain, "api.$domain") + } - private fun getSortKey(order: SortOrder?) = when (order) { - SortOrder.UPDATED -> "-chapter_date" - SortOrder.POPULARITY -> "-rating" - SortOrder.RATING -> "-votes" - SortOrder.NEWEST -> "-id" - else -> "-chapter_date" - } + private fun getSortKey(order: SortOrder?) = when (order) { + SortOrder.UPDATED -> "-chapter_date" + SortOrder.POPULARITY -> "-rating" + SortOrder.RATING -> "-votes" + SortOrder.NEWEST -> "-id" + else -> "-chapter_date" + } - private fun parsePage(jo: JSONObject, referer: String) = MangaPage( - id = generateUid(jo.getLong("id")), - url = jo.getString("link"), - preview = null, - referer = referer, - source = source, - ) + private fun parsePage(jo: JSONObject, referer: String) = MangaPage( + id = generateUid(jo.getLong("id")), + url = jo.getString("link"), + preview = null, + source = source, + ) - private suspend fun grabChapters(domain: String, branchId: Long): List { - val result = ArrayList(100) - var page = 1 - while (true) { - val content = webClient.httpGet( - url = "https://api.$domain/api/titles/chapters/?branch_id=$branchId&page=$page&count=100", - ).handle401().parseJson().getJSONArray("content") - val len = content.length() - if (len == 0) { - break - } - result.ensureCapacity(result.size + len) - for (i in 0 until len) { - result.add(content.getJSONObject(i)) - } - page++ - } - return result - } + private suspend fun grabChapters(domain: String, branchId: Long): List { + val result = ArrayList(100) + var page = 1 + while (true) { + val content = webClient.httpGet( + url = "https://api.$domain/api/titles/chapters/?branch_id=$branchId&page=$page&count=100", + ).handle401().parseJson().getJSONArray("content") + val len = content.length() + if (len == 0) { + break + } + result.ensureCapacity(result.size + len) + for (i in 0 until len) { + result.add(content.getJSONObject(i)) + } + page++ + } + return result + } - private fun Response.handle401() = apply { - if (code == HttpURLConnection.HTTP_UNAUTHORIZED) { - throw AuthRequiredException(source) - } - } + private fun Response.handle401() = apply { + if (code == HttpURLConnection.HTTP_UNAUTHORIZED) { + throw AuthRequiredException(source) + } + } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/TruyentranhLHParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/TruyentranhLHParser.kt index 715c3a1a..9c97582a 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/TruyentranhLHParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/TruyentranhLHParser.kt @@ -124,7 +124,6 @@ class TruyentranhLHParser(context: MangaLoaderContext) : MangaPage( id = generateUid(imageUrl), url = imageUrl, - referer = url, preview = null, source = source, ) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/UnionMangasParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/UnionMangasParser.kt index 53b959d0..491923a4 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/UnionMangasParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/UnionMangasParser.kt @@ -122,7 +122,6 @@ class UnionMangasParser(context: MangaLoaderContext) : PagedMangaParser(context, MangaPage( id = generateUid(href), url = href, - referer = fullUrl, preview = null, source = source, ) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt index fd975206..ca1bccb6 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt @@ -226,7 +226,6 @@ internal abstract class GroupleParser( id = generateUid(url), url = "$primaryServer|$serversStr|$url", preview = null, - referer = chapter.url, source = source, ) } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Madara5Parser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Madara5Parser.kt index 1b480d01..5c699324 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Madara5Parser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Madara5Parser.kt @@ -112,7 +112,6 @@ abstract class Madara5Parser @InternalParsersApi constructor( MangaPage( id = generateUid(url), url = url, - referer = fullUrl, preview = null, source = source, ) @@ -180,7 +179,5 @@ abstract class Madara5Parser @InternalParsersApi constructor( ) @MangaSourceParser("MANGAOWLS", "BeautyManga", "en") - class BeautyManga(context: MangaLoaderContext) : Madara5Parser(context, MangaSource.MANGAOWLS, "beautymanga.com") { - - } + class BeautyManga(context: MangaLoaderContext) : Madara5Parser(context, MangaSource.MANGAOWLS, "beautymanga.com") } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt index e03e8550..4091e616 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt @@ -186,7 +186,6 @@ internal abstract class MadaraParser( id = generateUid(url), url = url, preview = null, - referer = fullUrl, source = source, ) } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/NeatManga.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/NeatManga.kt index 3e8e4bb7..691938fc 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/NeatManga.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/NeatManga.kt @@ -88,7 +88,6 @@ internal class NeatManga(context: MangaLoaderContext) : MadaraParser(context, Ma id = generateUid(url), url = url, preview = null, - referer = fullUrl, source = source, ) } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt index ef776dc5..6199f401 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt @@ -187,7 +187,6 @@ internal abstract class MangaReaderParser( MangaPage( id = generateUid(images.getString(i)), url = images.getString(i), - referer = chapterUrl, preview = null, source = source, ), diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/multichan/ChanParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/multichan/ChanParser.kt index 2a886fca..082cfb5b 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/multichan/ChanParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/multichan/ChanParser.kt @@ -10,174 +10,173 @@ import java.text.SimpleDateFormat import java.util.* internal abstract class ChanParser( - context: MangaLoaderContext, - source: MangaSource, + context: MangaLoaderContext, + source: MangaSource, ) : MangaParser(context, source), MangaParserAuthProvider { - override val sortOrders: Set = EnumSet.of( - SortOrder.NEWEST, - SortOrder.POPULARITY, - SortOrder.ALPHABETICAL, - ) + override val sortOrders: Set = EnumSet.of( + SortOrder.NEWEST, + SortOrder.POPULARITY, + SortOrder.ALPHABETICAL, + ) - override val authUrl: String - get() = "https://${domain}" + override val authUrl: String + get() = "https://${domain}" - override val isAuthorized: Boolean - get() = context.cookieJar.getCookies(domain).any { it.name == "dle_user_id" } + override val isAuthorized: Boolean + get() = context.cookieJar.getCookies(domain).any { it.name == "dle_user_id" } - override suspend fun getList( - offset: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - val domain = domain - val url = when { - !query.isNullOrEmpty() -> { - if (offset != 0) { - return emptyList() - } - "https://$domain/?do=search&subaction=search&story=${query.urlEncoded()}" - } + override suspend fun getList( + offset: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + val domain = domain + val url = when { + !query.isNullOrEmpty() -> { + if (offset != 0) { + return emptyList() + } + "https://$domain/?do=search&subaction=search&story=${query.urlEncoded()}" + } - !tags.isNullOrEmpty() -> tags.joinToString( - prefix = "https://$domain/tags/", - postfix = "&n=${getSortKey2(sortOrder)}?offset=$offset", - separator = "+", - ) { tag -> tag.key } + !tags.isNullOrEmpty() -> tags.joinToString( + prefix = "https://$domain/tags/", + postfix = "&n=${getSortKey2(sortOrder)}?offset=$offset", + separator = "+", + ) { tag -> tag.key } - else -> "https://$domain/${getSortKey(sortOrder)}?offset=$offset" - } - val doc = webClient.httpGet(url).parseHtml() - val root = doc.body().selectFirst("div.main_fon")?.getElementById("content") - ?: doc.parseFailed("Cannot find root") - return root.select("div.content_row").mapNotNull { row -> - val a = row.selectFirst("div.manga_row1")?.selectFirst("h2")?.selectFirst("a") - ?: return@mapNotNull null - val href = a.attrAsRelativeUrl("href") - Manga( - id = generateUid(href), - url = href, - publicUrl = href.toAbsoluteUrl(a.host ?: domain), - altTitle = a.attr("title"), - title = a.text().substringAfterLast('(').substringBeforeLast(')'), - author = row.getElementsByAttributeValueStarting( - "href", - "/mangaka", - ).firstOrNull()?.text(), - coverUrl = row.selectFirst("div.manga_images")?.selectFirst("img") - ?.absUrl("src").orEmpty(), - tags = runCatching { - row.selectFirst("div.genre")?.select("a")?.mapToSet { - MangaTag( - title = it.text().toTagName(), - key = it.attr("href").substringAfterLast('/').urlEncoded(), - source = source, - ) - } - }.getOrNull().orEmpty(), - rating = RATING_UNKNOWN, - state = null, - isNsfw = false, - source = source, - ) - } - } + else -> "https://$domain/${getSortKey(sortOrder)}?offset=$offset" + } + val doc = webClient.httpGet(url).parseHtml() + val root = doc.body().selectFirst("div.main_fon")?.getElementById("content") + ?: doc.parseFailed("Cannot find root") + return root.select("div.content_row").mapNotNull { row -> + val a = row.selectFirst("div.manga_row1")?.selectFirst("h2")?.selectFirst("a") + ?: return@mapNotNull null + val href = a.attrAsRelativeUrl("href") + Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(a.host ?: domain), + altTitle = a.attr("title"), + title = a.text().substringAfterLast('(').substringBeforeLast(')'), + author = row.getElementsByAttributeValueStarting( + "href", + "/mangaka", + ).firstOrNull()?.text(), + coverUrl = row.selectFirst("div.manga_images")?.selectFirst("img") + ?.absUrl("src").orEmpty(), + tags = runCatching { + row.selectFirst("div.genre")?.select("a")?.mapToSet { + MangaTag( + title = it.text().toTagName(), + key = it.attr("href").substringAfterLast('/').urlEncoded(), + source = source, + ) + } + }.getOrNull().orEmpty(), + rating = RATING_UNKNOWN, + state = null, + isNsfw = false, + source = source, + ) + } + } - override suspend fun getDetails(manga: Manga): Manga { - val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - val root = doc.body().getElementById("dle-content") ?: doc.parseFailed("Cannot find root") - val dateFormat = SimpleDateFormat("yyyy-MM-dd", Locale.US) - return manga.copy( - description = root.getElementById("description")?.html()?.substringBeforeLast(" - val href = tr?.selectFirst("a")?.attrAsRelativeUrlOrNull("href") - ?: return@mapChapters null - MangaChapter( - id = generateUid(href), - name = tr.selectFirst("a")?.text().orEmpty(), - number = i + 1, - url = href, - scanlator = null, - branch = null, - uploadDate = dateFormat.tryParse(tr.selectFirst("div.date")?.text()), - source = source, - ) - }, - ) - } + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val root = doc.body().getElementById("dle-content") ?: doc.parseFailed("Cannot find root") + val dateFormat = SimpleDateFormat("yyyy-MM-dd", Locale.US) + return manga.copy( + description = root.getElementById("description")?.html()?.substringBeforeLast(" + val href = tr?.selectFirst("a")?.attrAsRelativeUrlOrNull("href") + ?: return@mapChapters null + MangaChapter( + id = generateUid(href), + name = tr.selectFirst("a")?.text().orEmpty(), + number = i + 1, + url = href, + scanlator = null, + branch = null, + uploadDate = dateFormat.tryParse(tr.selectFirst("div.date")?.text()), + source = source, + ) + }, + ) + } - override suspend fun getPages(chapter: MangaChapter): List { - val fullUrl = chapter.url.toAbsoluteUrl(domain) - val doc = webClient.httpGet(fullUrl).parseHtml() - val scripts = doc.select("script") - for (script in scripts) { - val data = script.html() - val pos = data.indexOf("\"fullimg") - if (pos == -1) { - continue - } - val json = data.substring(pos).substringAfter('[').substringBefore(';') - .substringBeforeLast(']') - val domain = domain - return json.split(",").mapNotNull { - it.trim() - .removeSurrounding('"', '\'') - .toRelativeUrl(domain) - .takeUnless(String::isBlank) - }.map { url -> - MangaPage( - id = generateUid(url), - url = url, - preview = null, - referer = fullUrl, - source = source, - ) - } - } - doc.parseFailed("Pages list not found at ${chapter.url}") - } + override suspend fun getPages(chapter: MangaChapter): List { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + val scripts = doc.select("script") + for (script in scripts) { + val data = script.html() + val pos = data.indexOf("\"fullimg") + if (pos == -1) { + continue + } + val json = data.substring(pos).substringAfter('[').substringBefore(';') + .substringBeforeLast(']') + val domain = domain + return json.split(",").mapNotNull { + it.trim() + .removeSurrounding('"', '\'') + .toRelativeUrl(domain) + .takeUnless(String::isBlank) + }.map { url -> + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } + doc.parseFailed("Pages list not found at ${chapter.url}") + } - override suspend fun getTags(): Set { - val domain = domain - val doc = webClient.httpGet("https://$domain/mostfavorites&sort=manga").parseHtml() - val root = doc.body().selectFirst("div.main_fon")?.getElementById("side") - ?.select("ul")?.last() ?: doc.parseFailed("Cannot find root") - return root.select("li.sidetag").mapToSet { li -> - val a = li.children().lastOrNull() ?: li.parseFailed("a is null") - MangaTag( - title = a.text().toTagName(), - key = a.attr("href").substringAfterLast('/'), - source = source, - ) - } - } + override suspend fun getTags(): Set { + val domain = domain + val doc = webClient.httpGet("https://$domain/mostfavorites&sort=manga").parseHtml() + val root = doc.body().selectFirst("div.main_fon")?.getElementById("side") + ?.select("ul")?.last() ?: doc.parseFailed("Cannot find root") + return root.select("li.sidetag").mapToSet { li -> + val a = li.children().lastOrNull() ?: li.parseFailed("a is null") + MangaTag( + title = a.text().toTagName(), + key = a.attr("href").substringAfterLast('/'), + source = source, + ) + } + } - override suspend fun getUsername(): String { - val doc = webClient.httpGet("https://${domain}").parseHtml().body() - val root = doc.requireElementById("top_user") - val a = root.getElementsByAttributeValueContaining("href", "/user/").firstOrNull() - ?: throw AuthRequiredException(source) - return a.attr("href").removeSuffix('/').substringAfterLast('/') - } + override suspend fun getUsername(): String { + val doc = webClient.httpGet("https://${domain}").parseHtml().body() + val root = doc.requireElementById("top_user") + val a = root.getElementsByAttributeValueContaining("href", "/user/").firstOrNull() + ?: throw AuthRequiredException(source) + return a.attr("href").removeSuffix('/').substringAfterLast('/') + } - private fun getSortKey(sortOrder: SortOrder) = - when (sortOrder) { - SortOrder.ALPHABETICAL -> "catalog" - SortOrder.POPULARITY -> "mostfavorites" - SortOrder.NEWEST -> "manga/new" - else -> "mostfavorites" - } + private fun getSortKey(sortOrder: SortOrder) = + when (sortOrder) { + SortOrder.ALPHABETICAL -> "catalog" + SortOrder.POPULARITY -> "mostfavorites" + SortOrder.NEWEST -> "manga/new" + else -> "mostfavorites" + } - private fun getSortKey2(sortOrder: SortOrder) = - when (sortOrder) { - SortOrder.ALPHABETICAL -> "abcasc" - SortOrder.POPULARITY -> "favdesc" - SortOrder.NEWEST -> "datedesc" - else -> "favdesc" - } + private fun getSortKey2(sortOrder: SortOrder) = + when (sortOrder) { + SortOrder.ALPHABETICAL -> "abcasc" + SortOrder.POPULARITY -> "favdesc" + SortOrder.NEWEST -> "datedesc" + else -> "favdesc" + } - private fun String.toTagName() = replace('_', ' ').toTitleCase() + private fun String.toTagName() = replace('_', ' ').toTitleCase() } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/rulib/MangaLibParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/rulib/MangaLibParser.kt index d00dd2dd..ed6fba15 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/rulib/MangaLibParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/rulib/MangaLibParser.kt @@ -20,284 +20,283 @@ import java.text.SimpleDateFormat import java.util.* internal open class MangaLibParser( - context: MangaLoaderContext, - source: MangaSource, + context: MangaLoaderContext, + source: MangaSource, ) : PagedMangaParser(context, source, pageSize = 60), MangaParserAuthProvider { - override val configKeyDomain = ConfigKey.Domain("mangalib.me", null) + override val configKeyDomain = ConfigKey.Domain("mangalib.me", null) - override val authUrl: String - get() = "https://${domain}/login" + override val authUrl: String + get() = "https://${domain}/login" - override val sortOrders: Set = EnumSet.of( - SortOrder.RATING, - SortOrder.ALPHABETICAL, - SortOrder.POPULARITY, - SortOrder.UPDATED, - SortOrder.NEWEST, - ) + override val sortOrders: Set = EnumSet.of( + SortOrder.RATING, + SortOrder.ALPHABETICAL, + SortOrder.POPULARITY, + SortOrder.UPDATED, + SortOrder.NEWEST, + ) - override suspend fun getListPage( - page: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - if (!query.isNullOrEmpty()) { - return if (page == searchPaginator.firstPage) search(query) else emptyList() - } - val url = buildString { - append("https://") - append(domain) - append("/manga-list?dir=") - append(getSortKey(sortOrder)) - append("&page=") - append(page) - tags?.forEach { tag -> - append("&genres[include][]=") - append(tag.key) - } - } - val doc = webClient.httpGet(url).parseHtml() - val root = doc.body().getElementById("manga-list") ?: doc.parseFailed("Root not found") - val items = root.selectFirst("div.media-cards-grid")?.select("div.media-card-wrap") - ?: return emptyList() - return items.mapNotNull { card -> - val a = card.selectFirst("a.media-card") ?: return@mapNotNull null - val href = a.attrAsRelativeUrl("href") - Manga( - id = generateUid(href), - title = card.selectFirst("h3")?.text().orEmpty(), - coverUrl = a.absUrl("data-src"), - altTitle = null, - author = null, - rating = RATING_UNKNOWN, - url = href, - publicUrl = href.toAbsoluteUrl(a.host ?: domain), - tags = emptySet(), - state = null, - isNsfw = false, - source = source, - ) - } - } + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + if (!query.isNullOrEmpty()) { + return if (page == searchPaginator.firstPage) search(query) else emptyList() + } + val url = buildString { + append("https://") + append(domain) + append("/manga-list?dir=") + append(getSortKey(sortOrder)) + append("&page=") + append(page) + tags?.forEach { tag -> + append("&genres[include][]=") + append(tag.key) + } + } + val doc = webClient.httpGet(url).parseHtml() + val root = doc.body().getElementById("manga-list") ?: doc.parseFailed("Root not found") + val items = root.selectFirst("div.media-cards-grid")?.select("div.media-card-wrap") + ?: return emptyList() + return items.mapNotNull { card -> + val a = card.selectFirst("a.media-card") ?: return@mapNotNull null + val href = a.attrAsRelativeUrl("href") + Manga( + id = generateUid(href), + title = card.selectFirst("h3")?.text().orEmpty(), + coverUrl = a.absUrl("data-src"), + altTitle = null, + author = null, + rating = RATING_UNKNOWN, + url = href, + publicUrl = href.toAbsoluteUrl(a.host ?: domain), + tags = emptySet(), + state = null, + isNsfw = false, + source = source, + ) + } + } - override suspend fun getDetails(manga: Manga): Manga { - val fullUrl = manga.url.toAbsoluteUrl(domain) - val doc = webClient.httpGet("$fullUrl?section=info").parseHtml() - val root = doc.body().getElementById("main-page") ?: throw ParseException("Root not found", fullUrl) - val title = root.selectFirst("div.media-header__wrap")?.children() - val info = root.selectFirst("div.media-content") - val chaptersDoc = webClient.httpGet("$fullUrl?section=chapters").parseHtml() - val scripts = chaptersDoc.select("script") - val dateFormat = SimpleDateFormat("yyy-MM-dd", Locale.US) - var chapters: ChaptersListBuilder? = null - scripts@ for (script in scripts) { - val raw = script.html().lines() - for (line in raw) { - if (line.startsWith("window.__DATA__")) { - val json = JSONObject(line.substringAfter('=').substringBeforeLast(';')) - val list = json.getJSONObject("chapters").getJSONArray("list") - val id = json.optJSONObject("user")?.getLong("id")?.toString() ?: "not" - val total = list.length() - chapters = ChaptersListBuilder(total) - for (i in 0 until total) { - val item = list.getJSONObject(i) - val chapterId = item.getLong("chapter_id") - val scanlator = item.getStringOrNull("username") - val url = buildString { - if (isAuthorized) { - append(manga.url) - append("/v") - append(item.getInt("chapter_volume")) - append("/c") - append(item.getString("chapter_number")) - append("?ui=") - append(id) - } else { - append(manga.url) - append("/v") - append(item.getInt("chapter_volume")) - append("/c") - append(item.getString("chapter_number")) - } - } - val nameChapter = item.getStringOrNull("chapter_name") - val volume = item.getInt("chapter_volume") - val number = item.getString("chapter_number") - val fullNameChapter = "Том $volume. Глава $number" - chapters.add( - MangaChapter( - id = generateUid(chapterId), - url = url, - source = source, - number = total - i, - uploadDate = dateFormat.tryParse( - item.getString("chapter_created_at").substringBefore(" "), - ), - scanlator = scanlator, - branch = null, - name = if (nameChapter.isNullOrBlank()) fullNameChapter else "$fullNameChapter - $nameChapter", - ), - ) - } - chapters.reverse() - break@scripts - } - } - } - return manga.copy( - title = title?.getOrNull(0)?.text()?.takeUnless(String::isBlank) ?: manga.title, - altTitle = title?.getOrNull(1)?.text()?.substringBefore('/')?.trim(), - rating = root.selectFirst("div.media-stats-item__score") - ?.selectFirst("span") - ?.text()?.toFloatOrNull()?.div(5f) ?: manga.rating, - author = info?.getElementsMatchingOwnText("Автор")?.firstOrNull() - ?.nextElementSibling()?.text() ?: manga.author, - tags = info?.selectFirst("div.media-tags") - ?.select("a.media-tag-item")?.mapNotNullToSet { a -> - val href = a.attr("href") - if (href.contains("genres")) { - MangaTag( - title = a.text().toTitleCase(), - key = href.substringAfterLast('='), - source = source, - ) - } else null - } ?: manga.tags, - isNsfw = isNsfw(doc), - description = info?.selectFirst("div.media-description__text")?.html(), - chapters = chapters?.toList(), - ) - } + override suspend fun getDetails(manga: Manga): Manga { + val fullUrl = manga.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet("$fullUrl?section=info").parseHtml() + val root = doc.body().getElementById("main-page") ?: throw ParseException("Root not found", fullUrl) + val title = root.selectFirst("div.media-header__wrap")?.children() + val info = root.selectFirst("div.media-content") + val chaptersDoc = webClient.httpGet("$fullUrl?section=chapters").parseHtml() + val scripts = chaptersDoc.select("script") + val dateFormat = SimpleDateFormat("yyy-MM-dd", Locale.US) + var chapters: ChaptersListBuilder? = null + scripts@ for (script in scripts) { + val raw = script.html().lines() + for (line in raw) { + if (line.startsWith("window.__DATA__")) { + val json = JSONObject(line.substringAfter('=').substringBeforeLast(';')) + val list = json.getJSONObject("chapters").getJSONArray("list") + val id = json.optJSONObject("user")?.getLong("id")?.toString() ?: "not" + val total = list.length() + chapters = ChaptersListBuilder(total) + for (i in 0 until total) { + val item = list.getJSONObject(i) + val chapterId = item.getLong("chapter_id") + val scanlator = item.getStringOrNull("username") + val url = buildString { + if (isAuthorized) { + append(manga.url) + append("/v") + append(item.getInt("chapter_volume")) + append("/c") + append(item.getString("chapter_number")) + append("?ui=") + append(id) + } else { + append(manga.url) + append("/v") + append(item.getInt("chapter_volume")) + append("/c") + append(item.getString("chapter_number")) + } + } + val nameChapter = item.getStringOrNull("chapter_name") + val volume = item.getInt("chapter_volume") + val number = item.getString("chapter_number") + val fullNameChapter = "Том $volume. Глава $number" + chapters.add( + MangaChapter( + id = generateUid(chapterId), + url = url, + source = source, + number = total - i, + uploadDate = dateFormat.tryParse( + item.getString("chapter_created_at").substringBefore(" "), + ), + scanlator = scanlator, + branch = null, + name = if (nameChapter.isNullOrBlank()) fullNameChapter else "$fullNameChapter - $nameChapter", + ), + ) + } + chapters.reverse() + break@scripts + } + } + } + return manga.copy( + title = title?.getOrNull(0)?.text()?.takeUnless(String::isBlank) ?: manga.title, + altTitle = title?.getOrNull(1)?.text()?.substringBefore('/')?.trim(), + rating = root.selectFirst("div.media-stats-item__score") + ?.selectFirst("span") + ?.text()?.toFloatOrNull()?.div(5f) ?: manga.rating, + author = info?.getElementsMatchingOwnText("Автор")?.firstOrNull() + ?.nextElementSibling()?.text() ?: manga.author, + tags = info?.selectFirst("div.media-tags") + ?.select("a.media-tag-item")?.mapNotNullToSet { a -> + val href = a.attr("href") + if (href.contains("genres")) { + MangaTag( + title = a.text().toTitleCase(), + key = href.substringAfterLast('='), + source = source, + ) + } else null + } ?: manga.tags, + isNsfw = isNsfw(doc), + description = info?.selectFirst("div.media-description__text")?.html(), + chapters = chapters?.toList(), + ) + } - override suspend fun getPages(chapter: MangaChapter): List { - val fullUrl = chapter.url.toAbsoluteUrl(domain) - val doc = webClient.httpGet(fullUrl).parseHtml() - if (doc.location().substringBefore('?').endsWith("/register")) { - throw AuthRequiredException(source) - } - val scripts = doc.head().select("script") - val pg = (doc.body().getElementById("pg")?.html() ?: doc.parseFailed("Element #pg not found")) - .substringAfter('=') - .substringBeforeLast(';') - val pages = JSONArray(pg) - for (script in scripts) { - val raw = script.html().trim() - if (raw.contains("window.__info")) { - val json = JSONObject( - raw.substringAfter("window.__info") - .substringAfter('=') - .substringBeforeLast(';'), - ) - val domain = json.getJSONObject("servers").run { - getStringOrNull("main") ?: getString( - json.getJSONObject("img").getString("server"), - ) - } - val url = json.getJSONObject("img").getString("url") - return pages.mapJSON { x -> - val pageUrl = "$domain/$url${x.getString("u")}" - MangaPage( - id = generateUid(pageUrl), - url = pageUrl, - preview = null, - referer = fullUrl, - source = source, - ) - } - } - } - throw ParseException("Script with info not found", fullUrl) - } + override suspend fun getPages(chapter: MangaChapter): List { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + if (doc.location().substringBefore('?').endsWith("/register")) { + throw AuthRequiredException(source) + } + val scripts = doc.head().select("script") + val pg = (doc.body().getElementById("pg")?.html() ?: doc.parseFailed("Element #pg not found")) + .substringAfter('=') + .substringBeforeLast(';') + val pages = JSONArray(pg) + for (script in scripts) { + val raw = script.html().trim() + if (raw.contains("window.__info")) { + val json = JSONObject( + raw.substringAfter("window.__info") + .substringAfter('=') + .substringBeforeLast(';'), + ) + val domain = json.getJSONObject("servers").run { + getStringOrNull("main") ?: getString( + json.getJSONObject("img").getString("server"), + ) + } + val url = json.getJSONObject("img").getString("url") + return pages.mapJSON { x -> + val pageUrl = "$domain/$url${x.getString("u")}" + MangaPage( + id = generateUid(pageUrl), + url = pageUrl, + preview = null, + source = source, + ) + } + } + } + throw ParseException("Script with info not found", fullUrl) + } - override suspend fun getTags(): Set { - val url = "https://${domain}/manga-list" - val doc = webClient.httpGet(url).parseHtml() - val scripts = doc.body().select("script") - for (script in scripts) { - val raw = script.html().trim() - if (raw.startsWith("window.__DATA")) { - val json = JSONObject(raw.substringAfter('=').substringBeforeLast(';')) - val genres = json.getJSONObject("filters").getJSONArray("genres") - val result = ArraySet(genres.length()) - for (x in genres.JSONIterator()) { - result += MangaTag( - source = source, - key = x.getInt("id").toString(), - title = x.getString("name").toTitleCase(), - ) - } - return result - } - } - throw ParseException("Script with genres not found", url) - } + override suspend fun getTags(): Set { + val url = "https://${domain}/manga-list" + val doc = webClient.httpGet(url).parseHtml() + val scripts = doc.body().select("script") + for (script in scripts) { + val raw = script.html().trim() + if (raw.startsWith("window.__DATA")) { + val json = JSONObject(raw.substringAfter('=').substringBeforeLast(';')) + val genres = json.getJSONObject("filters").getJSONArray("genres") + val result = ArraySet(genres.length()) + for (x in genres.JSONIterator()) { + result += MangaTag( + source = source, + key = x.getInt("id").toString(), + title = x.getString("name").toTitleCase(), + ) + } + return result + } + } + throw ParseException("Script with genres not found", url) + } - override val isAuthorized: Boolean - get() { - return context.cookieJar.getCookies(domain).any { - it.name.startsWith("remember_web_") - } - } + override val isAuthorized: Boolean + get() { + return context.cookieJar.getCookies(domain).any { + it.name.startsWith("remember_web_") + } + } - override suspend fun getUsername(): String { - val body = webClient.httpGet("https://${LibConst.LIB_SOCIAL_LINK}/messages").parseHtml().body() - if (body.baseUri().endsWith("/login")) { - throw AuthRequiredException(source) - } - return body.selectFirst(".profile-user__username")?.text() ?: body.parseFailed("Cannot find username") - } + override suspend fun getUsername(): String { + val body = webClient.httpGet("https://${LibConst.LIB_SOCIAL_LINK}/messages").parseHtml().body() + if (body.baseUri().endsWith("/login")) { + throw AuthRequiredException(source) + } + return body.selectFirst(".profile-user__username")?.text() ?: body.parseFailed("Cannot find username") + } - protected open fun isNsfw(doc: Document): Boolean { - val sidebar = doc.body().run { - selectFirst(".media-sidebar") ?: selectFirst(".media-info") - } ?: doc.parseFailed("Sidebar not found") - return sidebar.getElementsContainingOwnText("18+").isNotEmpty() - } + protected open fun isNsfw(doc: Document): Boolean { + val sidebar = doc.body().run { + selectFirst(".media-sidebar") ?: selectFirst(".media-info") + } ?: doc.parseFailed("Sidebar not found") + return sidebar.getElementsContainingOwnText("18+").isNotEmpty() + } - private fun getSortKey(sortOrder: SortOrder?) = when (sortOrder) { - SortOrder.RATING -> "desc&sort=rate" - SortOrder.ALPHABETICAL -> "asc&sort=name" - SortOrder.POPULARITY -> "desc&sort=views" - SortOrder.UPDATED -> "desc&sort=last_chapter_at" - SortOrder.NEWEST -> "desc&sort=created_at" - else -> "desc&sort=last_chapter_at" - } + private fun getSortKey(sortOrder: SortOrder?) = when (sortOrder) { + SortOrder.RATING -> "desc&sort=rate" + SortOrder.ALPHABETICAL -> "asc&sort=name" + SortOrder.POPULARITY -> "desc&sort=views" + SortOrder.UPDATED -> "desc&sort=last_chapter_at" + SortOrder.NEWEST -> "desc&sort=created_at" + else -> "desc&sort=last_chapter_at" + } - private suspend fun search(query: String): List { - val domain = domain - val json = webClient.httpGet("https://$domain/search?type=manga&q=$query") - .parseJsonArray() - return json.mapJSON { jo -> - val slug = jo.getString("slug") - val url = "/$slug" - val covers = jo.getJSONObject("covers") - val title = jo.getString("rus_name").ifEmpty { jo.getString("name") } - Manga( - id = generateUid(url), - url = url, - publicUrl = "https://$domain/$slug", - title = title, - altTitle = jo.getString("name").takeUnless { it == title }, - author = null, - tags = emptySet(), - rating = jo.getString("rate_avg").toFloatOrNull()?.div(5f) ?: RATING_UNKNOWN, - state = null, - isNsfw = false, - source = source, - coverUrl = covers.getString("thumbnail").toAbsoluteUrl(domain), - largeCoverUrl = covers.getString("default").toAbsoluteUrl(domain), - ) - } - } + private suspend fun search(query: String): List { + val domain = domain + val json = webClient.httpGet("https://$domain/search?type=manga&q=$query") + .parseJsonArray() + return json.mapJSON { jo -> + val slug = jo.getString("slug") + val url = "/$slug" + val covers = jo.getJSONObject("covers") + val title = jo.getString("rus_name").ifEmpty { jo.getString("name") } + Manga( + id = generateUid(url), + url = url, + publicUrl = "https://$domain/$slug", + title = title, + altTitle = jo.getString("name").takeUnless { it == title }, + author = null, + tags = emptySet(), + rating = jo.getString("rate_avg").toFloatOrNull()?.div(5f) ?: RATING_UNKNOWN, + state = null, + isNsfw = false, + source = source, + coverUrl = covers.getString("thumbnail").toAbsoluteUrl(domain), + largeCoverUrl = covers.getString("default").toAbsoluteUrl(domain), + ) + } + } - @MangaSourceParser("MANGALIB", "MangaLib", "ru") - class Impl(context: MangaLoaderContext) : MangaLibParser(context, MangaSource.MANGALIB) + @MangaSourceParser("MANGALIB", "MangaLib", "ru") + class Impl(context: MangaLoaderContext) : MangaLibParser(context, MangaSource.MANGALIB) - object LibConst { + object LibConst { - val LIB_SOCIAL_LINK = "lib.social" + val LIB_SOCIAL_LINK = "lib.social" - } + } } From 96b9dc8b5c0d2e12debee69a4bd41d7f15d7a679 Mon Sep 17 00:00:00 2001 From: Zakhar Timoshenko Date: Fri, 24 Mar 2023 16:05:24 +0300 Subject: [PATCH 086/121] [ComicK] Updated API --- .../org/koitharu/kotatsu/parsers/site/ComickFunParser.kt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ComickFunParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ComickFunParser.kt index ad3b87b2..aae85a03 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ComickFunParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ComickFunParser.kt @@ -45,7 +45,7 @@ internal class ComickFunParser(context: MangaLoaderContext) : MangaParser(contex val url = buildString { append("https://api.") append(domain) - append("/search?tachiyomi=true") + append("/v1.0/search?tachiyomi=true") if (!query.isNullOrEmpty()) { if (offset > 0) { return emptyList() @@ -118,7 +118,7 @@ internal class ComickFunParser(context: MangaLoaderContext) : MangaParser(contex ) }, author = jo.getJSONArray("artists").optJSONObject(0)?.getString("name"), - chapters = getChapters(comic.getLong("id")), + chapters = getChapters(comic.getString("hid")), ) } @@ -164,9 +164,9 @@ internal class ComickFunParser(context: MangaLoaderContext) : MangaParser(contex return tags } - private suspend fun getChapters(id: Long): List { + private suspend fun getChapters(hid: String): List { val ja = webClient.httpGet( - url = "https://api.${domain}/comic/$id/chapter?tachiyomi=true&limit=$CHAPTERS_LIMIT", + url = "https://api.${domain}/comic/$hid/chapters?limit=$CHAPTERS_LIMIT", ).parseJson().getJSONArray("chapters") val dateFormat = SimpleDateFormat("yyyy-MM-dd") val counters = HashMap() From 481af22ae7b34607a263d3206da1571a2e6ec78b Mon Sep 17 00:00:00 2001 From: Zakhar Timoshenko Date: Fri, 24 Mar 2023 18:02:31 +0300 Subject: [PATCH 087/121] [Remanga] Add mirror --- .../kotlin/org/koitharu/kotatsu/parsers/site/RemangaParser.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/RemangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/RemangaParser.kt index 4ae288d5..062793e1 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/RemangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/RemangaParser.kt @@ -38,7 +38,7 @@ internal class RemangaParser( override val headers get() = getApiHeaders() - override val configKeyDomain = ConfigKey.Domain("remanga.org", null) + override val configKeyDomain = ConfigKey.Domain("remanga.org", arrayOf("remanga.org, реманга.орг")) override val authUrl: String get() = "https://${domain}/user/login" From cc418570d506b948c1242e5cb63842b3884f32af Mon Sep 17 00:00:00 2001 From: Koitharu Date: Sat, 25 Mar 2023 07:32:18 +0200 Subject: [PATCH 088/121] [Remanga] Fix mirror --- .../kotlin/org/koitharu/kotatsu/parsers/site/RemangaParser.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/RemangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/RemangaParser.kt index 062793e1..33a5dbeb 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/RemangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/RemangaParser.kt @@ -38,7 +38,7 @@ internal class RemangaParser( override val headers get() = getApiHeaders() - override val configKeyDomain = ConfigKey.Domain("remanga.org", arrayOf("remanga.org, реманга.орг")) + override val configKeyDomain = ConfigKey.Domain("remanga.org", arrayOf("remanga.org", "реманга.орг")) override val authUrl: String get() = "https://${domain}/user/login" From a3ffecc00f35f2aa2efab66165575ed14272e9f9 Mon Sep 17 00:00:00 2001 From: Koitharu Date: Sat, 25 Mar 2023 10:14:35 +0200 Subject: [PATCH 089/121] [IsekaiScan] Fix chapters parsing --- .../parsers/site/madara/IsekaiScanEuParser.kt | 44 +++++++++++-------- 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/IsekaiScanEuParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/IsekaiScanEuParser.kt index 49b506a6..2a4b629f 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/IsekaiScanEuParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/IsekaiScanEuParser.kt @@ -1,6 +1,7 @@ package org.koitharu.kotatsu.parsers.site.madara import org.jsoup.nodes.Document +import org.jsoup.nodes.Element import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaSourceParser import org.koitharu.kotatsu.parsers.model.Manga @@ -18,6 +19,9 @@ internal class IsekaiScanEuParser(context: MangaLoaderContext) : override val datePattern = "MM/dd/yyyy" override suspend fun getChapters(manga: Manga, doc: Document): List { + doc.selectFirst("ul.version-chap")?.let { + return parseChapters(it) + } val mangaId = doc.body().requireElementById("manga-chapters-holder").attr("data-id") val ul = webClient.httpPost( "https://${domain}/wp-admin/admin-ajax.php", @@ -26,24 +30,7 @@ internal class IsekaiScanEuParser(context: MangaLoaderContext) : "manga" to mangaId, ), ).parseHtml().body().selectFirstOrThrow("ul") - val dateFormat = SimpleDateFormat(datePattern, Locale.US) - return ul.select("li").asReversed().mapChapters { i, li -> - val a = li.selectFirst("a") - val href = a?.attrAsRelativeUrlOrNull("href") ?: li.parseFailed("Link is missing") - MangaChapter( - id = generateUid(href), - name = a.ownText(), - number = i + 1, - url = href, - uploadDate = parseChapterDate( - dateFormat, - li.selectFirst("span.chapter-release-date i")?.text(), - ), - source = source, - scanlator = null, - branch = null, - ) - } + return parseChapters(ul) } override suspend fun getTags(): Set { @@ -70,4 +57,25 @@ internal class IsekaiScanEuParser(context: MangaLoaderContext) : ) } } + + private fun parseChapters(ul: Element): List { + val dateFormat = SimpleDateFormat(datePattern, Locale.US) + return ul.select("li").mapChapters { i, li -> + val a = li.selectFirst("a") + val href = a?.attrAsRelativeUrlOrNull("href") ?: li.parseFailed("Link is missing") + MangaChapter( + id = generateUid(href), + name = a.ownText(), + number = i + 1, + url = href, + uploadDate = parseChapterDate( + dateFormat, + li.selectFirst("span.chapter-release-date i")?.text(), + ), + source = source, + scanlator = null, + branch = null, + ) + } + } } From 35695904d14741cd063fb55d42a754cd0f3070a0 Mon Sep 17 00:00:00 2001 From: Koitharu Date: Thu, 30 Mar 2023 18:52:11 +0300 Subject: [PATCH 090/121] [Remanga] Add Accept header to image requests --- .../parsers/site/grouple/GroupleParser.kt | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt index ca1bccb6..2808c2aa 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt @@ -4,6 +4,7 @@ import kotlinx.coroutines.async import kotlinx.coroutines.coroutineScope import okhttp3.Headers import okhttp3.HttpUrl.Companion.toHttpUrl +import okhttp3.Interceptor import okhttp3.Response import okhttp3.internal.headersContentLength import org.json.JSONArray @@ -24,12 +25,13 @@ private const val PAGE_SIZE_SEARCH = 50 private const val NSFW_ALERT = "сексуальные сцены" private const val NOTHING_FOUND = "Ничего не найдено" private const val MIN_IMAGE_SIZE = 1024L +private const val HEADER_ACCEPT = "Accept" internal abstract class GroupleParser( context: MangaLoaderContext, source: MangaSource, private val siteId: Int, -) : MangaParser(context, source), MangaParserAuthProvider { +) : MangaParser(context, source), MangaParserAuthProvider, Interceptor { @Volatile private var cachedPagesServer: String? = null @@ -282,6 +284,23 @@ internal abstract class GroupleParser( } else res } + override fun intercept(chain: Interceptor.Chain): Response { + val request = chain.request() + if (!request.header(HEADER_ACCEPT).isNullOrEmpty()) { + return chain.proceed(request) + } + val ext = request.url.pathSegments.lastOrNull()?.substringAfterLast('.', "")?.lowercase(Locale.ROOT) + return if (ext == "jpg" || ext == "jpeg" || ext == "png" || ext == "webp") { + chain.proceed( + request.newBuilder() + .header(HEADER_ACCEPT, "image/webp,image/png;q=0.9,image/jpeg,*/*;q=0.8") + .build(), + ) + } else { + chain.proceed(request) + } + } + override fun onCreateConfig(keys: MutableCollection>) { super.onCreateConfig(keys) keys.add(userAgentKey) From 36e88b2c0c055bd39da9c01f11def604796e9006 Mon Sep 17 00:00:00 2001 From: Koitharu Date: Tue, 4 Apr 2023 07:40:47 +0300 Subject: [PATCH 091/121] [Remanga] Fix chapters numbers --- .../kotlin/org/koitharu/kotatsu/parsers/site/RemangaParser.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/RemangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/RemangaParser.kt index 33a5dbeb..1acffda3 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/RemangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/RemangaParser.kt @@ -158,7 +158,7 @@ internal class RemangaParser( MangaChapter( id = generateUid(id), url = "/api/titles/chapters/$id/", - number = chapters.size - i, + number = jo.getIntOrDefault("index", chapters.size - i), name = buildString { append("Том ") append(jo.optString("tome", "0")) From dc0151720e9e9a09680c34f5aea998a18107cc64 Mon Sep 17 00:00:00 2001 From: Koitharu Date: Tue, 4 Apr 2023 07:52:24 +0300 Subject: [PATCH 092/121] [IsekaiScanEu] Fix chapters order #162 --- .../kotatsu/parsers/site/madara/IsekaiScanEuParser.kt | 2 +- .../org/koitharu/kotatsu/parsers/util/Chapters.kt | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/IsekaiScanEuParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/IsekaiScanEuParser.kt index 2a4b629f..9e2a099c 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/IsekaiScanEuParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/IsekaiScanEuParser.kt @@ -60,7 +60,7 @@ internal class IsekaiScanEuParser(context: MangaLoaderContext) : private fun parseChapters(ul: Element): List { val dateFormat = SimpleDateFormat(datePattern, Locale.US) - return ul.select("li").mapChapters { i, li -> + return ul.select("li").mapChapters(reversed = true) { i, li -> val a = li.selectFirst("a") val href = a?.attrAsRelativeUrlOrNull("href") ?: li.parseFailed("Link is missing") MangaChapter( diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Chapters.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Chapters.kt index a9ed843c..f506fbec 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Chapters.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Chapters.kt @@ -4,7 +4,10 @@ import org.koitharu.kotatsu.parsers.InternalParsersApi import org.koitharu.kotatsu.parsers.model.MangaChapter @InternalParsersApi -inline fun Iterable.mapChapters(transform: (index: Int, T) -> MangaChapter?): List { +inline fun Iterable.mapChapters( + reversed: Boolean = false, + transform: (index: Int, T) -> MangaChapter?, +): List { val builder = ChaptersListBuilder(collectionSize()) var index = 0 for (item in this) { @@ -12,6 +15,9 @@ inline fun Iterable.mapChapters(transform: (index: Int, T) -> MangaChapte index++ } } + if (reversed) { + builder.reverse() + } return builder.toList() } @@ -39,4 +45,4 @@ internal class ChaptersListBuilder(initialSize: Int) { } fun toList(): List = list -} \ No newline at end of file +} From 234010099998aa31e44c6a3659d91a4ce8198f74 Mon Sep 17 00:00:00 2001 From: Koitharu Date: Thu, 6 Apr 2023 18:49:19 +0300 Subject: [PATCH 093/121] Fix false nsfw flags on some madara parsers #164 --- .../koitharu/kotatsu/parsers/site/madara/MadaraParser.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt index 4091e616..2dd1da08 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt @@ -390,7 +390,7 @@ internal abstract class MadaraParser( @MangaSourceParser("BAKAMAN", "BakaMan", "th") class BakaMan(context: MangaLoaderContext) : MadaraParser(context, MangaSource.BAKAMAN, "bakaman.net") { - override val isNsfwSource = true + override val isNsfwSource = false } @MangaSourceParser("HENTAI20", "Hentai20", "en") @@ -421,7 +421,7 @@ internal abstract class MadaraParser( @MangaSourceParser("BIBIMANGA", "BibiManga", "en") class BibiManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.BIBIMANGA, "bibimanga.com") { - override val isNsfwSource = true + override val isNsfwSource = false } @MangaSourceParser("TREE_MANGA", "Tree Manga", "en") @@ -439,7 +439,7 @@ internal abstract class MadaraParser( override val tagPrefix = "webtoon-genre/" - override val isNsfwSource = true + override val isNsfwSource = false } @MangaSourceParser("MANGA_MANHUA", "Manga Manhua", "en") From 5e3fddb652dd238b13e900154e7271d0a821d42e Mon Sep 17 00:00:00 2001 From: Koitharu Date: Wed, 12 Apr 2023 18:35:46 +0300 Subject: [PATCH 094/121] [Grouple] Fix relative page urls --- .../kotatsu/parsers/site/grouple/GroupleParser.kt | 11 ++++++----- .../org/koitharu/kotatsu/parsers/util/Parse.kt | 12 +++++++++++- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt index 2808c2aa..4910deea 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt @@ -240,8 +240,8 @@ internal abstract class GroupleParser( val path = parts.last() val servers = parts.dropLast(1).toSet() val cachedServer = cachedPagesServer - if (cachedServer != null && cachedServer in servers && tryHead(cachedServer + path)) { - return cachedServer + path + if (cachedServer != null && cachedServer in servers && tryHead(concatUrl(cachedServer, path))) { + return concatUrl(cachedServer, path) } if (servers.isEmpty()) { throw ParseException("No servers found for page", page.url) @@ -250,7 +250,8 @@ internal abstract class GroupleParser( coroutineScope { servers.map { server -> async { - if (tryHead(server + path)) server else null + val host = server.trim().ifEmpty { "https://$domain/" } + if (tryHead(concatUrl(host, path))) host else null } }.awaitFirst { it != null } }.also { @@ -259,7 +260,7 @@ internal abstract class GroupleParser( } catch (e: NoSuchElementException) { servers.random() } - return checkNotNull(server) + path + return concatUrl(checkNotNull(server), path) } override suspend fun getTags(): Set { @@ -352,7 +353,7 @@ internal abstract class GroupleParser( return webClient.httpPost(url, payload) } - suspend fun tryHead(url: String): Boolean = runCatchingCancellable { + private suspend fun tryHead(url: String): Boolean = runCatchingCancellable { val response = webClient.httpHead(url) response.isSuccessful && response.headersContentLength() >= MIN_IMAGE_SIZE }.getOrDefault(false) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Parse.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Parse.kt index 7c965778..f6c193e1 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Parse.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Parse.kt @@ -67,6 +67,16 @@ fun String.toAbsoluteUrl(domain: String): String = when { else -> this } +fun concatUrl(host: String, path: String): String { + val hostWithSlash = host.endsWith('/') + val pathWithSlash = path.startsWith('/') + return when { + hostWithSlash && pathWithSlash -> host + path.drop(1) + !hostWithSlash && !pathWithSlash -> "$host/$path" + else -> host + path + } +} + fun DateFormat.tryParse(str: String?): Long = if (str.isNullOrEmpty()) { assert(false) { "Date string is null or empty" } 0L @@ -78,4 +88,4 @@ fun DateFormat.tryParse(str: String?): Long = if (str.isNullOrEmpty()) { }.getOrDefault(0L) } -private fun Response.requireBody(): ResponseBody = requireNotNull(body) { "Response body is null" } \ No newline at end of file +private fun Response.requireBody(): ResponseBody = requireNotNull(body) { "Response body is null" } From 129602ea70392748335e4f12604a5e98620ec62d Mon Sep 17 00:00:00 2001 From: Koitharu Date: Sat, 15 Apr 2023 12:44:17 +0300 Subject: [PATCH 095/121] [IsekaiScan] Fix chapters numbers #162 --- .../kotatsu/parsers/site/BatoToParser.kt | 3 +- .../kotatsu/parsers/site/BlogTruyenParser.kt | 2 +- .../kotatsu/parsers/site/Manhwa18Parser.kt | 2 +- .../kotatsu/parsers/site/NetTruyenParser.kt | 2 +- .../kotatsu/parsers/site/NineMangaParser.kt | 2 +- .../parsers/site/TruyentranhLHParser.kt | 2 +- .../kotatsu/parsers/site/UnionMangasParser.kt | 2 +- .../parsers/site/grouple/GroupleParser.kt | 2 +- .../parsers/site/madara/Hentai4Free.kt | 2 +- .../parsers/site/madara/Madara5Parser.kt | 2 +- .../parsers/site/madara/Madara6Parser.kt | 2 +- .../parsers/site/madara/MadaraParser.kt | 2 +- .../parsers/site/madara/MangaDistrict.kt | 2 +- .../kotatsu/parsers/site/madara/NeatManga.kt | 2 +- .../site/mangareader/MangaReaderParser.kt | 2 +- .../parsers/site/multichan/ChanParser.kt | 2 +- .../parsers/site/multichan/YaoiChanParser.kt | 48 +++++++++---------- .../koitharu/kotatsu/parsers/util/Chapters.kt | 8 ++-- 18 files changed, 43 insertions(+), 46 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BatoToParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BatoToParser.kt index 127c455f..e4e1f417 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BatoToParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BatoToParser.kt @@ -103,8 +103,7 @@ internal class BatoToParser(context: MangaLoaderContext) : PagedMangaParser( chapters = root.selectFirst(".episode-list") ?.selectFirst(".main") ?.children() - ?.reversed() - ?.mapChapters { i, div -> + ?.mapChapters(reversed = true) { i, div -> div.parseChapter(i) }.orEmpty(), ) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BlogTruyenParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BlogTruyenParser.kt index 27ce2241..2167845f 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BlogTruyenParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BlogTruyenParser.kt @@ -75,7 +75,7 @@ class BlogTruyenParser(context: MangaLoaderContext) : private fun parseChapterList(doc: Document): List { val chapterList = doc.select("#list-chapters > p") - return chapterList.asReversed().mapChapters { index, element -> + return chapterList.mapChapters(reversed = true) { index, element -> val titleElement = element.selectFirst("span.title > a") ?: return@mapChapters null val name = titleElement.text() val relativeUrl = titleElement.attrAsRelativeUrl("href") diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/Manhwa18Parser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/Manhwa18Parser.kt index 93d1e214..51673a03 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/Manhwa18Parser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/Manhwa18Parser.kt @@ -56,7 +56,7 @@ class Manhwa18Parser(context: MangaLoaderContext) : description = docs.selectFirst(".series-summary .summary-content")?.html(), tags = tags.orEmpty(), state = state, - chapters = docs.select(".card-body > .list-chapters > a").asReversed().mapChapters { index, element -> + chapters = docs.select(".card-body > .list-chapters > a").mapChapters(reversed = true) { index, element -> // attrAsRelativeUrl only return page url without the '/' val chapterUrl = element.attrAsAbsoluteUrlOrNull("href")?.toRelativeUrl(domain) ?: return@mapChapters null diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NetTruyenParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NetTruyenParser.kt index 6b89233e..85aa6beb 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NetTruyenParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NetTruyenParser.kt @@ -35,7 +35,7 @@ class NetTruyenParser(context: MangaLoaderContext) : ?.toFloatOrNull() ?: 0f val chapterElements = doc.getElementById("nt_listchapter")?.select("ul > li") ?: doc.parseFailed() - val chapters = chapterElements.asReversed().mapChapters { index, element -> + val chapters = chapterElements.mapChapters(reversed = true) { index, element -> val a = element.selectFirst("div.chapter > a") ?: return@mapChapters null val relativeUrl = a.attrAsRelativeUrlOrNull("href") ?: return@mapChapters null val timeText = element.selectFirst("div.col-xs-4.text-center.no-wrap.small")?.text() diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NineMangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NineMangaParser.kt index f3b2f6b8..12c40dc3 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NineMangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NineMangaParser.kt @@ -110,7 +110,7 @@ internal abstract class NineMangaParser( description = infoRoot.getElementsByAttributeValue("itemprop", "description").first() ?.html()?.substringAfter(""), chapters = root.selectFirst("div.chapterbox")?.select("ul.sub_vol_ul > li") - ?.asReversed()?.mapChapters { i, li -> + ?.mapChapters(reversed = true) { i, li -> val a = li.selectFirst("a.chapter_list_a") val href = a?.attrAsRelativeUrlOrNull("href") ?.replace("%20", " ") ?: li.parseFailed("Link not found") diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/TruyentranhLHParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/TruyentranhLHParser.kt index 9c97582a..f3d28e9b 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/TruyentranhLHParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/TruyentranhLHParser.kt @@ -53,7 +53,7 @@ class TruyentranhLHParser(context: MangaLoaderContext) : state = state, rating = rating ?: RATING_UNKNOWN, description = infoHeaderEl?.selectFirst(".series-summary .summary-content")?.html(), - chapters = docs.select("ul.list-chapters.at-series > a").reversed().mapChapters { index, element -> + chapters = docs.select("ul.list-chapters.at-series > a").mapChapters(reversed = true) { index, element -> MangaChapter( id = generateUid(element.attrAsRelativeUrl("href")), name = element.selectFirst(".chapter-name")?.text()?.trim().orEmpty(), diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/UnionMangasParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/UnionMangasParser.kt index 491923a4..b71af6f3 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/UnionMangasParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/UnionMangasParser.kt @@ -90,7 +90,7 @@ class UnionMangasParser(context: MangaLoaderContext) : PagedMangaParser(context, it.toMangaTag() } ?: manga.tags, isNsfw = root.selectFirst(".alert-danger")?.html()?.contains("18 anos") == true, - chapters = root.select("div.row.capitulos").asReversed().mapChapters { i, div -> + chapters = root.select("div.row.capitulos").mapChapters(reversed = true) { i, div -> val a = div.selectFirstOrThrow("a") val href = a.attrAsRelativeUrl("href") val title = a.text() diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt index 4910deea..81c81b4a 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt @@ -174,7 +174,7 @@ internal abstract class GroupleParser( author = root.selectFirst("a.person-link")?.text() ?: manga.author, isNsfw = root.select(".alert-warning").any { it.ownText().contains(NSFW_ALERT) }, chapters = root.selectFirst("div.chapters-link")?.selectFirst("table") - ?.select("tr:has(td > a)")?.asReversed()?.mapChapters { i, tr -> + ?.select("tr:has(td > a)")?.mapChapters(reversed = true) { i, tr -> val a = tr.selectFirst("a.chapter-link") ?: return@mapChapters null val href = a.attrAsRelativeUrl("href") var translators = "" diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Hentai4Free.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Hentai4Free.kt index bec398fb..45b0de08 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Hentai4Free.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Hentai4Free.kt @@ -48,7 +48,7 @@ internal class Hentai4Free(context: MangaLoaderContext) : ).parseHtml() val ul = doc2.body().selectFirstOrThrow("ul") val dateFormat = SimpleDateFormat(datePattern, Locale.US) - return ul.select("li").asReversed().mapChapters { i, li -> + return ul.select("li").mapChapters(reversed = true) { i, li -> val a = li.selectFirst("a") val href = a?.attrAsRelativeUrlOrNull("href") ?: li.parseFailed("Link is missing") MangaChapter( diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Madara5Parser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Madara5Parser.kt index 5c699324..6b7f23d6 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Madara5Parser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Madara5Parser.kt @@ -127,7 +127,7 @@ abstract class Madara5Parser @InternalParsersApi constructor( private suspend fun loadChapters(mangaId: Long): List { val dateFormat = SimpleDateFormat(datePattern, sourceLocale) val doc = webClient.httpGet("https://${domain}/ajax-list-chapter?mangaID=$mangaId").parseHtml() - return doc.select("li.wp-manga-chapter").asReversed().mapChapters { i, li -> + return doc.select("li.wp-manga-chapter").mapChapters(reversed = true) { i, li -> val a = li.selectFirstOrThrow("a") val href = a.attrAsRelativeUrl("href") MangaChapter( diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Madara6Parser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Madara6Parser.kt index a476644b..3eb178fd 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Madara6Parser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Madara6Parser.kt @@ -47,7 +47,7 @@ internal abstract class Madara6Parser( val url = mangaUrl.toAbsoluteUrl(domain).removeSuffix('/') + "/ajax/chapters/" val dateFormat = SimpleDateFormat(datePattern, sourceLocale) val doc = webClient.httpPost(url, emptyMap()).parseHtml() - return doc.select("li.wp-manga-chapter").asReversed().mapChapters { i, li -> + return doc.select("li.wp-manga-chapter").mapChapters(reversed = true) { i, li -> val a = li.selectFirstOrThrow("a") val href = a.attrAsRelativeUrl("href") MangaChapter( diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt index 2dd1da08..9d75fba8 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt @@ -154,7 +154,7 @@ internal abstract class MadaraParser( val root2 = doc.body().selectFirstOrThrow("div.content-area") .selectFirstOrThrow("div.c-page") val dateFormat = SimpleDateFormat(datePattern, sourceLocale) - return root2.select("li").asReversed().mapChapters { i, li -> + return root2.select("li").mapChapters(reversed = true) { i, li -> val a = li.selectFirst("a") val href = a?.attrAsRelativeUrlOrNull("href") ?: li.parseFailed("Link is missing") MangaChapter( diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangaDistrict.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangaDistrict.kt index 75439adc..6a97a0d7 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangaDistrict.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangaDistrict.kt @@ -28,7 +28,7 @@ internal class MangaDistrict(context: MangaLoaderContext) : ).parseHtml() val ul = doc2.body().selectFirstOrThrow("ul") val dateFormat = SimpleDateFormat(datePattern, Locale.US) - return ul.select("li").asReversed().mapChapters { i, li -> + return ul.select("li").mapChapters(reversed = true) { i, li -> val a = li.selectFirst("a") val href = a?.attrAsRelativeUrlOrNull("href") ?: li.parseFailed("Link is missing") MangaChapter( diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/NeatManga.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/NeatManga.kt index 691938fc..ca4f5cda 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/NeatManga.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/NeatManga.kt @@ -53,7 +53,7 @@ internal class NeatManga(context: MangaLoaderContext) : MadaraParser(context, Ma ).parseHtml() val ul = doc2.body().selectFirstOrThrow("ul") val dateFormat = SimpleDateFormat(datePattern, Locale.US) - return ul.select("li").asReversed().mapChapters { i, li -> + return ul.select("li").mapChapters(reversed = true) { i, li -> val a = li.selectFirst("a") val href = a?.attrAsRelativeUrlOrNull("href") ?: li.parseFailed("Link is missing") MangaChapter( diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt index 6199f401..376d232e 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt @@ -38,7 +38,7 @@ internal abstract class MangaReaderParser( override suspend fun getDetails(manga: Manga): Manga { val docs = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - val chapters = docs.select("#chapterlist > ul > li").reversed().mapChapters { index, element -> + val chapters = docs.select("#chapterlist > ul > li").mapChapters(reversed = true) { index, element -> val url = element.selectFirst("a")?.attrAsRelativeUrl("href") ?: return@mapChapters null MangaChapter( id = generateUid(url), diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/multichan/ChanParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/multichan/ChanParser.kt index 082cfb5b..d89a4a87 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/multichan/ChanParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/multichan/ChanParser.kt @@ -92,7 +92,7 @@ internal abstract class ChanParser( return manga.copy( description = root.getElementById("description")?.html()?.substringBeforeLast(" + chapters = root.select("table.table_cha tr:gt(1)").mapChapters(reversed = true) { i, tr -> val href = tr?.selectFirst("a")?.attrAsRelativeUrlOrNull("href") ?: return@mapChapters null MangaChapter( diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/multichan/YaoiChanParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/multichan/YaoiChanParser.kt index 52b537ba..6e680678 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/multichan/YaoiChanParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/multichan/YaoiChanParser.kt @@ -11,29 +11,29 @@ import org.koitharu.kotatsu.parsers.util.* @MangaSourceParser("YAOICHAN", "Яой-тян", "ru") internal class YaoiChanParser(context: MangaLoaderContext) : ChanParser(context, MangaSource.YAOICHAN) { - override val configKeyDomain = ConfigKey.Domain("yaoi-chan.me", null) + override val configKeyDomain = ConfigKey.Domain("yaoi-chan.me", null) - override suspend fun getDetails(manga: Manga): Manga { - val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - val root = doc.body().requireElementById("dle-content") - return manga.copy( - description = root.getElementById("description")?.html()?.substringBeforeLast(" - table.select("div.manga") - }.mapNotNull { it.selectFirst("a") }.reversed().mapChapters { i, a -> - val href = a.attrAsRelativeUrl("href") - MangaChapter( - id = generateUid(href), - name = a.text().trim(), - number = i + 1, - url = href, - uploadDate = 0L, - source = source, - scanlator = null, - branch = null, - ) - }, - ) - } + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val root = doc.body().requireElementById("dle-content") + return manga.copy( + description = root.getElementById("description")?.html()?.substringBeforeLast(" + table.select("div.manga") + }.mapNotNull { it.selectFirst("a") }.mapChapters(reversed = true) { i, a -> + val href = a.attrAsRelativeUrl("href") + MangaChapter( + id = generateUid(href), + name = a.text().trim(), + number = i + 1, + url = href, + uploadDate = 0L, + source = source, + scanlator = null, + branch = null, + ) + }, + ) + } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Chapters.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Chapters.kt index f506fbec..289080ec 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Chapters.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Chapters.kt @@ -4,20 +4,18 @@ import org.koitharu.kotatsu.parsers.InternalParsersApi import org.koitharu.kotatsu.parsers.model.MangaChapter @InternalParsersApi -inline fun Iterable.mapChapters( +inline fun List.mapChapters( reversed: Boolean = false, transform: (index: Int, T) -> MangaChapter?, ): List { val builder = ChaptersListBuilder(collectionSize()) var index = 0 - for (item in this) { + val elements = if (reversed) this.asReversed() else this + for (item in elements) { if (builder.add(transform(index, item))) { index++ } } - if (reversed) { - builder.reverse() - } return builder.toList() } From ece7d71e09fcb53ba3c275d5465cadf394f2dcea Mon Sep 17 00:00:00 2001 From: Koitharu Date: Sat, 15 Apr 2023 12:51:29 +0300 Subject: [PATCH 096/121] Update domains --- .../kotatsu/parsers/site/NetTruyenParser.kt | 5 +- .../parsers/site/madara/MadaraParser.kt | 2 +- .../site/madara/MangasOriginesParser.kt | 65 +++++++++---------- .../site/mangareader/MangaReaderParser.kt | 6 +- 4 files changed, 40 insertions(+), 38 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NetTruyenParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NetTruyenParser.kt index 85aa6beb..a994989f 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NetTruyenParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NetTruyenParser.kt @@ -19,7 +19,10 @@ class NetTruyenParser(context: MangaLoaderContext) : PagedMangaParser(context, MangaSource.NETTRUYEN, pageSize = 36) { override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("nettruyenin.com", null) + get() = ConfigKey.Domain( + "www.nettruyento.com", + arrayOf("www.nettruyento.com", "nettruyento.com", "nettruyenin.com"), + ) override val sortOrders: Set get() = EnumSet.of(SortOrder.UPDATED, SortOrder.POPULARITY, SortOrder.NEWEST, SortOrder.RATING) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt index 9d75fba8..c9b8c72c 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt @@ -316,7 +316,7 @@ internal abstract class MadaraParser( } @MangaSourceParser("HACHIMANGA", "HachiManga", "ja") - class HachiManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.HACHIMANGA, "hachimanga.com") { + class HachiManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.HACHIMANGA, "hachiraw.com") { override val sourceLocale: Locale = Locale.ENGLISH } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangasOriginesParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangasOriginesParser.kt index f2cf3cab..6d5e0268 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangasOriginesParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangasOriginesParser.kt @@ -2,7 +2,6 @@ package org.koitharu.kotatsu.parsers.site.madara import org.jsoup.nodes.Element import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser import org.koitharu.kotatsu.parsers.model.Manga import org.koitharu.kotatsu.parsers.model.MangaChapter import org.koitharu.kotatsu.parsers.model.MangaSource @@ -12,41 +11,41 @@ import org.koitharu.kotatsu.parsers.util.mapToSet import org.koitharu.kotatsu.parsers.util.selectFirstOrThrow import java.util.* -@MangaSourceParser("MANGAS_ORIGINES", "Mangas Origines", "fr") +//@MangaSourceParser("MANGAS_ORIGINES", "Mangas Origines", "fr") TODO: check internal class MangasOriginesParser(context: MangaLoaderContext) : - Madara6Parser(context, MangaSource.MANGAS_ORIGINES, "mangas-origines.fr") { + Madara6Parser(context, MangaSource.MANGAS_ORIGINES, "mangas-origines.fr") { - override val tagPrefix = "catalogues-genre/" + override val tagPrefix = "catalogues-genre/" - override fun parseDetails(manga: Manga, body: Element, chapters: List): Manga { - val root = body.selectFirstOrThrow(".site-content") - val postContent = root.selectFirstOrThrow(".post-content") - val tags = postContent.getElementsContainingOwnText("Genre") - .firstOrNull()?.tableValue() - ?.getElementsByAttributeValueContaining("href", tagPrefix) - ?.mapToSet { a -> a.asMangaTag() } ?: manga.tags - return manga.copy( - largeCoverUrl = root.selectFirst("picture") - ?.selectFirst("img[data-src]") - ?.attrAsAbsoluteUrlOrNull("data-src"), - description = (root.selectFirst(".detail-content") - ?: root.selectFirstOrThrow(".manga-excerpt")).html(), - author = postContent.getElementsContainingOwnText("Auteur") - .firstOrNull()?.tableValue()?.text()?.trim(), - state = postContent.getElementsContainingOwnText("STATUS") - .firstOrNull()?.tableValue()?.text()?.asMangaState(), - tags = tags, - isNsfw = body.hasClass("adult-content"), - chapters = chapters, - ) - } + override fun parseDetails(manga: Manga, body: Element, chapters: List): Manga { + val root = body.selectFirstOrThrow(".site-content") + val postContent = root.selectFirstOrThrow(".post-content") + val tags = postContent.getElementsContainingOwnText("Genre") + .firstOrNull()?.tableValue() + ?.getElementsByAttributeValueContaining("href", tagPrefix) + ?.mapToSet { a -> a.asMangaTag() } ?: manga.tags + return manga.copy( + largeCoverUrl = root.selectFirst("picture") + ?.selectFirst("img[data-src]") + ?.attrAsAbsoluteUrlOrNull("data-src"), + description = (root.selectFirst(".detail-content") + ?: root.selectFirstOrThrow(".manga-excerpt")).html(), + author = postContent.getElementsContainingOwnText("Auteur") + .firstOrNull()?.tableValue()?.text()?.trim(), + state = postContent.getElementsContainingOwnText("STATUS") + .firstOrNull()?.tableValue()?.text()?.asMangaState(), + tags = tags, + isNsfw = body.hasClass("adult-content"), + chapters = chapters, + ) + } - override fun String.asMangaState() = when (trim().lowercase(Locale.FRANCE)) { - "en cours" -> MangaState.ONGOING - "abandonné", - "terminé", - -> MangaState.FINISHED + override fun String.asMangaState() = when (trim().lowercase(Locale.FRANCE)) { + "en cours" -> MangaState.ONGOING + "abandonné", + "terminé", + -> MangaState.FINISHED - else -> null - } + else -> null + } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt index 376d232e..0a7015f5 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt @@ -231,7 +231,7 @@ internal abstract class MangaReaderParser( class ManhwaLandParser(context: MangaLoaderContext) : MangaReaderParser(context, MangaSource.MANHWALAND, pageSize = 20, searchPageSize = 10) { override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("manhwaland.guru", null) + get() = ConfigKey.Domain("manhwaland.us", arrayOf("manhwaland.us", "manhwaland.guru")) override val listUrl: String get() = "/series" @@ -353,7 +353,7 @@ internal abstract class MangaReaderParser( class ManhwadesuParser(context: MangaLoaderContext) : MangaReaderParser(context, MangaSource.MANHWADESU, pageSize = 20, searchPageSize = 10) { override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("manhwadesu.org", null) + get() = ConfigKey.Domain("manhwadesu.pro", arrayOf("manhwadesu.pro", "manhwadesu.org")) override val listUrl: String get() = "/komik" override val tableMode: Boolean get() = false @@ -593,7 +593,7 @@ internal abstract class MangaReaderParser( class KomikDewasaParser(context: MangaLoaderContext) : MangaReaderParser(context, MangaSource.KOMIKDEWASA, pageSize = 20, searchPageSize = 10) { override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("komikdewasa.info", null) + get() = ConfigKey.Domain("komikdewasa.us", arrayOf("komikdewasa.us", "komikdewasa.info")) override val listUrl: String get() = "/manga" From 2db1a8d95866cb738994b6a1253415ccbce8e8e8 Mon Sep 17 00:00:00 2001 From: Koitharu Date: Sat, 15 Apr 2023 12:59:29 +0300 Subject: [PATCH 097/121] Remove broken parser --- .../site/madara/MangasOriginesParser.kt | 51 ------------------- 1 file changed, 51 deletions(-) delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangasOriginesParser.kt diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangasOriginesParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangasOriginesParser.kt deleted file mode 100644 index 6d5e0268..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangasOriginesParser.kt +++ /dev/null @@ -1,51 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.madara - -import org.jsoup.nodes.Element -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.model.Manga -import org.koitharu.kotatsu.parsers.model.MangaChapter -import org.koitharu.kotatsu.parsers.model.MangaSource -import org.koitharu.kotatsu.parsers.model.MangaState -import org.koitharu.kotatsu.parsers.util.attrAsAbsoluteUrlOrNull -import org.koitharu.kotatsu.parsers.util.mapToSet -import org.koitharu.kotatsu.parsers.util.selectFirstOrThrow -import java.util.* - -//@MangaSourceParser("MANGAS_ORIGINES", "Mangas Origines", "fr") TODO: check -internal class MangasOriginesParser(context: MangaLoaderContext) : - Madara6Parser(context, MangaSource.MANGAS_ORIGINES, "mangas-origines.fr") { - - override val tagPrefix = "catalogues-genre/" - - override fun parseDetails(manga: Manga, body: Element, chapters: List): Manga { - val root = body.selectFirstOrThrow(".site-content") - val postContent = root.selectFirstOrThrow(".post-content") - val tags = postContent.getElementsContainingOwnText("Genre") - .firstOrNull()?.tableValue() - ?.getElementsByAttributeValueContaining("href", tagPrefix) - ?.mapToSet { a -> a.asMangaTag() } ?: manga.tags - return manga.copy( - largeCoverUrl = root.selectFirst("picture") - ?.selectFirst("img[data-src]") - ?.attrAsAbsoluteUrlOrNull("data-src"), - description = (root.selectFirst(".detail-content") - ?: root.selectFirstOrThrow(".manga-excerpt")).html(), - author = postContent.getElementsContainingOwnText("Auteur") - .firstOrNull()?.tableValue()?.text()?.trim(), - state = postContent.getElementsContainingOwnText("STATUS") - .firstOrNull()?.tableValue()?.text()?.asMangaState(), - tags = tags, - isNsfw = body.hasClass("adult-content"), - chapters = chapters, - ) - } - - override fun String.asMangaState() = when (trim().lowercase(Locale.FRANCE)) { - "en cours" -> MangaState.ONGOING - "abandonné", - "terminé", - -> MangaState.FINISHED - - else -> null - } -} From 1b269efaa46fe50911f8f9cac0b87c56fabe6fc0 Mon Sep 17 00:00:00 2001 From: Koitharu Date: Sat, 15 Apr 2023 13:02:26 +0300 Subject: [PATCH 098/121] Update readme --- CONTRIBUTING.md | 3 ++- README.md | 12 +++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ab56094d..4ba46f33 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -87,4 +87,5 @@ It is recommended to run unit tests before submitting a PR. ## Help -If you need a help or have some questions, ask a community in our [Discord server](https://discord.gg/NNJ5RgVBC5). +If you need a help or have some questions, ask a community in our [Telegram chat](https://t.me/kotatsuapp) +or [Discord server](https://discord.gg/NNJ5RgVBC5). diff --git a/README.md b/README.md index 0debf1c8..4612c1a9 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ This library provides manga sources. -[![](https://jitpack.io/v/KotatsuApp/kotatsu-parsers.svg)](https://jitpack.io/#KotatsuApp/kotatsu-parsers) ![Kotlin](https://img.shields.io/github/languages/top/KotatsuApp/kotatsu-parsers) ![License](https://img.shields.io/github/license/KotatsuApp/Kotatsu) [![Discord](https://img.shields.io/discord/898363402467045416?color=5865f2&label=discord)](https://discord.gg/NNJ5RgVBC5) +[![](https://jitpack.io/v/KotatsuApp/kotatsu-parsers.svg)](https://jitpack.io/#KotatsuApp/kotatsu-parsers) ![Kotlin](https://img.shields.io/github/languages/top/KotatsuApp/kotatsu-parsers) ![License](https://img.shields.io/github/license/KotatsuApp/Kotatsu) [![Telegram](https://img.shields.io/badge/chat-telegram-60ACFF)](https://t.me/kotatsuapp) [![Discord](https://img.shields.io/discord/898363402467045416?color=5865f2&label=discord)](https://discord.gg/NNJ5RgVBC5) ## Usage @@ -44,9 +44,10 @@ This library provides manga sources. ``` `mangaLoaderContext` is an implementation of the `MangaLoaderContext` class. - See examples of [Android](https://github.com/KotatsuApp/Kotatsu/blob/devel/app/src/main/java/org/koitharu/kotatsu/core/parser/MangaLoaderContextImpl.kt) - and [Non-Android](https://github.com/KotatsuApp/kotatsu-dl/blob/master/src/main/kotlin/org/koitharu/kotatsu_dl/env/MangaLoaderContextImpl.kt) - implementation. + See examples + of [Android](https://github.com/KotatsuApp/Kotatsu/blob/devel/app/src/main/java/org/koitharu/kotatsu/core/parser/MangaLoaderContextImpl.kt) + and [Non-Android](https://github.com/KotatsuApp/kotatsu-dl/blob/master/src/main/kotlin/org/koitharu/kotatsu_dl/env/MangaLoaderContextImpl.kt) + implementation. Note that the `MangaSource.LOCAL` and `MangaSource.DUMMY` parsers cannot be instantiated. @@ -56,4 +57,5 @@ See [CONTRIBUTING.md](./CONTRIBUTING.md) for the guidelines. ## DMCA disclaimer -The developers of this application have no affiliation with the content available in the app. It is collected from sources freely available through any web browser. +The developers of this application have no affiliation with the content available in the app. It is collected from +sources freely available through any web browser. From 749f682ef911f94a6ee9ab4dd5ca356f0adb9f48 Mon Sep 17 00:00:00 2001 From: Koitharu Date: Sat, 15 Apr 2023 13:22:33 +0300 Subject: [PATCH 099/121] [Madara] Fix empty tags --- .../koitharu/kotatsu/parsers/site/madara/MadaraParser.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt index c9b8c72c..3adfa0a2 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt @@ -72,10 +72,10 @@ internal abstract class MadaraParser( altTitle = null, rating = div.selectFirst("span.total_votes")?.ownText() ?.toFloatOrNull()?.div(5f) ?: -1f, - tags = summary?.selectFirst(".mg_genres")?.select("a")?.mapToSet { a -> + tags = summary?.selectFirst(".mg_genres")?.select("a")?.mapNotNullToSet { a -> MangaTag( - key = a.attr("href").removeSuffix("/").substringAfterLast('/'), - title = a.text().toTitleCase(), + key = a.attr("href").removeSuffix('/').substringAfterLast('/'), + title = a.text().ifEmpty { return@mapNotNullToSet null }.toTitleCase(), source = source, ) }.orEmpty(), From b49aee318840ae04e3e71695e6102ea5ee44179c Mon Sep 17 00:00:00 2001 From: vianh Date: Sun, 16 Apr 2023 22:52:22 +0700 Subject: [PATCH 100/121] [DoujinDesu] New parser --- .../kotatsu/parsers/site/DoujinDesuParser.kt | 134 ++++++++++++++++++ 1 file changed, 134 insertions(+) create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/DoujinDesuParser.kt diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/DoujinDesuParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/DoujinDesuParser.kt new file mode 100644 index 00000000..04746949 --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/DoujinDesuParser.kt @@ -0,0 +1,134 @@ +package org.koitharu.kotatsu.parsers.site + +import org.koitharu.kotatsu.parsers.InternalParsersApi +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.PagedMangaParser +import org.koitharu.kotatsu.parsers.config.ConfigKey +import org.koitharu.kotatsu.parsers.model.* +import org.koitharu.kotatsu.parsers.util.* +import java.text.SimpleDateFormat +import java.util.* + +@MangaSourceParser("DOUJINDESU", "DoujinDesu", "id") +class DoujinDesuParser(context: MangaLoaderContext) : PagedMangaParser(context, MangaSource.DOUJINDESU, pageSize = 18) { + @InternalParsersApi + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("212.32.226.234", null) + + override val sortOrders: Set + get() = EnumSet.of(SortOrder.UPDATED, SortOrder.NEWEST, SortOrder.ALPHABETICAL, SortOrder.POPULARITY) + + override suspend fun getDetails(manga: Manga): Manga { + val docs = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml().selectFirstOrThrow("#archive") + val metadataEl = docs.selectFirst(".wrapper > .metadata tbody") + val state = when (metadataEl?.selectFirst("tr:contains(Status)")?.selectLast("td")?.text()) { + "Finished" -> MangaState.FINISHED + "Publishing" -> MangaState.ONGOING + else -> null + } + return manga.copy( + author = metadataEl?.selectFirst("tr:contains(Author)")?.selectLast("td")?.text(), + description = docs.selectFirst(".wrapper > .metadata > .pb-2")?.selectFirst("p")?.html(), + state = state, + rating = metadataEl?.selectFirst(".rating-prc")?.ownText()?.toFloatOrNull()?.div(10f) ?: RATING_UNKNOWN, + tags = docs.select(".tags > a").mapToSet { + MangaTag( + key = it.attr("title"), + title = it.text(), + source = source, + ) + }, + chapters = docs.select("#chapter_list > ul > li").mapChapters(reversed = true) { index, element -> + val titleTag = element.selectFirstOrThrow(".epsleft > .lchx > a") + val dateFormat = SimpleDateFormat("EEEE, dd MMMM yyyy", Locale("in")) + MangaChapter( + id = generateUid(titleTag.attrAsRelativeUrl("href")), + name = titleTag.text(), + number = index + 1, + url = titleTag.attrAsRelativeUrl("href"), + scanlator = null, + uploadDate = dateFormat.tryParse(element.select(".epsleft > .date").text()), + branch = null, + source = source, + ) + }, + ) + } + + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder + ): List { + val url = urlBuilder().apply { + addPathSegment("manga") + addPathSegment("page") + addPathSegment("$page/") + val order = when (sortOrder) { + SortOrder.UPDATED -> "update" + SortOrder.POPULARITY -> "popular" + SortOrder.ALPHABETICAL -> "title" + SortOrder.NEWEST -> "latest" + else -> throw IllegalArgumentException("Sort order not supported") + } + addQueryParameter("order", order) + addQueryParameter("title", query.orEmpty()) + tags?.forEach { + addEncodedQueryParameter("genre[]".urlEncoded(), it.key.urlEncoded()) + } + }.build() + + return webClient.httpGet(url).parseHtml().select("#archives > div.entries > .entry") + .map { + val titleTag = it.selectFirstOrThrow(".metadata > a") + val relativeUrl = titleTag.attrAsRelativeUrl("href") + Manga( + id = generateUid(relativeUrl), + title = titleTag.attr("title"), + altTitle = null, + url = relativeUrl, + publicUrl = relativeUrl.toAbsoluteUrl(domain), + rating = RATING_UNKNOWN, + isNsfw = true, + coverUrl = it.selectFirst(".thumbnail > img")?.attrAsAbsoluteUrl("src").orEmpty(), + tags = emptySet(), + state = null, + author = null, + largeCoverUrl = null, + description = null, + source = source, + ) + } + } + + override suspend fun getPages(chapter: MangaChapter): List { + val id = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + .selectFirstOrThrow("#reader") + .attr("data-id") + return webClient.httpPost("/themes/ajax/ch.php".toAbsoluteUrl(domain), "id=$id").parseHtml() + .select("img") + .map { + val url = it.attrAsRelativeUrl("src") + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } + + override suspend fun getTags(): Set { + return webClient.httpGet("/genre/".toAbsoluteUrl(domain)).parseHtml() + .select("#taxonomy .entries > .entry > a") + .mapToSet { + MangaTag( + key = it.attr("title"), + title = it.attr("title"), + source = source, + ) + } + } +} From 1b6d1456f37a7b56ebd9fae7f85caa86c797917f Mon Sep 17 00:00:00 2001 From: Koitharu Date: Mon, 17 Apr 2023 08:35:58 +0300 Subject: [PATCH 101/121] Add support for POST with json body --- .../parsers/network/OkHttpWebClient.kt | 170 +++++++++--------- .../kotatsu/parsers/network/WebClient.kt | 114 ++++++------ 2 files changed, 150 insertions(+), 134 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/network/OkHttpWebClient.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/network/OkHttpWebClient.kt index 7d61ba1e..0bfe7ef3 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/network/OkHttpWebClient.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/network/OkHttpWebClient.kt @@ -12,96 +12,98 @@ import org.koitharu.kotatsu.parsers.util.await import org.koitharu.kotatsu.parsers.util.parseJson class OkHttpWebClient( - private val httpClient: OkHttpClient, - private val mangaSource: MangaSource, + private val httpClient: OkHttpClient, + private val mangaSource: MangaSource, ) : WebClient { - override suspend fun httpGet(url: HttpUrl): Response { - val request = Request.Builder() - .get() - .url(url) - .addTags() - return httpClient.newCall(request.build()).await().ensureSuccess() - } + override suspend fun httpGet(url: HttpUrl): Response { + val request = Request.Builder() + .get() + .url(url) + .addTags() + return httpClient.newCall(request.build()).await().ensureSuccess() + } - override suspend fun httpHead(url: HttpUrl): Response { - val request = Request.Builder() - .head() - .url(url) - .addTags() - return httpClient.newCall(request.build()).await().ensureSuccess() - } + override suspend fun httpHead(url: HttpUrl): Response { + val request = Request.Builder() + .head() + .url(url) + .addTags() + return httpClient.newCall(request.build()).await().ensureSuccess() + } - override suspend fun httpPost(url: HttpUrl, form: Map): Response { - val body = FormBody.Builder() - form.forEach { (k, v) -> - body.addEncoded(k, v) - } - val request = Request.Builder() - .post(body.build()) - .url(url) - .addTags() - return httpClient.newCall(request.build()).await().ensureSuccess() - } + override suspend fun httpPost(url: HttpUrl, form: Map): Response { + val body = FormBody.Builder() + form.forEach { (k, v) -> + body.addEncoded(k, v) + } + val request = Request.Builder() + .post(body.build()) + .url(url) + .addTags() + return httpClient.newCall(request.build()).await().ensureSuccess() + } - override suspend fun httpPost(url: HttpUrl, payload: String): Response { - val body = FormBody.Builder() - payload.split('&').forEach { - val pos = it.indexOf('=') - if (pos != -1) { - val k = it.substring(0, pos) - val v = it.substring(pos + 1) - body.addEncoded(k, v) - } - } - val request = Request.Builder() - .post(body.build()) - .url(url) - .addTags() - return httpClient.newCall(request.build()).await().ensureSuccess() - } + override suspend fun httpPost(url: HttpUrl, payload: String): Response { + val body = FormBody.Builder() + payload.split('&').forEach { + val pos = it.indexOf('=') + if (pos != -1) { + val k = it.substring(0, pos) + val v = it.substring(pos + 1) + body.addEncoded(k, v) + } + } + val request = Request.Builder() + .post(body.build()) + .url(url) + .addTags() + return httpClient.newCall(request.build()).await().ensureSuccess() + } - override suspend fun graphQLQuery(endpoint: String, query: String): JSONObject { - val body = JSONObject() - body.put("operationName", null as Any?) - body.put("variables", JSONObject()) - body.put("query", "{$query}") - val mediaType = "application/json; charset=utf-8".toMediaType() - val requestBody = body.toString().toRequestBody(mediaType) - val request = Request.Builder() - .post(requestBody) - .url(endpoint) - .addTags() - val json = httpClient.newCall(request.build()).await().ensureSuccess().parseJson() - json.optJSONArray("errors")?.let { - if (it.length() != 0) { - throw GraphQLException(it) - } - } - return json - } + override suspend fun httpPost(url: HttpUrl, body: JSONObject): Response { + val mediaType = "application/json; charset=utf-8".toMediaType() + val requestBody = body.toString().toRequestBody(mediaType) + val request = Request.Builder() + .post(requestBody) + .url(url) + .addTags() + return httpClient.newCall(request.build()).await().ensureSuccess() + } - private fun Request.Builder.addTags(): Request.Builder { - if (mangaSource != null) { - tag(MangaSource::class.java, mangaSource) - } - return this - } + override suspend fun graphQLQuery(endpoint: String, query: String): JSONObject { + val body = JSONObject() + body.put("operationName", null as Any?) + body.put("variables", JSONObject()) + body.put("query", "{$query}") + val json = httpPost(endpoint, body).parseJson() + json.optJSONArray("errors")?.let { + if (it.length() != 0) { + throw GraphQLException(it) + } + } + return json + } - private fun Response.ensureSuccess(): Response { - val exception: Exception? = when (code) { // Catch some error codes, not all - 404 -> NotFoundException(message, request.url.toString()) - in 500..599 -> HttpStatusException(message, code, request.url.toString()) - else -> null - } - if (exception != null) { - runCatching { - close() - }.onFailure { - exception.addSuppressed(it) - } - throw exception - } - return this - } + private fun Request.Builder.addTags(): Request.Builder { + tag(MangaSource::class.java, mangaSource) + return this + } + + private fun Response.ensureSuccess(): Response { + val exception: Exception? = when (code) { // Catch some error codes, not all + 404 -> NotFoundException(message, request.url.toString()) + in 500..599 -> HttpStatusException(message, code, request.url.toString()) + else -> null + } + if (exception != null) { + runCatching { + close() + }.onFailure { + exception.addSuppressed(it) + } + throw exception + } + return this + } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/network/WebClient.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/network/WebClient.kt index 93770c4f..a304e0ab 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/network/WebClient.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/network/WebClient.kt @@ -7,62 +7,76 @@ import org.json.JSONObject interface WebClient { - /** - * Do a GET http request to specific url - * @param url - */ - suspend fun httpGet(url: String): Response = httpGet(url.toHttpUrl()) + /** + * Do a GET http request to specific url + * @param url + */ + suspend fun httpGet(url: String): Response = httpGet(url.toHttpUrl()) - /** - * Do a GET http request to specific url - * @param url - */ - suspend fun httpGet(url: HttpUrl): Response + /** + * Do a GET http request to specific url + * @param url + */ + suspend fun httpGet(url: HttpUrl): Response - /** - * Do a HEAD http request to specific url - * @param url - */ - suspend fun httpHead(url: String): Response = httpHead(url.toHttpUrl()) + /** + * Do a HEAD http request to specific url + * @param url + */ + suspend fun httpHead(url: String): Response = httpHead(url.toHttpUrl()) - /** - * Do a HEAD http request to specific url - * @param url - */ - suspend fun httpHead(url: HttpUrl): Response + /** + * Do a HEAD http request to specific url + * @param url + */ + suspend fun httpHead(url: HttpUrl): Response - /** - * Do a POST http request to specific url with `multipart/form-data` payload - * @param url - * @param form payload as key=>value map - */ - suspend fun httpPost(url: String, form: Map): Response = httpPost(url.toHttpUrl(), form) + /** + * Do a POST http request to specific url with `multipart/form-data` payload + * @param url + * @param form payload as key=>value map + */ + suspend fun httpPost(url: String, form: Map): Response = httpPost(url.toHttpUrl(), form) - /** - * Do a POST http request to specific url with `multipart/form-data` payload - * @param url - * @param form payload as key=>value map - */ - suspend fun httpPost(url: HttpUrl, form: Map): Response + /** + * Do a POST http request to specific url with `multipart/form-data` payload + * @param url + * @param form payload as key=>value map + */ + suspend fun httpPost(url: HttpUrl, form: Map): Response - /** - * Do a POST http request to specific url with `multipart/form-data` payload - * @param url - * @param payload payload as `key=value` string with `&` separator - */ - suspend fun httpPost(url: String, payload: String): Response = httpPost(url.toHttpUrl(), payload) + /** + * Do a POST http request to specific url with `multipart/form-data` payload + * @param url + * @param payload payload as `key=value` string with `&` separator + */ + suspend fun httpPost(url: String, payload: String): Response = httpPost(url.toHttpUrl(), payload) - /** - * Do a POST http request to specific url with `multipart/form-data` payload - * @param url - * @param payload payload as `key=value` string with `&` separator - */ - suspend fun httpPost(url: HttpUrl, payload: String): Response + /** + * Do a POST http request to specific url with `multipart/form-data` payload + * @param url + * @param payload payload as `key=value` string with `&` separator + */ + suspend fun httpPost(url: HttpUrl, payload: String): Response - /** - * Do a GraphQL request to specific url - * @param endpoint an url - * @param query GraphQL request payload - */ - suspend fun graphQLQuery(endpoint: String, query: String): JSONObject + /** + * Do a POST http request to specific url with json payload + * @param url + * @param body + */ + suspend fun httpPost(url: String, body: JSONObject): Response = httpPost(url.toHttpUrl(), body) + + /** + * Do a POST http request to specific url with json payload + * @param url + * @param body + */ + suspend fun httpPost(url: HttpUrl, body: JSONObject): Response + + /** + * Do a GraphQL request to specific url + * @param endpoint an url + * @param query GraphQL request payload + */ + suspend fun graphQLQuery(endpoint: String, query: String): JSONObject } From 34d2e1530252612d22a7da9c0d4498dc7183e484 Mon Sep 17 00:00:00 2001 From: CakesTwix Date: Tue, 18 Apr 2023 18:34:22 +0300 Subject: [PATCH 102/121] [New source] HoneyManga --- .../kotatsu/parsers/site/HoneyMangaParser.kt | 206 ++++++++++++++++++ 1 file changed, 206 insertions(+) create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/HoneyMangaParser.kt diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/HoneyMangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/HoneyMangaParser.kt new file mode 100644 index 00000000..161943d2 --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/HoneyMangaParser.kt @@ -0,0 +1,206 @@ +package org.koitharu.kotatsu.core.parser + +import okhttp3.Headers +import okhttp3.Interceptor +import okhttp3.Response +import org.json.JSONArray +import org.json.JSONObject +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.PagedMangaParser +import org.koitharu.kotatsu.parsers.config.ConfigKey +import org.koitharu.kotatsu.parsers.model.Manga +import org.koitharu.kotatsu.parsers.model.MangaChapter +import org.koitharu.kotatsu.parsers.model.MangaPage +import org.koitharu.kotatsu.parsers.model.MangaSource +import org.koitharu.kotatsu.parsers.model.MangaState +import org.koitharu.kotatsu.parsers.model.MangaTag +import org.koitharu.kotatsu.parsers.model.RATING_UNKNOWN +import org.koitharu.kotatsu.parsers.model.SortOrder +import org.koitharu.kotatsu.parsers.util.domain +import org.koitharu.kotatsu.parsers.util.generateUid +import org.koitharu.kotatsu.parsers.util.json.getStringOrNull +import org.koitharu.kotatsu.parsers.util.json.mapJSON +import org.koitharu.kotatsu.parsers.util.json.mapJSONIndexed +import org.koitharu.kotatsu.parsers.util.parseJson +import org.koitharu.kotatsu.parsers.util.parseJsonArray +import org.koitharu.kotatsu.parsers.util.removeSuffix +import org.koitharu.kotatsu.parsers.util.tryParse +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale + + +private const val PAGE_SIZE = 20 + +@MangaSourceParser("HONEYMANGA", "Honey Manga", "uk") +class HoneyMangaParser(context: MangaLoaderContext) : PagedMangaParser(context, MangaSource.HONEYMANGA, PAGE_SIZE), Interceptor { + + private val urlApi = "https://data.api.$domain" + private val mangaApi = "$urlApi/v2/manga/cursor-list" + private val chapterApi = "$urlApi/v2/chapter/cursor-list" + private val genresListApi = "$urlApi/genres-tags/genres-list" + private val framesApi = "$urlApi/chapter/frames" + private val searchApi = "https://search.api.$domain/api/v1/title/search-matching?query=" + + private val imageStorageUrl = "https://manga-storage.fra1.digitaloceanspaces.com/public-resources" + override val headers + get() = Headers.Builder() + .add("User-Agent", "Mozilla/5.0 (Android 13; Mobile; rv:68.0) Gecko/68.0 Firefox/109.0") + .build() + + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("honey-manga.com.ua", null) + + override val sortOrders: Set = EnumSet.of( + SortOrder.POPULARITY, + SortOrder.NEWEST, + ) + + override suspend fun getDetails(manga: Manga): Manga { + val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.US) + val body = JSONObject() + body.put("mangaId", manga.url) + body.put("pageSize", 999999) // Hack lol (no) + body.put("page", 1) + body.put("sortOrder", "ASC") + val chapterRequest = webClient.httpPost(chapterApi, body).parseJson() + return manga.copy( + chapters = chapterRequest.getJSONArray("data").mapJSONIndexed() { i, jo -> + MangaChapter( + id = generateUid(jo.getString("id")), + name = buildString { + append("Том ") + append(jo.optString("volume", "0")) + append(". ") + append("Розділ ") + append(jo.optString("chapterNum", "0")) + if (jo.optString("title") != "Title") { + append(" - ") + append(jo.optString("title")) + } + }, + number = i + 1, + url = jo.optString("chapterResourcesId"), + scanlator = null, + uploadDate = dateFormat.tryParse(jo.getString("lastUpdated")), + branch = null, + source = source + ) + } + ) + } + + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder + ): List { + val body = JSONObject() + var content: JSONArray? = null + content = if (!query.isNullOrEmpty()) { + if((query.length < 3) || (page > 1)) return emptyList() + body.put("query", query) + webClient.httpGet(searchApi + query).parseJsonArray() + } else { + body.put("page", page) + body.put("pageSize", PAGE_SIZE) + body.put("filters", JSONArray()) + val sort = JSONObject() + sort.put("sortBy", getSortKey(sortOrder)) + sort.put("sortOrder", "DESC") + body.put("sort", sort) + webClient.httpPost(mangaApi, body).parseJson().getJSONArray("data") + } + return content!!.mapJSON { jo -> + val id = jo.getString("id") + val posterUrl = jo.getString("posterUrl") + Manga( + id = generateUid(id), + title = jo.getString("title"), + altTitle = jo.getStringOrNull("alternativeTitle"), + url = id, + publicUrl = "https://$domain/book/$id", + rating = RATING_UNKNOWN, + isNsfw = isNsfw(jo.getStringOrNull("adult")), + coverUrl = getCoverUrl(posterUrl, 256), + tags = getTags(jo.optJSONArray("genresAndTags")!!), + state = when (jo.getStringOrNull("titleStatus")) { + "Онгоінг" -> MangaState.ONGOING + "Завершено" -> MangaState.FINISHED + else -> null + }, + author = null, + largeCoverUrl = getCoverUrl(posterUrl, 1080), + description = jo.getStringOrNull("description"), + chapters = null, + source = source, + ) + } + } + + override suspend fun getPages(chapter: MangaChapter): List { + val body = JSONObject() + body.put("chapterId", chapter.url) + + val content = webClient.httpPost(framesApi, body).parseJson().getJSONObject("resourceIds") + val mangaPage = mutableListOf() + (0 until content.length()).forEach { i -> + val item = content.get(i.toString()).toString() + + mangaPage.add(MangaPage(id = generateUid(item), "$imageStorageUrl/$item", getCoverUrl(item, 256), source)) + } + return mangaPage + } + + override suspend fun getTags(): Set { + // https://data.api.honey-manga.com.ua/genres-tags/genres-list + val tagsSet = mutableListOf() + val content = webClient.httpGet(genresListApi).parseJsonArray() + (0 until content.length()).forEach { i -> + val item = content.get(i).toString() + + tagsSet.add(MangaTag(item, item, source)) + } + + return tagsSet.toSet() + } + + // Need for disable encoding (with encoding not working) + override fun intercept(chain: Interceptor.Chain): Response { + val request = chain.request() + val newRequest = if (request.header("Content-Encoding") != null) { + request.newBuilder().removeHeader("CommonHeaders.CONTENT_ENCODING").build() + } else { + request + } + return chain.proceed(newRequest) + } + + private fun isNsfw(adultValue: String?): Boolean { + val intValue = adultValue?.removeSuffix('+')?.toIntOrNull() + return intValue != null && intValue >= 18 + } + + private fun getCoverUrl(id: String, w: Int): String { + return "https://$domain/_next/image?url=https%3A%2F%2Fmanga-storage.fra1.digitaloceanspaces.com%2Fpublic-resources%2F$id&w=$w&q=75" + } + + private fun getTags(jsonTags: JSONArray): Set { + val tagsSet = mutableListOf() + (0 until jsonTags.length()).forEach { i -> + val item = jsonTags.get(i) + + tagsSet.add(MangaTag(item.toString(), item.toString(), source)) + } + + return tagsSet.toSet() + } + + private fun getSortKey(order: SortOrder?) = when (order) { + SortOrder.POPULARITY -> "likes" + SortOrder.NEWEST -> "lastUpdated" + else -> "likes" + } +} From a3565abdc37d2fb5e287ca79e2dd4c1adb9dfb25 Mon Sep 17 00:00:00 2001 From: CakesTwix Date: Tue, 18 Apr 2023 19:35:15 +0300 Subject: [PATCH 103/121] [HoneyManga] Improve code --- .../kotatsu/parsers/site/HoneyMangaParser.kt | 54 ++++++++++--------- 1 file changed, 28 insertions(+), 26 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/HoneyMangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/HoneyMangaParser.kt index 161943d2..d9d8ead9 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/HoneyMangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/HoneyMangaParser.kt @@ -1,5 +1,6 @@ package org.koitharu.kotatsu.core.parser +import androidx.collection.ArraySet import okhttp3.Headers import okhttp3.Interceptor import okhttp3.Response @@ -26,6 +27,7 @@ import org.koitharu.kotatsu.parsers.util.parseJson import org.koitharu.kotatsu.parsers.util.parseJsonArray import org.koitharu.kotatsu.parsers.util.removeSuffix import org.koitharu.kotatsu.parsers.util.tryParse +import org.koitharu.kotatsu.parsers.util.urlEncoded import java.text.SimpleDateFormat import java.util.EnumSet import java.util.Locale @@ -36,14 +38,15 @@ private const val PAGE_SIZE = 20 @MangaSourceParser("HONEYMANGA", "Honey Manga", "uk") class HoneyMangaParser(context: MangaLoaderContext) : PagedMangaParser(context, MangaSource.HONEYMANGA, PAGE_SIZE), Interceptor { - private val urlApi = "https://data.api.$domain" - private val mangaApi = "$urlApi/v2/manga/cursor-list" - private val chapterApi = "$urlApi/v2/chapter/cursor-list" - private val genresListApi = "$urlApi/genres-tags/genres-list" - private val framesApi = "$urlApi/chapter/frames" + private val urlApi get() = "https://data.api.$domain" + private val mangaApi get() = "$urlApi/v2/manga/cursor-list" + private val chapterApi get() = "$urlApi/v2/chapter/cursor-list" + private val genresListApi get() = "$urlApi/genres-tags/genres-list" + private val framesApi get() = "$urlApi/chapter/frames" private val searchApi = "https://search.api.$domain/api/v1/title/search-matching?query=" private val imageStorageUrl = "https://manga-storage.fra1.digitaloceanspaces.com/public-resources" + override val headers get() = Headers.Builder() .add("User-Agent", "Mozilla/5.0 (Android 13; Mobile; rv:68.0) Gecko/68.0 Firefox/109.0") @@ -98,12 +101,10 @@ class HoneyMangaParser(context: MangaLoaderContext) : PagedMangaParser(context, sortOrder: SortOrder ): List { val body = JSONObject() - var content: JSONArray? = null - content = if (!query.isNullOrEmpty()) { - if((query.length < 3) || (page > 1)) return emptyList() - body.put("query", query) - webClient.httpGet(searchApi + query).parseJsonArray() - } else { + var content: JSONArray = JSONArray() + + // Popular/Newest + content = if (query.isNullOrEmpty()) { body.put("page", page) body.put("pageSize", PAGE_SIZE) body.put("filters", JSONArray()) @@ -112,8 +113,12 @@ class HoneyMangaParser(context: MangaLoaderContext) : PagedMangaParser(context, sort.put("sortOrder", "DESC") body.put("sort", sort) webClient.httpPost(mangaApi, body).parseJson().getJSONArray("data") + // Search + } else { + if((query.length < 3) || (page > 1)) return emptyList() + webClient.httpGet(searchApi + query.urlEncoded()).parseJsonArray() } - return content!!.mapJSON { jo -> + return content.mapJSON { jo -> val id = jo.getString("id") val posterUrl = jo.getString("posterUrl") Manga( @@ -125,7 +130,7 @@ class HoneyMangaParser(context: MangaLoaderContext) : PagedMangaParser(context, rating = RATING_UNKNOWN, isNsfw = isNsfw(jo.getStringOrNull("adult")), coverUrl = getCoverUrl(posterUrl, 256), - tags = getTags(jo.optJSONArray("genresAndTags")!!), + tags = getTitleTags(jo.optJSONArray("genresAndTags")), state = when (jo.getStringOrNull("titleStatus")) { "Онгоінг" -> MangaState.ONGOING "Завершено" -> MangaState.FINISHED @@ -145,13 +150,10 @@ class HoneyMangaParser(context: MangaLoaderContext) : PagedMangaParser(context, body.put("chapterId", chapter.url) val content = webClient.httpPost(framesApi, body).parseJson().getJSONObject("resourceIds") - val mangaPage = mutableListOf() - (0 until content.length()).forEach { i -> + return List(content.length()) { i -> val item = content.get(i.toString()).toString() - - mangaPage.add(MangaPage(id = generateUid(item), "$imageStorageUrl/$item", getCoverUrl(item, 256), source)) + MangaPage(id = generateUid(item), "$imageStorageUrl/$item", getCoverUrl(item, 256), source) } - return mangaPage } override suspend fun getTags(): Set { @@ -187,8 +189,14 @@ class HoneyMangaParser(context: MangaLoaderContext) : PagedMangaParser(context, return "https://$domain/_next/image?url=https%3A%2F%2Fmanga-storage.fra1.digitaloceanspaces.com%2Fpublic-resources%2F$id&w=$w&q=75" } - private fun getTags(jsonTags: JSONArray): Set { - val tagsSet = mutableListOf() + private fun getSortKey(order: SortOrder?) = when (order) { + SortOrder.POPULARITY -> "likes" + SortOrder.NEWEST -> "lastUpdated" + else -> "likes" + } + + private fun getTitleTags(jsonTags: JSONArray): Set { + val tagsSet = ArraySet(jsonTags.length()) (0 until jsonTags.length()).forEach { i -> val item = jsonTags.get(i) @@ -197,10 +205,4 @@ class HoneyMangaParser(context: MangaLoaderContext) : PagedMangaParser(context, return tagsSet.toSet() } - - private fun getSortKey(order: SortOrder?) = when (order) { - SortOrder.POPULARITY -> "likes" - SortOrder.NEWEST -> "lastUpdated" - else -> "likes" - } } From 3112b7937cc0a2dd71ec6a7d7135622cb135ad17 Mon Sep 17 00:00:00 2001 From: vianh Date: Tue, 18 Apr 2023 23:49:46 +0700 Subject: [PATCH 104/121] [DoujinDesu] Apply suggesions from code review --- .../kotatsu/parsers/site/DoujinDesuParser.kt | 47 +++++++++++-------- 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/DoujinDesuParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/DoujinDesuParser.kt index 04746949..9e9faa7a 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/DoujinDesuParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/DoujinDesuParser.kt @@ -1,6 +1,5 @@ package org.koitharu.kotatsu.parsers.site -import org.koitharu.kotatsu.parsers.InternalParsersApi import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaSourceParser import org.koitharu.kotatsu.parsers.PagedMangaParser @@ -12,7 +11,7 @@ import java.util.* @MangaSourceParser("DOUJINDESU", "DoujinDesu", "id") class DoujinDesuParser(context: MangaLoaderContext) : PagedMangaParser(context, MangaSource.DOUJINDESU, pageSize = 18) { - @InternalParsersApi + override val configKeyDomain: ConfigKey.Domain get() = ConfigKey.Domain("212.32.226.234", null) @@ -21,6 +20,7 @@ class DoujinDesuParser(context: MangaLoaderContext) : PagedMangaParser(context, override suspend fun getDetails(manga: Manga): Manga { val docs = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml().selectFirstOrThrow("#archive") + val chapterDateFormat = SimpleDateFormat("EEEE, dd MMMM yyyy", sourceLocale) val metadataEl = docs.selectFirst(".wrapper > .metadata tbody") val state = when (metadataEl?.selectFirst("tr:contains(Status)")?.selectLast("td")?.text()) { "Finished" -> MangaState.FINISHED @@ -39,20 +39,22 @@ class DoujinDesuParser(context: MangaLoaderContext) : PagedMangaParser(context, source = source, ) }, - chapters = docs.select("#chapter_list > ul > li").mapChapters(reversed = true) { index, element -> - val titleTag = element.selectFirstOrThrow(".epsleft > .lchx > a") - val dateFormat = SimpleDateFormat("EEEE, dd MMMM yyyy", Locale("in")) - MangaChapter( - id = generateUid(titleTag.attrAsRelativeUrl("href")), - name = titleTag.text(), - number = index + 1, - url = titleTag.attrAsRelativeUrl("href"), - scanlator = null, - uploadDate = dateFormat.tryParse(element.select(".epsleft > .date").text()), - branch = null, - source = source, - ) - }, + chapters = docs.requireElementById("chapter_list") + .select("ul > li") + .mapChapters(reversed = true) { index, element -> + val titleTag = element.selectFirstOrThrow(".epsleft > .lchx > a") + val url = titleTag.attrAsRelativeUrl("href") + MangaChapter( + id = generateUid(url), + name = titleTag.text(), + number = index + 1, + url = url, + scanlator = null, + uploadDate = chapterDateFormat.tryParse(element.select(".epsleft > .date").text()), + branch = null, + source = source, + ) + }, ) } @@ -60,7 +62,7 @@ class DoujinDesuParser(context: MangaLoaderContext) : PagedMangaParser(context, page: Int, query: String?, tags: Set?, - sortOrder: SortOrder + sortOrder: SortOrder, ): List { val url = urlBuilder().apply { addPathSegment("manga") @@ -80,7 +82,10 @@ class DoujinDesuParser(context: MangaLoaderContext) : PagedMangaParser(context, } }.build() - return webClient.httpGet(url).parseHtml().select("#archives > div.entries > .entry") + return webClient.httpGet(url).parseHtml() + .requireElementById("archives") + .selectFirstOrThrow("div.entries") + .select(".entry") .map { val titleTag = it.selectFirstOrThrow(".metadata > a") val relativeUrl = titleTag.attrAsRelativeUrl("href") @@ -105,7 +110,7 @@ class DoujinDesuParser(context: MangaLoaderContext) : PagedMangaParser(context, override suspend fun getPages(chapter: MangaChapter): List { val id = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() - .selectFirstOrThrow("#reader") + .requireElementById("reader") .attr("data-id") return webClient.httpPost("/themes/ajax/ch.php".toAbsoluteUrl(domain), "id=$id").parseHtml() .select("img") @@ -122,7 +127,9 @@ class DoujinDesuParser(context: MangaLoaderContext) : PagedMangaParser(context, override suspend fun getTags(): Set { return webClient.httpGet("/genre/".toAbsoluteUrl(domain)).parseHtml() - .select("#taxonomy .entries > .entry > a") + .requireElementById("taxonomy") + .selectFirstOrThrow(".entries") + .select(".entry > a") .mapToSet { MangaTag( key = it.attr("title"), From e5ececa42db3f7af745723f4f54450ad6864de9f Mon Sep 17 00:00:00 2001 From: CakesTwix Date: Wed, 19 Apr 2023 19:58:25 +0300 Subject: [PATCH 105/121] [HoneyManga] Update code, get(), CONTENT_ENCODING --- .../org/koitharu/kotatsu/parsers/site/HoneyMangaParser.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/HoneyMangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/HoneyMangaParser.kt index d9d8ead9..e2fa848f 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/HoneyMangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/HoneyMangaParser.kt @@ -43,7 +43,7 @@ class HoneyMangaParser(context: MangaLoaderContext) : PagedMangaParser(context, private val chapterApi get() = "$urlApi/v2/chapter/cursor-list" private val genresListApi get() = "$urlApi/genres-tags/genres-list" private val framesApi get() = "$urlApi/chapter/frames" - private val searchApi = "https://search.api.$domain/api/v1/title/search-matching?query=" + private val searchApi get() = "https://search.api.$domain/api/v1/title/search-matching?query=" private val imageStorageUrl = "https://manga-storage.fra1.digitaloceanspaces.com/public-resources" @@ -173,7 +173,7 @@ class HoneyMangaParser(context: MangaLoaderContext) : PagedMangaParser(context, override fun intercept(chain: Interceptor.Chain): Response { val request = chain.request() val newRequest = if (request.header("Content-Encoding") != null) { - request.newBuilder().removeHeader("CommonHeaders.CONTENT_ENCODING").build() + request.newBuilder().removeHeader("Content-Encoding").build() } else { request } From 647a694d9171bb4c510c4deaa0fb54909114e127 Mon Sep 17 00:00:00 2001 From: CakesTwix Date: Wed, 19 Apr 2023 20:18:45 +0300 Subject: [PATCH 106/121] [HoneyManga] throw IllegalArgumentException if search < 3 --- .../org/koitharu/kotatsu/parsers/site/HoneyMangaParser.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/HoneyMangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/HoneyMangaParser.kt index e2fa848f..ce6de408 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/HoneyMangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/HoneyMangaParser.kt @@ -115,7 +115,7 @@ class HoneyMangaParser(context: MangaLoaderContext) : PagedMangaParser(context, webClient.httpPost(mangaApi, body).parseJson().getJSONArray("data") // Search } else { - if((query.length < 3) || (page > 1)) return emptyList() + if((query.length < 3) || (page > 1)) throw IllegalArgumentException("Запит має містити щонайменше 3 символи / The query must contain at least 3 characters") webClient.httpGet(searchApi + query.urlEncoded()).parseJsonArray() } return content.mapJSON { jo -> From e6511061a78b001f718072b58b18d7da8d09052b Mon Sep 17 00:00:00 2001 From: Koitharu Date: Thu, 20 Apr 2023 18:07:20 +0300 Subject: [PATCH 107/121] [Grouple] Fix server selecting --- .../koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt index 81c81b4a..6b58614f 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt @@ -240,7 +240,7 @@ internal abstract class GroupleParser( val path = parts.last() val servers = parts.dropLast(1).toSet() val cachedServer = cachedPagesServer - if (cachedServer != null && cachedServer in servers && tryHead(concatUrl(cachedServer, path))) { + if (!cachedServer.isNullOrEmpty() && cachedServer in servers && tryHead(concatUrl(cachedServer, path))) { return concatUrl(cachedServer, path) } if (servers.isEmpty()) { @@ -260,7 +260,7 @@ internal abstract class GroupleParser( } catch (e: NoSuchElementException) { servers.random() } - return concatUrl(checkNotNull(server), path) + return concatUrl(checkNotNull(server).ifEmpty { "https://$domain/" }, path) } override suspend fun getTags(): Set { From 8def9b5446b31f868e39fb71a9525951e47f67de Mon Sep 17 00:00:00 2001 From: Koitharu Date: Sun, 23 Apr 2023 09:20:11 +0300 Subject: [PATCH 108/121] Improve network error handling --- .../parsers/network/OkHttpWebClient.kt | 10 ++++++-- .../kotatsu/parsers/site/RemangaParser.kt | 23 +++++-------------- 2 files changed, 14 insertions(+), 19 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/network/OkHttpWebClient.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/network/OkHttpWebClient.kt index 0bfe7ef3..2f70e093 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/network/OkHttpWebClient.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/network/OkHttpWebClient.kt @@ -5,11 +5,13 @@ import okhttp3.MediaType.Companion.toMediaType import okhttp3.RequestBody.Companion.toRequestBody import org.json.JSONObject import org.jsoup.HttpStatusException +import org.koitharu.kotatsu.parsers.exception.AuthRequiredException import org.koitharu.kotatsu.parsers.exception.GraphQLException import org.koitharu.kotatsu.parsers.exception.NotFoundException import org.koitharu.kotatsu.parsers.model.MangaSource import org.koitharu.kotatsu.parsers.util.await import org.koitharu.kotatsu.parsers.util.parseJson +import java.net.HttpURLConnection class OkHttpWebClient( private val httpClient: OkHttpClient, @@ -92,8 +94,12 @@ class OkHttpWebClient( private fun Response.ensureSuccess(): Response { val exception: Exception? = when (code) { // Catch some error codes, not all - 404 -> NotFoundException(message, request.url.toString()) - in 500..599 -> HttpStatusException(message, code, request.url.toString()) + HttpURLConnection.HTTP_NOT_FOUND -> NotFoundException(message, request.url.toString()) + HttpURLConnection.HTTP_UNAUTHORIZED -> request.tag(MangaSource::class.java)?.let { + AuthRequiredException(it) + } ?: HttpStatusException(message, code, request.url.toString()) + + in 400..599 -> HttpStatusException(message, code, request.url.toString()) else -> null } if (exception != null) { diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/RemangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/RemangaParser.kt index 1acffda3..23b67594 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/RemangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/RemangaParser.kt @@ -1,7 +1,6 @@ package org.koitharu.kotatsu.parsers.site import okhttp3.Headers -import okhttp3.Response import org.json.JSONArray import org.json.JSONException import org.json.JSONObject @@ -10,13 +9,11 @@ import org.koitharu.kotatsu.parsers.MangaParserAuthProvider import org.koitharu.kotatsu.parsers.MangaSourceParser import org.koitharu.kotatsu.parsers.PagedMangaParser import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.exception.AuthRequiredException import org.koitharu.kotatsu.parsers.exception.ContentUnavailableException import org.koitharu.kotatsu.parsers.exception.ParseException import org.koitharu.kotatsu.parsers.model.* import org.koitharu.kotatsu.parsers.util.* import org.koitharu.kotatsu.parsers.util.json.* -import java.net.HttpURLConnection import java.net.URLDecoder import java.text.DateFormat import java.text.SimpleDateFormat @@ -121,7 +118,7 @@ internal class RemangaParser( ?: throw ParseException("Cannot obtain slug from ${manga.url}", manga.publicUrl) val data = webClient.httpGet( url = "https://api.$domain/api/titles$slug/", - ).handle401().parseJson() + ).parseJson() val content = try { data.getJSONObject("content") } catch (e: JSONException) { @@ -180,9 +177,7 @@ internal class RemangaParser( } override suspend fun getPages(chapter: MangaChapter): List { - val referer = "https://${domain}/" val content = webClient.httpGet(chapter.url.toAbsoluteUrl(getDomain("api"))) - .handle401() .parseJson() .getJSONObject("content") val pages = content.optJSONArray("pages") @@ -200,8 +195,8 @@ internal class RemangaParser( val result = ArrayList(pages.length()) for (i in 0 until pages.length()) { when (val item = pages.get(i)) { - is JSONObject -> result += parsePage(item, referer) - is JSONArray -> item.mapJSONTo(result) { parsePage(it, referer) } + is JSONObject -> result += parsePage(item) + is JSONArray -> item.mapJSONTo(result) { parsePage(it) } else -> throw ParseException("Unknown json item $item", chapter.url) } } @@ -224,7 +219,7 @@ internal class RemangaParser( override suspend fun getUsername(): String { val jo = webClient.httpGet( url = "https://api.${domain}/api/users/current/", - ).handle401().parseJson() + ).parseJson() return jo.getJSONObject("content").getString("username") } @@ -250,7 +245,7 @@ internal class RemangaParser( else -> "-chapter_date" } - private fun parsePage(jo: JSONObject, referer: String) = MangaPage( + private fun parsePage(jo: JSONObject) = MangaPage( id = generateUid(jo.getLong("id")), url = jo.getString("link"), preview = null, @@ -263,7 +258,7 @@ internal class RemangaParser( while (true) { val content = webClient.httpGet( url = "https://api.$domain/api/titles/chapters/?branch_id=$branchId&page=$page&count=100", - ).handle401().parseJson().getJSONArray("content") + ).parseJson().getJSONArray("content") val len = content.length() if (len == 0) { break @@ -276,10 +271,4 @@ internal class RemangaParser( } return result } - - private fun Response.handle401() = apply { - if (code == HttpURLConnection.HTTP_UNAUTHORIZED) { - throw AuthRequiredException(source) - } - } } From 66adc65a96b9427ef9f1a9b70cc568c1fe430a22 Mon Sep 17 00:00:00 2001 From: Koitharu Date: Mon, 24 Apr 2023 16:48:17 +0300 Subject: [PATCH 109/121] [JapScan] New parser --- .../kotatsu/parsers/site/JapScanParser.kt | 193 ++++++++++++++++++ .../parsers/site/grouple/GroupleParser.kt | 7 +- .../koitharu/kotatsu/parsers/util/Jsoup.kt | 4 + .../koitharu/kotatsu/parsers/util/Parse.kt | 6 + 4 files changed, 206 insertions(+), 4 deletions(-) create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/JapScanParser.kt diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/JapScanParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/JapScanParser.kt new file mode 100644 index 00000000..8af433f5 --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/JapScanParser.kt @@ -0,0 +1,193 @@ +package org.koitharu.kotatsu.parsers.site + +import okhttp3.Headers +import org.json.JSONObject +import org.jsoup.nodes.Element +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.PagedMangaParser +import org.koitharu.kotatsu.parsers.config.ConfigKey +import org.koitharu.kotatsu.parsers.exception.ParseException +import org.koitharu.kotatsu.parsers.model.* +import org.koitharu.kotatsu.parsers.util.* +import org.koitharu.kotatsu.parsers.util.json.getStringOrNull +import org.koitharu.kotatsu.parsers.util.json.mapJSON +import java.text.SimpleDateFormat +import java.util.* + +@MangaSourceParser("JAPSCAN", "JapScan", "fr") +internal class JapScanParser(context: MangaLoaderContext) : PagedMangaParser(context, MangaSource.JAPSCAN, 30) { + + override val sortOrders: Set = EnumSet.of(SortOrder.ALPHABETICAL) + + override val configKeyDomain = ConfigKey.Domain("www.japscan.lol", arrayOf("www.japscan.lol", "japscan.ws")) + + override val headers: Headers = Headers.Builder() + .add("User-Agent", "Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/113.0") + .build() + + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + require(tags.isNullOrEmpty()) { "Tags are not supported by this source" } + if (!query.isNullOrEmpty()) { + return if (page == paginator.firstPage) getListPageSearch(query) else emptyList() + } + val url = urlBuilder() + .addPathSegment("mangas") + .addPathSegment(page.toString()) + .build() + val root = webClient.httpGet(url).parseHtml() + .requireElementById("main") + .selectFirstOrThrow(".flex-wrap") + return root.select(".mainTitle") + .map { p -> + val div = checkNotNull(p.parent()) + val a = div.selectFirstOrThrow("a") + val href = a.attrAsRelativeUrl("href") + Manga( + id = generateUid(href), + title = p.text(), + altTitle = null, + url = href, + publicUrl = href.toAbsoluteUrl(domain), + rating = RATING_UNKNOWN, + isNsfw = false, + coverUrl = div.selectFirstOrThrow("img").attrAsAbsoluteUrl("src"), + tags = setOf(), + state = null, + author = null, + source = source, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val root = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml().requireElementById("main") + val dateFormat = SimpleDateFormat("dd MMM yyyy", Locale.ENGLISH) + return manga.copy( + altTitle = root.tableValue("Nom Original:"), + isNsfw = root.tableValue("Âge conseillé :")?.extractIntOrNull().let { it != null && it >= 18 }, + tags = root.tableValue("Type(s):")?.split(", ")?.mapNotNullToSet { + it.toTag() + }.orEmpty() + root.tableValue("Genre(s):")?.split(", ")?.mapNotNullToSet { + it.toTag() + }.orEmpty(), + state = when (root.tableValue("Statut:")) { + "En Cours" -> MangaState.ONGOING + "Terminé", "Abondonné" -> MangaState.FINISHED + else -> null + }, + author = root.tableValue("Artiste(s):")?.substringBefore(','), + description = root.selectFirst("p.list-group-item-primary")?.html(), + chapters = root.requireElementById("chapters_list") + .select("div.chapters_list") + .mapChapters(reversed = true) { i, div -> + val a = div.selectFirst("a") ?: return@mapChapters null + val href = a.attrAsRelativeUrl("href") + MangaChapter( + id = generateUid(href), + name = a.text(), + number = i, + url = href, + scanlator = null, + uploadDate = dateFormat.tryParse(div.selectFirst("span.float-right")?.text()), + branch = null, + source = source, + ) + }, + ) + } + + override suspend fun getPages(chapter: MangaChapter): List { + val chapterUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(chapterUrl).parseHtml() + val scriptUrl = doc.getElementsByTag("script").firstNotNullOf { script -> + script.attrAsAbsoluteUrlOrNull("src")?.takeIf { it.contains("/zjs/") } + } + val embeddedData = doc.requireElementById("data").attr("data-data") + val script = webClient.httpGet(scriptUrl).parseRaw() + + val sample = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz".toList() + val keyRegex = Regex("""'([\dA-Z]{62})'""", RegexOption.IGNORE_CASE) + + val keyTables = keyRegex.findAll(script) + .mapNotNullTo(ArrayList(2)) { match -> + match.groupValues[1].takeIf { + it.toList().sorted() == sample + } + } + check(keyTables.size == 2) + + var error: Exception? = null + repeat(2) { i -> + val key = keyTables[i].zip(keyTables[1 - i]).toMap() + try { + val unscrambledData = embeddedData.map { key[it] ?: it }.joinToString("") + if (unscrambledData.startsWith("ey")) { + val array = JSONObject(context.decodeBase64(unscrambledData).toString(Charsets.UTF_8)) + .getJSONArray("imagesLink") + val result = ArrayList(array.length()) + repeat(array.length()) { index -> + val url = array.getString(index) + result += MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + return result + } + } catch (e: Exception) { + error = e + } + } + throw (error ?: ParseException("Cannot decode pages list", chapterUrl)) + } + + override suspend fun getTags(): Set { + return emptySet() // not supported + } + + private suspend fun getListPageSearch( + query: String, + ): List { + val json = webClient.httpPost( + "https://$domain/live-search/", + mapOf("search" to query.urlEncoded()), + ).parseJsonArray() + return json.mapJSON { jo -> + val url = jo.getString("url") + Manga( + id = generateUid(url), + title = jo.getString("name"), + altTitle = jo.getStringOrNull("alternate_names")?.substringBefore(','), + url = url, + publicUrl = url.toAbsoluteUrl(domain), + rating = RATING_UNKNOWN, + isNsfw = false, + coverUrl = jo.getString("image").toAbsoluteUrl(domain), + tags = emptySet(), + state = null, + author = null, + source = source, + ) + } + } + + private fun Element.tableValue(label: String): String? { + return getElementsMatchingOwnText(label).firstOrNull()?.parent()?.ownTextOrNull() + } + + private fun String.extractIntOrNull(): Int? = this.filter(Char::isDigit).toIntOrNull() + + private fun String.toTag() = MangaTag( + title = this.toTitleCase(sourceLocale), + key = this.replace(' ', '-').lowercase(sourceLocale), + source = source, + ) +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt index 6b58614f..40a2a0a2 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt @@ -40,10 +40,9 @@ internal abstract class GroupleParser( "Mozilla/5.0 (X11; U; UNICOS lcLinux; en-US) Gecko/20140730 (KHTML, like Gecko, Safari/419.3) Arora/0.8.0", ) - override val headers: Headers - get() = Headers.Builder() - .add("User-Agent", config[userAgentKey]) - .build() + override val headers: Headers = Headers.Builder() + .add("User-Agent", config[userAgentKey]) + .build() override val sortOrders: Set = EnumSet.of( SortOrder.UPDATED, diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Jsoup.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Jsoup.kt index 6e5faeb8..24d1a6bd 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Jsoup.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Jsoup.kt @@ -115,3 +115,7 @@ fun Element.selectLast(cssQuery: String): Element? { fun Element.selectLastOrThrow(cssQuery: String): Element { return selectLast(cssQuery) ?: throw ParseException("Cannot find \"$cssQuery\"", baseUri()) } + +fun Element.textOrNull(): String? = text().takeUnless { it.isEmpty() } + +fun Element.ownTextOrNull(): String? = ownText().takeUnless { it.isEmpty() } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Parse.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Parse.kt index f6c193e1..d1aecc44 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Parse.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Parse.kt @@ -46,6 +46,12 @@ fun Response.parseJsonArray(): JSONArray = try { closeQuietly() } +fun Response.parseRaw(): String = try { + requireBody().string() +} finally { + closeQuietly() +} + /** * Convert url to relative if it is on [domain] * @return an url relative to the [domain] or absolute, if domain is mismatching From 9adc0c5358510790589360165437a28f1ace69e1 Mon Sep 17 00:00:00 2001 From: Koitharu Date: Mon, 24 Apr 2023 19:34:29 +0300 Subject: [PATCH 110/121] [Bentomanga] New parser --- .../parsers/network/OkHttpWebClient.kt | 5 +- .../kotatsu/parsers/network/WebClient.kt | 7 +- .../kotatsu/parsers/site/BentomangaParser.kt | 235 ++++++++++++++++++ 3 files changed, 245 insertions(+), 2 deletions(-) create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/BentomangaParser.kt diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/network/OkHttpWebClient.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/network/OkHttpWebClient.kt index 2f70e093..7b5dc961 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/network/OkHttpWebClient.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/network/OkHttpWebClient.kt @@ -18,11 +18,14 @@ class OkHttpWebClient( private val mangaSource: MangaSource, ) : WebClient { - override suspend fun httpGet(url: HttpUrl): Response { + override suspend fun httpGet(url: HttpUrl, extraHeaders: Headers?): Response { val request = Request.Builder() .get() .url(url) .addTags() + if (extraHeaders != null) { + request.headers(extraHeaders) + } return httpClient.newCall(request.build()).await().ensureSuccess() } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/network/WebClient.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/network/WebClient.kt index a304e0ab..a486824d 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/network/WebClient.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/network/WebClient.kt @@ -1,5 +1,6 @@ package org.koitharu.kotatsu.parsers.network +import okhttp3.Headers import okhttp3.HttpUrl import okhttp3.HttpUrl.Companion.toHttpUrl import okhttp3.Response @@ -13,11 +14,15 @@ interface WebClient { */ suspend fun httpGet(url: String): Response = httpGet(url.toHttpUrl()) + suspend fun httpGet(url: String, extraHeaders: Headers?): Response = httpGet(url.toHttpUrl(), extraHeaders) + /** * Do a GET http request to specific url * @param url */ - suspend fun httpGet(url: HttpUrl): Response + suspend fun httpGet(url: HttpUrl): Response = httpGet(url, null) + + suspend fun httpGet(url: HttpUrl, extraHeaders: Headers?): Response /** * Do a HEAD http request to specific url diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BentomangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BentomangaParser.kt new file mode 100644 index 00000000..1dc2a11d --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BentomangaParser.kt @@ -0,0 +1,235 @@ +package org.koitharu.kotatsu.parsers.site + +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.coroutineScope +import okhttp3.Headers +import org.jsoup.nodes.Element +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.PagedMangaParser +import org.koitharu.kotatsu.parsers.config.ConfigKey +import org.koitharu.kotatsu.parsers.model.* +import org.koitharu.kotatsu.parsers.util.* +import java.util.* + +@MangaSourceParser("BENTOMANGA", "Bentomanga", "fr") +internal class BentomangaParser(context: MangaLoaderContext) : PagedMangaParser(context, MangaSource.BENTOMANGA, 10) { + + override val sortOrders: Set = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.RATING, + SortOrder.NEWEST, + SortOrder.ALPHABETICAL, + ) + + override val configKeyDomain = ConfigKey.Domain("www.bentomanga.com", null) + + override val headers: Headers = Headers.Builder() + .add("User-Agent", "Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/113.0") + .build() + + init { + paginator.firstPage = 0 + searchPaginator.firstPage = 0 + } + + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + val url = urlBuilder() + .addPathSegment("manga_list") + .addQueryParameter("limit", page.toString()) + .addQueryParameter( + "order_by", + when (sortOrder) { + SortOrder.UPDATED -> "update" + SortOrder.POPULARITY -> "views" + SortOrder.RATING -> "top" + SortOrder.NEWEST -> "create" + SortOrder.ALPHABETICAL -> "name" + }, + ) + if (!tags.isNullOrEmpty()) { + url.addQueryParameter("withCategories", tags.joinToString(",") { it.key }) + } + if (!query.isNullOrEmpty()) { + url.addQueryParameter("search", query) + } + val root = webClient.httpGet(url.build()).parseHtml().requireElementById("mangas_content") + return root.select(".manga[data-manga]").map { div -> + val header = div.selectFirstOrThrow(".manga_header") + val href = header.selectFirstOrThrow("a").attrAsRelativeUrl("href") + Manga( + id = generateUid(href), + title = div.selectFirstOrThrow("h1").text(), + altTitle = null, + url = href, + publicUrl = href.toAbsoluteUrl(domain), + rating = div.getElementsByAttributeValue("data-icon", "avg_rate") + .firstOrNull() + ?.textOrNull() + ?.toFloatOrNull() + ?.div(10f) + ?: RATING_UNKNOWN, + isNsfw = div.selectFirst(".badge-adult_content") != null, + coverUrl = div.selectFirstOrThrow("img").src(), + tags = div.selectFirst(".component-manga-categories") + .assertNotNull("tags") + ?.select("a") + ?.mapToSet { a -> + MangaTag( + title = a.text().toTitleCase(sourceLocale), + key = a.attr("href").substringAfterLast('='), + source = source, + ) + }.orEmpty(), + state = null, + author = null, + description = div.selectFirst(".manga_synopsis")?.html().assertNotNull("description"), + source = source, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val mangaUrl = manga.url.toAbsoluteUrl(domain) + val root = webClient.httpGet(mangaUrl).parseHtml() + .requireElementById("container_manga_show") + return manga.copy( + altTitle = root.selectFirst(".component-manga-title_alt")?.textOrNull().assertNotNull("altTitle"), + description = root.selectFirst(".datas_synopsis")?.html().assertNotNull("description") + ?: manga.description, + state = when (root.selectFirst(".datas_more-status-data")?.textOrNull().assertNotNull("status")) { + "En cours" -> MangaState.ONGOING + else -> null + }, + author = root.selectFirst(".datas_more-authors-people")?.textOrNull().assertNotNull("author"), + chapters = run { + val input = root.selectFirst("input[name=\"limit\"]") ?: return@run emptyList() + val max = input.attr("max").toInt() + if (max <= 1) { + parseChapters(root) + } else { + coroutineScope { + val result = ArrayList(parseChapters(root)) + result.ensureCapacity(result.size * max) + (2..max).map { i -> + async { + loadChapters(mangaUrl, i) + } + }.awaitAll() + .flattenTo(result) + result + } + } + }.reversed(), + ) + } + + override suspend fun getPages(chapter: MangaChapter): List { + val chapterUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(chapterUrl).parseHtml() + val chapterId = doc.head().getElementsByAttribute("data-chapter-id").first()!!.attr("data-chapter-id") + val json = webClient.httpGet( + "https://$domain/api/?id=$chapterId&type=chapter", + Headers.headersOf( + "Referer", chapterUrl, + "x-requested-with", "XMLHttpRequest", + ), + ).parseJson() + val baseUrl = json.getString("baseImagesUrl") + val pages = json.getJSONArray("page_array") + return (0 until pages.length()).map { i -> + val url = concatUrl(baseUrl, pages.getString(i)) + MangaPage( + id = generateUid(url), + url = url.toAbsoluteUrl(domain), + preview = null, + source = source, + ) + } + } + + override suspend fun getTags(): Set { + val root = webClient.httpGet(urlBuilder().addPathSegment("manga_list").build()) + .parseHtml() + .requireElementById("search_options-form") + return root.getElementsByAttributeValue("name", "categories[]") + .mapToSet { input -> + val div = input.parents().first()!! + MangaTag( + title = div.text().toTitleCase(sourceLocale), + key = input.attr("value"), + source = source, + ) + } + } + + private suspend fun loadChapters(baseUrl: String, page: Int): List { + return parseChapters(webClient.httpGet("$baseUrl?limit=$page").parseHtml().body()) + } + + private fun parseChapters(root: Element): List { + return root.requireElementById("chapters_content") + .select(".component-chapter").map { div -> + val a = div.selectFirstOrThrow("a") + val href = a.attrAsRelativeUrl("href") + val title = div.selectFirstOrThrow(".chapter_volume").text() + val name = div.selectFirst(".chapter_title")?.textOrNull() + MangaChapter( + id = generateUid(href), + name = if (name != null && name != title) "$title: $name" else title, + number = href.substringAfterLast('/').toIntOrNull() ?: 0, + url = href, + scanlator = div.selectFirst(".team_link-name")?.textOrNull().assertNotNull("scanlator"), + uploadDate = div.selectFirst(".component-chapter-date") + ?.ownTextOrNull() + .parseDate(), + branch = null, + source = source, + ) + } + } + + private fun String?.parseDate(): Long { + if (this == null) { + assert(false) { "Date is null" } + return 0L + } + val parts = split(' ') + assert(parts.size == 2) { "Wrong date $this" } + val count = parts.getOrNull(0)?.toIntOrNull() ?: return 0L + val unit = parts.getOrNull(1) ?: return 0L + val calendarUnit = when (unit) { + "s" -> Calendar.SECOND + "min" -> Calendar.MINUTE + "h" -> Calendar.HOUR + "j" -> Calendar.DAY_OF_YEAR + "sem." -> Calendar.WEEK_OF_YEAR + "mois" -> Calendar.MONTH + "ans", "an" -> Calendar.YEAR + else -> { + assert(false) { "Unknown time unit $unit" } + return 0L + } + } + val calendar = Calendar.getInstance() + calendar.add(calendarUnit, -count) + return calendar.timeInMillis + } + + private fun Element.src(): String { + return attrAsAbsoluteUrlOrNull("data-cfsrc") + ?: attrAsAbsoluteUrlOrNull("src") + ?: attrAsAbsoluteUrlOrNull("data-src") + ?: run { + assert(false) { "Image src not found" } + "" + } + } +} From 5b94badfc2042ad846892c7de215de54317cc870 Mon Sep 17 00:00:00 2001 From: Koitharu Date: Tue, 25 Apr 2023 08:13:32 +0300 Subject: [PATCH 111/121] [MangaRead] Fix list parsing --- .../parsers/site/madara/MadaraParser.kt | 6 - .../kotatsu/parsers/site/madara/MangaRead.kt | 120 ++++++++++++++++++ .../kotatsu/parsers/MangaParserTest.kt | 1 + 3 files changed, 121 insertions(+), 6 deletions(-) create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangaRead.kt diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt index 3adfa0a2..d89b4109 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt @@ -304,12 +304,6 @@ internal abstract class MadaraParser( it.substring(0, pos) to it.substring(pos + 1) }.toMutableMap() - @MangaSourceParser("MANGAREAD", "MangaRead", "en") - class MangaRead(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGAREAD, "www.mangaread.org") { - override val tagPrefix = "genres/" - override val datePattern = "dd.MM.yyyy" - } - @MangaSourceParser("MANGAWEEBS", "MangaWeebs", "en") class MangaWeebs(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGAWEEBS, "mangaweebs.in") { override val datePattern = "dd MMMM HH:mm" diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangaRead.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangaRead.kt new file mode 100644 index 00000000..d376dfaf --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangaRead.kt @@ -0,0 +1,120 @@ +package org.koitharu.kotatsu.parsers.site.madara + +import androidx.collection.arraySetOf +import org.jsoup.nodes.Element +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.model.* +import org.koitharu.kotatsu.parsers.util.* +import java.util.* + +@MangaSourceParser("MANGAREAD", "MangaRead", "en") +internal class MangaRead(context: MangaLoaderContext) : + MadaraParser(context, MangaSource.MANGAREAD, "www.mangaread.org") { + + override val tagPrefix = "genres/" + override val datePattern = "dd.MM.yyyy" + private val nsfwTags = arraySetOf("yaoi", "yuri", "mature") + + init { + paginator.firstPage = 1 + searchPaginator.firstPage = 1 + } + + override val sortOrders: Set = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.ALPHABETICAL, + SortOrder.RATING, + SortOrder.POPULARITY, + ) + + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + val url = urlBuilder() + .addPathSegment("page") + .addPathSegment(page.toString()) + .addQueryParameter("s", query.orEmpty()) + .addQueryParameter("post_type", "wp-manga") + tags?.forEachIndexed { index, tag -> + url.addQueryParameter("genre[$index]", tag.key) + } + url.addQueryParameter("op", "") + .addQueryParameter("author", "") + .addQueryParameter("artist", "") + .addQueryParameter("release", "") + .addQueryParameter("adult", "") + if (query.isNullOrEmpty()) { + url.addQueryParameter( + "&m_orderby=", + when (sortOrder) { + SortOrder.RATING -> "trending" + SortOrder.ALPHABETICAL -> "alphabet" + SortOrder.POPULARITY -> "views" + SortOrder.NEWEST -> "new-manga" + SortOrder.UPDATED -> "latest" + }, + ) + } + val root = webClient.httpGet(url.build()).parseHtml().body().selectFirstOrThrow(".search-wrap") + return root.select(".c-tabs-item__content").map { div -> + val a = div.selectFirstOrThrow("a") + val img = div.selectLastOrThrow("img") + val href = a.attrAsRelativeUrl("href") + val postContent = root.selectFirstOrThrow(".post-content") + val tagSet = postContent.getElementsContainingOwnText("Genre") + .firstOrNull()?.tableValue() + ?.getElementsByAttributeValueContaining("href", tagPrefix) + ?.mapToSet { it.asMangaTag() }.orEmpty() + Manga( + id = generateUid(href), + title = a.attr("title"), + altTitle = postContent.getElementsContainingOwnText("Alternative") + .firstOrNull()?.tableValue()?.text()?.trim(), + url = href, + publicUrl = a.attrAsAbsoluteUrl("href"), + coverUrl = img.src().orEmpty(), + author = postContent.getElementsContainingOwnText("Author") + .firstOrNull()?.tableValue()?.text()?.trim(), + state = postContent.getElementsContainingOwnText("Status") + .firstOrNull()?.tableValue()?.text()?.asMangaState(), + isNsfw = isNsfw(tagSet), + rating = div.selectFirstOrThrow(".score").text() + .toFloatOrNull()?.div(5f) ?: RATING_UNKNOWN, + tags = tagSet, + source = source, + ) + } + } + + private fun Element.tableValue(): Element { + for (p in parents()) { + val children = p.children() + if (children.size == 2) { + return children[1] + } + } + parseFailed("Cannot find tableValue for node ${text()}") + } + + private fun isNsfw(tags: Set): Boolean { + return tags.any { it.key in nsfwTags } + } + + private fun String.asMangaState() = when (trim().lowercase(sourceLocale)) { + "ongoing" -> MangaState.ONGOING + "completed" -> MangaState.FINISHED + else -> null + } + + private fun Element.asMangaTag() = MangaTag( + title = ownText(), + key = attr("href").removeSuffix('/').substringAfterLast('/') + .replace('-', '+'), + source = source, + ) +} diff --git a/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaParserTest.kt b/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaParserTest.kt index 9ea50a30..4c508ad5 100644 --- a/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaParserTest.kt +++ b/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaParserTest.kt @@ -36,6 +36,7 @@ internal class MangaParserTest { val page2 = parser.getList(page1.size, sortOrder = null, tags = null) assert(page1.isNotEmpty()) { "Page 1 is empty" } assert(page2.isNotEmpty()) { "Page 2 is empty" } + assert(page1 != page2) { "Pages are equal" } val intersection = page1.intersect(page2.toSet()) assert(intersection.isEmpty()) { "Pages are intersected by " + intersection.size From 306d46ea93ec8f4f61d3cb0190b39e4b1218ab6e Mon Sep 17 00:00:00 2001 From: Koitharu Date: Wed, 26 Apr 2023 09:24:39 +0300 Subject: [PATCH 112/121] [Grouple] Fix pages url extraction --- .../parsers/site/grouple/GroupleParser.kt | 52 ++++++++++++------- .../kotatsu/parsers/util/Coroutines.kt | 45 +++++++--------- .../kotatsu/parsers/MangaParserTest.kt | 18 ++++--- 3 files changed, 64 insertions(+), 51 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt index 40a2a0a2..cc85cd1d 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt @@ -1,9 +1,11 @@ package org.koitharu.kotatsu.parsers.site.grouple -import kotlinx.coroutines.async -import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.flow.channelFlow +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.launch import okhttp3.Headers import okhttp3.HttpUrl.Companion.toHttpUrl +import okhttp3.HttpUrl.Companion.toHttpUrlOrNull import okhttp3.Interceptor import okhttp3.Response import okhttp3.internal.headersContentLength @@ -236,30 +238,42 @@ internal abstract class GroupleParser( override suspend fun getPageUrl(page: MangaPage): String { val parts = page.url.split('|') + if (parts.size < 2) { + throw ParseException("No servers found for page", page.url) + } val path = parts.last() - val servers = parts.dropLast(1).toSet() - val cachedServer = cachedPagesServer - if (!cachedServer.isNullOrEmpty() && cachedServer in servers && tryHead(concatUrl(cachedServer, path))) { - return concatUrl(cachedServer, path) + // fast path + cachedPagesServer?.let { host -> + val url = concatUrl("https://$host/", path) + if (tryHead(url)) { + return url + } else { + cachedPagesServer = null + } } - if (servers.isEmpty()) { - throw ParseException("No servers found for page", page.url) + // slow path + val candidates = HashSet((parts.size - 1) * 2) + for (i in 0 until parts.size - 1) { + val server = parts[i].trim().ifEmpty { "https://$domain/" } + candidates.add(concatUrl(server, path)) + candidates.add(concatUrl(server, path.substringBeforeLast('?'))) } - val server = try { - coroutineScope { - servers.map { server -> - async { - val host = server.trim().ifEmpty { "https://$domain/" } - if (tryHead(concatUrl(host, path))) host else null + return try { + channelFlow { + for (url in candidates) { + launch { + if (tryHead(url)) { + send(url) + } } - }.awaitFirst { it != null } - }.also { - cachedPagesServer = it + } + }.first().also { + cachedPagesServer = it.toHttpUrlOrNull()?.host } } catch (e: NoSuchElementException) { - servers.random() + assert(false) { e.toString() } + candidates.random() } - return concatUrl(checkNotNull(server).ifEmpty { "https://$domain/" }, path) } override suspend fun getTags(): Set { diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Coroutines.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Coroutines.kt index 52fb8b17..fffb94f2 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Coroutines.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Coroutines.kt @@ -2,39 +2,34 @@ package org.koitharu.kotatsu.parsers.util import kotlinx.coroutines.Deferred import kotlinx.coroutines.Job -import kotlinx.coroutines.selects.select +import kotlinx.coroutines.flow.channelFlow +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.launch import kotlin.coroutines.cancellation.CancellationException fun Iterable.cancelAll(cause: CancellationException? = null) { forEach { it.cancel(cause) } } -suspend fun Iterable>.awaitFirst(): T = select { - for (async in this@awaitFirst) { - async.onAwait { it } - } -}.also { this@awaitFirst.cancelAll() } +suspend fun Iterable>.awaitFirst(): T { + return channelFlow { + for (deferred in this@awaitFirst) { + launch { + send(deferred.await()) + } + } + }.first().also { this@awaitFirst.cancelAll() } +} suspend fun Collection>.awaitFirst(condition: (T) -> Boolean): T { - var result: Any? = NULL - var counter = size - while (result === NULL && counter > 0) { - val candidate = select { - for (async in this@awaitFirst) { - async.onAwait { it } + return channelFlow { + for (deferred in this@awaitFirst) { + launch { + val result = deferred.await() + if (condition(result)) { + send(result) + } } } - if (condition(candidate)) { - result = candidate - } - counter-- - } - cancelAll() - if (result === NULL) { - throw NoSuchElementException() - } - @Suppress("UNCHECKED_CAST") - return result as T + }.first().also { this@awaitFirst.cancelAll() } } - -private val NULL = Any() diff --git a/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaParserTest.kt b/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaParserTest.kt index 4c508ad5..84d7343a 100644 --- a/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaParserTest.kt +++ b/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaParserTest.kt @@ -114,20 +114,24 @@ internal class MangaParserTest { @MangaSources fun pages(source: MangaSource) = runTest { val parser = source.newParser(context) - val list = parser.getList(0, sortOrder = SortOrder.POPULARITY, tags = null) + val list = parser.getList(0, sortOrder = SortOrder.UPDATED, tags = null) val manga = list.first() - val chapter = parser.getDetails(manga).chapters?.firstOrNull() ?: error("Chapter is null") + val chapter = parser.getDetails(manga).chapters?.firstOrNull() ?: error("Chapter is null at ${manga.publicUrl}") val pages = parser.getPages(chapter) assert(pages.isNotEmpty()) assert(pages.isDistinctBy { it.id }) assert(pages.all { it.source == source }) - val page = pages.medianOrNull() ?: error("No page") - val pageUrl = parser.getPageUrl(page) - assert(pageUrl.isNotEmpty()) - assert(pageUrl.isUrlAbsolute()) - checkImageRequest(pageUrl, page.source) + arrayOf( + pages.first(), + pages.medianOrNull() ?: error("No page"), + ).forEach { page -> + val pageUrl = parser.getPageUrl(page) + assert(pageUrl.isNotEmpty()) + assert(pageUrl.isUrlAbsolute()) + checkImageRequest(pageUrl, page.source) + } } @ParameterizedTest(name = "{index}|favicon|{0}") From b4a4c860494371daa80f2412e06b2f037f5521d3 Mon Sep 17 00:00:00 2001 From: CakesTwix Date: Wed, 26 Apr 2023 16:35:42 +0300 Subject: [PATCH 113/121] [HoneyManga] Impl. filter by genres and fix cover url --- .../kotatsu/parsers/site/HoneyMangaParser.kt | 47 +++++++++++++------ 1 file changed, 33 insertions(+), 14 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/HoneyMangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/HoneyMangaParser.kt index ce6de408..a7057cc1 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/HoneyMangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/HoneyMangaParser.kt @@ -1,4 +1,4 @@ -package org.koitharu.kotatsu.core.parser +package org.koitharu.kotatsu.parsers.site import androidx.collection.ArraySet import okhttp3.Headers @@ -102,21 +102,39 @@ class HoneyMangaParser(context: MangaLoaderContext) : PagedMangaParser(context, ): List { val body = JSONObject() var content: JSONArray = JSONArray() - + body.put("page", page) + body.put("pageSize", PAGE_SIZE) + val sort = JSONObject() + sort.put("sortBy", getSortKey(sortOrder)) + sort.put("sortOrder", "DESC") + body.put("sort", sort) + + // Tags + content = if (!tags.isNullOrEmpty()) { + val filters: JSONArray = JSONArray() + val tag_filter = JSONObject() + tag_filter.put("filterBy", "genres") + tag_filter.put("filterOperator", "ALL") + val tag: JSONArray = JSONArray() + tags.forEach { + tag.put(it.title) + } + tag_filter.put("filterValue", tag) + filters.put(tag_filter) + body.put("filters", filters) + webClient.httpPost(mangaApi, body).parseJson().getJSONArray("data") + // Search + } else if(!query.isNullOrEmpty()) { + if((query.length < 3)) throw IllegalArgumentException("Запит має містити щонайменше 3 символи / The query must contain at least 3 characters") + if(page == 1) { + webClient.httpGet(searchApi + query.urlEncoded()).parseJsonArray() + } else { + JSONArray() + } // Popular/Newest - content = if (query.isNullOrEmpty()) { - body.put("page", page) - body.put("pageSize", PAGE_SIZE) + } else { body.put("filters", JSONArray()) - val sort = JSONObject() - sort.put("sortBy", getSortKey(sortOrder)) - sort.put("sortOrder", "DESC") - body.put("sort", sort) webClient.httpPost(mangaApi, body).parseJson().getJSONArray("data") - // Search - } else { - if((query.length < 3) || (page > 1)) throw IllegalArgumentException("Запит має містити щонайменше 3 символи / The query must contain at least 3 characters") - webClient.httpGet(searchApi + query.urlEncoded()).parseJsonArray() } return content.mapJSON { jo -> val id = jo.getString("id") @@ -186,7 +204,8 @@ class HoneyMangaParser(context: MangaLoaderContext) : PagedMangaParser(context, } private fun getCoverUrl(id: String, w: Int): String { - return "https://$domain/_next/image?url=https%3A%2F%2Fmanga-storage.fra1.digitaloceanspaces.com%2Fpublic-resources%2F$id&w=$w&q=75" + // https://honey-manga.com.ua/_next/image?url=https%3A%2F%2Fhoneymangastorage.b-cdn.net%2Fpublic-resources%2F1c4613c2-ffe3-405a-b26a-2cab59ddd223%3Foptimizer%3Dimage%26width%3D512%26height%3D512&w=3840&q=75 + return "https://$domain/_next/image?url=https%3A%2F%2Fhoneymangastorage.b-cdn.net%2Fpublic-resources%2F$id&w=$w&q=75" } private fun getSortKey(order: SortOrder?) = when (order) { From 0d03c68c8bd18e5f476a9891c695cf5a29d019ce Mon Sep 17 00:00:00 2001 From: Koitharu Date: Fri, 28 Apr 2023 15:20:00 +0300 Subject: [PATCH 114/121] [Manhwaland] Fix list url --- .../kotatsu/parsers/site/mangareader/MangaReaderParser.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt index 0a7015f5..17f41d05 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt @@ -234,7 +234,7 @@ internal abstract class MangaReaderParser( get() = ConfigKey.Domain("manhwaland.us", arrayOf("manhwaland.us", "manhwaland.guru")) override val listUrl: String - get() = "/series" + get() = "/manga" override val tableMode: Boolean get() = false } From f43c4825397243a9339c9820d9c27286bcc10887 Mon Sep 17 00:00:00 2001 From: Koitharu Date: Fri, 28 Apr 2023 15:31:03 +0300 Subject: [PATCH 115/121] [Honeymanga] Refactoring --- .../kotatsu/parsers/site/HoneyMangaParser.kt | 122 +++++++++--------- .../koitharu/kotatsu/parsers/MangaSources.kt | 2 +- 2 files changed, 60 insertions(+), 64 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/HoneyMangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/HoneyMangaParser.kt index a7057cc1..eea9c81c 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/HoneyMangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/HoneyMangaParser.kt @@ -10,33 +10,22 @@ import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaSourceParser import org.koitharu.kotatsu.parsers.PagedMangaParser import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.model.Manga -import org.koitharu.kotatsu.parsers.model.MangaChapter -import org.koitharu.kotatsu.parsers.model.MangaPage -import org.koitharu.kotatsu.parsers.model.MangaSource -import org.koitharu.kotatsu.parsers.model.MangaState -import org.koitharu.kotatsu.parsers.model.MangaTag -import org.koitharu.kotatsu.parsers.model.RATING_UNKNOWN -import org.koitharu.kotatsu.parsers.model.SortOrder -import org.koitharu.kotatsu.parsers.util.domain -import org.koitharu.kotatsu.parsers.util.generateUid +import org.koitharu.kotatsu.parsers.model.* +import org.koitharu.kotatsu.parsers.util.* import org.koitharu.kotatsu.parsers.util.json.getStringOrNull import org.koitharu.kotatsu.parsers.util.json.mapJSON import org.koitharu.kotatsu.parsers.util.json.mapJSONIndexed -import org.koitharu.kotatsu.parsers.util.parseJson -import org.koitharu.kotatsu.parsers.util.parseJsonArray -import org.koitharu.kotatsu.parsers.util.removeSuffix -import org.koitharu.kotatsu.parsers.util.tryParse -import org.koitharu.kotatsu.parsers.util.urlEncoded import java.text.SimpleDateFormat -import java.util.EnumSet -import java.util.Locale +import java.util.* private const val PAGE_SIZE = 20 +private const val INFINITE = 999999 +private const val HEADER_ENCODING = "Content-Encoding" @MangaSourceParser("HONEYMANGA", "Honey Manga", "uk") -class HoneyMangaParser(context: MangaLoaderContext) : PagedMangaParser(context, MangaSource.HONEYMANGA, PAGE_SIZE), Interceptor { +class HoneyMangaParser(context: MangaLoaderContext) : PagedMangaParser(context, MangaSource.HONEYMANGA, PAGE_SIZE), + Interceptor { private val urlApi get() = "https://data.api.$domain" private val mangaApi get() = "$urlApi/v2/manga/cursor-list" @@ -64,7 +53,7 @@ class HoneyMangaParser(context: MangaLoaderContext) : PagedMangaParser(context, val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.US) val body = JSONObject() body.put("mangaId", manga.url) - body.put("pageSize", 999999) // Hack lol (no) + body.put("pageSize", INFINITE) // Hack lol (no) body.put("page", 1) body.put("sortOrder", "ASC") val chapterRequest = webClient.httpPost(chapterApi, body).parseJson() @@ -88,9 +77,9 @@ class HoneyMangaParser(context: MangaLoaderContext) : PagedMangaParser(context, scanlator = null, uploadDate = dateFormat.tryParse(jo.getString("lastUpdated")), branch = null, - source = source + source = source, ) - } + }, ) } @@ -98,10 +87,9 @@ class HoneyMangaParser(context: MangaLoaderContext) : PagedMangaParser(context, page: Int, query: String?, tags: Set?, - sortOrder: SortOrder + sortOrder: SortOrder, ): List { val body = JSONObject() - var content: JSONArray = JSONArray() body.put("page", page) body.put("pageSize", PAGE_SIZE) val sort = JSONObject() @@ -109,32 +97,44 @@ class HoneyMangaParser(context: MangaLoaderContext) : PagedMangaParser(context, sort.put("sortOrder", "DESC") body.put("sort", sort) - // Tags - content = if (!tags.isNullOrEmpty()) { - val filters: JSONArray = JSONArray() - val tag_filter = JSONObject() - tag_filter.put("filterBy", "genres") - tag_filter.put("filterOperator", "ALL") - val tag: JSONArray = JSONArray() - tags.forEach { - tag.put(it.title) + val content = when { + !tags.isNullOrEmpty() -> { + // Tags + val filters = JSONArray() + val tagFilter = JSONObject() + tagFilter.put("filterBy", "genres") + tagFilter.put("filterOperator", "ALL") + val tag = JSONArray() + tags.forEach { + tag.put(it.title) + } + tagFilter.put("filterValue", tag) + filters.put(tagFilter) + body.put("filters", filters) + webClient.httpPost(mangaApi, body).parseJson().getJSONArray("data") + } - tag_filter.put("filterValue", tag) - filters.put(tag_filter) - body.put("filters", filters) - webClient.httpPost(mangaApi, body).parseJson().getJSONArray("data") - // Search - } else if(!query.isNullOrEmpty()) { - if((query.length < 3)) throw IllegalArgumentException("Запит має містити щонайменше 3 символи / The query must contain at least 3 characters") - if(page == 1) { - webClient.httpGet(searchApi + query.urlEncoded()).parseJsonArray() - } else { - JSONArray() + + !query.isNullOrEmpty() -> { + // Search + when { + query.length < 3 -> throw IllegalArgumentException( + "The query must contain at least 3 characters (Запит має містити щонайменше 3 символи)", + ) + + page == searchPaginator.firstPage -> webClient + .httpGet(searchApi + query.urlEncoded()) + .parseJsonArray() + + else -> JSONArray() + } + } + + else -> { + // Popular/Newest + body.put("filters", JSONArray()) + webClient.httpPost(mangaApi, body).parseJson().getJSONArray("data") } - // Popular/Newest - } else { - body.put("filters", JSONArray()) - webClient.httpPost(mangaApi, body).parseJson().getJSONArray("data") } return content.mapJSON { jo -> val id = jo.getString("id") @@ -166,32 +166,29 @@ class HoneyMangaParser(context: MangaLoaderContext) : PagedMangaParser(context, override suspend fun getPages(chapter: MangaChapter): List { val body = JSONObject() body.put("chapterId", chapter.url) - val content = webClient.httpPost(framesApi, body).parseJson().getJSONObject("resourceIds") return List(content.length()) { i -> - val item = content.get(i.toString()).toString() + val item = content.getString(i.toString()) MangaPage(id = generateUid(item), "$imageStorageUrl/$item", getCoverUrl(item, 256), source) } } override suspend fun getTags(): Set { // https://data.api.honey-manga.com.ua/genres-tags/genres-list - val tagsSet = mutableListOf() val content = webClient.httpGet(genresListApi).parseJsonArray() - (0 until content.length()).forEach { i -> - val item = content.get(i).toString() - + val tagsSet = ArraySet(content.length()) + repeat(content.length()) { i -> + val item = content.getString(i) tagsSet.add(MangaTag(item, item, source)) } - - return tagsSet.toSet() + return tagsSet } // Need for disable encoding (with encoding not working) override fun intercept(chain: Interceptor.Chain): Response { val request = chain.request() - val newRequest = if (request.header("Content-Encoding") != null) { - request.newBuilder().removeHeader("Content-Encoding").build() + val newRequest = if (request.header(HEADER_ENCODING) != null) { + request.newBuilder().removeHeader(HEADER_ENCODING).build() } else { request } @@ -214,14 +211,13 @@ class HoneyMangaParser(context: MangaLoaderContext) : PagedMangaParser(context, else -> "likes" } - private fun getTitleTags(jsonTags: JSONArray): Set { + private fun getTitleTags(jsonTags: JSONArray): Set { val tagsSet = ArraySet(jsonTags.length()) - (0 until jsonTags.length()).forEach { i -> - val item = jsonTags.get(i) + repeat(jsonTags.length()) { i -> + val item = jsonTags.getString(i) - tagsSet.add(MangaTag(item.toString(), item.toString(), source)) + tagsSet.add(MangaTag(title = item.toTitleCase(sourceLocale), key = item, source = source)) } - - return tagsSet.toSet() + return tagsSet } } diff --git a/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaSources.kt b/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaSources.kt index ef660faa..2ca95f83 100644 --- a/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaSources.kt +++ b/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaSources.kt @@ -3,5 +3,5 @@ package org.koitharu.kotatsu.parsers import org.junit.jupiter.params.provider.EnumSource import org.koitharu.kotatsu.parsers.model.MangaSource -@EnumSource(MangaSource::class, names = ["LOCAL", "DUMMY"], mode = EnumSource.Mode.EXCLUDE) +@EnumSource(MangaSource::class, names = ["HONEYMANGA"], mode = EnumSource.Mode.INCLUDE) internal annotation class MangaSources From 7414cf4a4ea843b9c131cfd7d05e38d7a6d2abb2 Mon Sep 17 00:00:00 2001 From: Koitharu Date: Fri, 28 Apr 2023 15:43:13 +0300 Subject: [PATCH 116/121] Add @JvmField on models fields --- .../kotatsu/parsers/config/ConfigKey.kt | 4 +-- .../koitharu/kotatsu/parsers/model/Favicon.kt | 9 +++--- .../kotatsu/parsers/model/Favicons.kt | 4 +-- .../koitharu/kotatsu/parsers/model/Manga.kt | 30 +++++++++---------- .../kotatsu/parsers/model/MangaChapter.kt | 16 +++++----- .../kotatsu/parsers/model/MangaPage.kt | 8 ++--- .../kotatsu/parsers/model/MangaTag.kt | 6 ++-- 7 files changed, 39 insertions(+), 38 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/config/ConfigKey.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/config/ConfigKey.kt index 8603ac62..62b8107b 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/config/ConfigKey.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/config/ConfigKey.kt @@ -1,14 +1,14 @@ package org.koitharu.kotatsu.parsers.config sealed class ConfigKey( - val key: String, + @JvmField val key: String, ) { abstract val defaultValue: T class Domain( override val defaultValue: String, - val presetValues: Array?, + @JvmField val presetValues: Array?, ) : ConfigKey("domain") class ShowSuspiciousContent( diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/model/Favicon.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/model/Favicon.kt index 7256524e..41e092ff 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/model/Favicon.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/model/Favicon.kt @@ -3,11 +3,12 @@ package org.koitharu.kotatsu.parsers.model import okhttp3.HttpUrl.Companion.toHttpUrl class Favicon internal constructor( - val url: String, - val size: Int, - internal val rel: String?, + @JvmField val url: String, + @JvmField val size: Int, + @JvmField internal val rel: String?, ) : Comparable { + @JvmField val type: String = url.toHttpUrl().pathSegments.last() .substringAfterLast('.', "").lowercase() @@ -48,4 +49,4 @@ class Favicon internal constructor( "mask-icon" -> -1 else -> 0 } -} \ No newline at end of file +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/model/Favicons.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/model/Favicons.kt index f9c9aab7..6c37fe4f 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/model/Favicons.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/model/Favicons.kt @@ -2,7 +2,7 @@ package org.koitharu.kotatsu.parsers.model class Favicons internal constructor( favicons: Collection, - val referer: String, + @JvmField val referer: String, ) : Collection { private val icons = favicons.sortedDescending() @@ -42,4 +42,4 @@ class Favicons internal constructor( } return result } -} \ No newline at end of file +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/model/Manga.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/model/Manga.kt index 0f45a992..afda6dc8 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/model/Manga.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/model/Manga.kt @@ -6,67 +6,67 @@ class Manga( /** * Unique identifier for manga */ - val id: Long, + @JvmField val id: Long, /** * Manga title, human-readable */ - val title: String, + @JvmField val title: String, /** * Alternative title (for example on other language), may be null */ - val altTitle: String?, + @JvmField val altTitle: String?, /** * Relative url to manga (**without** a domain) or any other uri. * Used principally in parsers */ - val url: String, + @JvmField val url: String, /** * Absolute url to manga, must be ready to open in browser */ - val publicUrl: String, + @JvmField val publicUrl: String, /** * Normalized manga rating, must be in range of 0..1 or [RATING_UNKNOWN] if rating s unknown * @see hasRating */ - val rating: Float, + @JvmField val rating: Float, /** * Indicates that manga may contain sensitive information (18+, NSFW) */ - val isNsfw: Boolean, + @JvmField val isNsfw: Boolean, /** * Absolute link to the cover * @see largeCoverUrl */ - val coverUrl: String, + @JvmField val coverUrl: String, /** * Tags (genres) of the manga */ - val tags: Set, + @JvmField val tags: Set, /** * Manga status (ongoing, finished) or null if unknown */ - val state: MangaState?, + @JvmField val state: MangaState?, /** * Author of the manga, may be null */ - val author: String?, + @JvmField val author: String?, /** * Large cover url (absolute), null if is no large cover * @see coverUrl */ - val largeCoverUrl: String? = null, + @JvmField val largeCoverUrl: String? = null, /** * Manga description, may be html or null */ - val description: String? = null, + @JvmField val description: String? = null, /** * List of chapters */ - val chapters: List? = null, + @JvmField val chapters: List? = null, /** * Manga source */ - val source: MangaSource, + @JvmField val source: MangaSource, ) { /** diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/model/MangaChapter.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/model/MangaChapter.kt index 4059be66..21e6c6e6 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/model/MangaChapter.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/model/MangaChapter.kt @@ -4,34 +4,34 @@ class MangaChapter( /** * An unique id of chapter */ - val id: Long, + @JvmField val id: Long, /** * User-readable name of chapter */ - val name: String, + @JvmField val name: String, /** * Chapter number starting from 1 */ - val number: Int, + @JvmField val number: Int, /** * Relative url to chapter (**without** a domain) or any other uri. * Used principally in parsers */ - val url: String, + @JvmField val url: String, /** * User-readable name of scanlator (releaser) or null if unknown */ - val scanlator: String?, + @JvmField val scanlator: String?, /** * Chapter upload date in milliseconds */ - val uploadDate: Long, + @JvmField val uploadDate: Long, /** * User-readable name of branch. * A branch is a group of chapters that overlap (e.g. different languages) */ - val branch: String?, - val source: MangaSource, + @JvmField val branch: String?, + @JvmField val source: MangaSource, ) : Comparable { override fun compareTo(other: MangaChapter): Int { diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/model/MangaPage.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/model/MangaPage.kt index b741fac3..2c463761 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/model/MangaPage.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/model/MangaPage.kt @@ -6,19 +6,19 @@ class MangaPage( /** * Unique identifier for manga */ - val id: Long, + @JvmField val id: Long, /** * Relative url to page (**without** a domain) or any other uri. * Used principally in parsers. * May contain link to image or html page. * @see MangaParser.getPageUrl */ - val url: String, + @JvmField val url: String, /** * Absolute url of the small page image if exists, null otherwise */ - val preview: String?, - val source: MangaSource, + @JvmField val preview: String?, + @JvmField val source: MangaSource, ) { override fun equals(other: Any?): Boolean { diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/model/MangaTag.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/model/MangaTag.kt index be0f9083..184a850e 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/model/MangaTag.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/model/MangaTag.kt @@ -6,13 +6,13 @@ class MangaTag( /** * User-readable tag title, should be in Title case */ - val title: String, + @JvmField val title: String, /** * Identifier of a tag, must be unique among the source. * @see MangaParser.getList */ - val key: String, - val source: MangaSource, + @JvmField val key: String, + @JvmField val source: MangaSource, ) { override fun equals(other: Any?): Boolean { From b2675c3caca175b754698a1bbc30743a1af3568a Mon Sep 17 00:00:00 2001 From: Koitharu Date: Fri, 28 Apr 2023 16:17:47 +0300 Subject: [PATCH 117/121] Fix @MangaSources annotation --- src/test/kotlin/org/koitharu/kotatsu/parsers/MangaSources.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaSources.kt b/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaSources.kt index 2ca95f83..ef660faa 100644 --- a/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaSources.kt +++ b/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaSources.kt @@ -3,5 +3,5 @@ package org.koitharu.kotatsu.parsers import org.junit.jupiter.params.provider.EnumSource import org.koitharu.kotatsu.parsers.model.MangaSource -@EnumSource(MangaSource::class, names = ["HONEYMANGA"], mode = EnumSource.Mode.INCLUDE) +@EnumSource(MangaSource::class, names = ["LOCAL", "DUMMY"], mode = EnumSource.Mode.EXCLUDE) internal annotation class MangaSources From 3e349d3db34bcfa7cba60a9b178642831d362801 Mon Sep 17 00:00:00 2001 From: Koitharu Date: Fri, 28 Apr 2023 17:21:19 +0300 Subject: [PATCH 118/121] [Ninemanga] Fix redirects --- .../kotatsu/parsers/site/NineMangaParser.kt | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NineMangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NineMangaParser.kt index 12c40dc3..cfffccbf 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NineMangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NineMangaParser.kt @@ -2,6 +2,9 @@ package org.koitharu.kotatsu.parsers.site import okhttp3.Headers import okhttp3.HttpUrl.Companion.toHttpUrl +import okhttp3.Interceptor +import okhttp3.Response +import okio.IOException import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaSourceParser import org.koitharu.kotatsu.parsers.PagedMangaParser @@ -15,7 +18,7 @@ internal abstract class NineMangaParser( context: MangaLoaderContext, source: MangaSource, defaultDomain: String, -) : PagedMangaParser(context, source, pageSize = 26) { +) : PagedMangaParser(context, source, pageSize = 26), Interceptor { override val configKeyDomain = ConfigKey.Domain(defaultDomain, null) @@ -25,12 +28,26 @@ internal abstract class NineMangaParser( override val headers = Headers.Builder() .add("Accept-Language", "en-US;q=0.7,en;q=0.3") + .add("User-Agent", "Mozilla/5.0 (Android 13; Mobile; rv:68.0) Gecko/68.0 Firefox/109.0") .build() override val sortOrders: Set = Collections.singleton( SortOrder.POPULARITY, ) + override fun intercept(chain: Interceptor.Chain): Response { + val request = chain.request() + .newBuilder() + .removeHeader("Referer") + .build() + val response = chain.proceed(request) + val responseUrl = response.request.url + if (responseUrl.host != request.url.host) { + throw IOException("Unexpected redirect to ${responseUrl.host}") + } + return response + } + override suspend fun getListPage( page: Int, query: String?, From 65170c0fe92e795ccd14882fb4057bb6a7f5020e Mon Sep 17 00:00:00 2001 From: Koitharu Date: Mon, 1 May 2023 09:03:11 +0300 Subject: [PATCH 119/121] [Mangadex] Fix chapters numbers --- .../org/koitharu/kotatsu/parsers/site/MangaDexParser.kt | 4 ++-- .../kotlin/org/koitharu/kotatsu/parsers/util/json/JsonExt.kt | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaDexParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaDexParser.kt index 96b795ef..db033c80 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaDexParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaDexParser.kt @@ -150,12 +150,12 @@ internal class MangaDexParser(context: MangaLoaderContext) : MangaParser(context } val locale = attrs.getStringOrNull("translatedLanguage")?.let { Locale.forLanguageTag(it) } val relations = jo.getJSONArray("relationships").associateByKey("type") - val number = attrs.getIntOrDefault("chapter", 0) + val number = attrs.getFloatOrDefault("chapter", 0f) MangaChapter( id = generateUid(id), name = attrs.getStringOrNull("title")?.takeUnless(String::isEmpty) ?: "Chapter #$number", - number = number, + number = number.toInt(), url = id, scanlator = relations["scanlation_group"]?.getStringOrNull("name"), uploadDate = dateFormat.tryParse(attrs.getString("publishAt")), diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/json/JsonExt.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/json/JsonExt.kt index 9e429b3c..7a994a84 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/json/JsonExt.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/json/JsonExt.kt @@ -140,4 +140,4 @@ fun JSONArray?.isNullOrEmpty(): Boolean { fun JSONArray.toJSONList(): List { return List(length()) { i -> getJSONObject(i) } -} \ No newline at end of file +} From 96b9ac36f3d3446115e2b73a515756b63b5e87e4 Mon Sep 17 00:00:00 2001 From: Koitharu Date: Sat, 6 May 2023 17:28:34 +0300 Subject: [PATCH 120/121] [Bentomanga] Fixes --- .../koitharu/kotatsu/parsers/site/BentomangaParser.kt | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BentomangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BentomangaParser.kt index 1dc2a11d..f435c60c 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BentomangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BentomangaParser.kt @@ -9,8 +9,10 @@ import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaSourceParser import org.koitharu.kotatsu.parsers.PagedMangaParser import org.koitharu.kotatsu.parsers.config.ConfigKey +import org.koitharu.kotatsu.parsers.exception.ParseException import org.koitharu.kotatsu.parsers.model.* import org.koitharu.kotatsu.parsers.util.* +import org.koitharu.kotatsu.parsers.util.json.getIntOrDefault import java.util.* @MangaSourceParser("BENTOMANGA", "Bentomanga", "fr") @@ -110,7 +112,7 @@ internal class BentomangaParser(context: MangaLoaderContext) : PagedMangaParser( }, author = root.selectFirst(".datas_more-authors-people")?.textOrNull().assertNotNull("author"), chapters = run { - val input = root.selectFirst("input[name=\"limit\"]") ?: return@run emptyList() + val input = root.selectFirst("input[name=\"limit\"]") ?: return@run parseChapters(root) val max = input.attr("max").toInt() if (max <= 1) { parseChapters(root) @@ -142,6 +144,9 @@ internal class BentomangaParser(context: MangaLoaderContext) : PagedMangaParser( "x-requested-with", "XMLHttpRequest", ), ).parseJson() + if (json.getIntOrDefault("type", 1) == 2) { + throw ParseException("Light Novels are not supported", chapterUrl) + } val baseUrl = json.getString("baseImagesUrl") val pages = json.getJSONArray("page_array") return (0 until pages.length()).map { i -> @@ -186,7 +191,7 @@ internal class BentomangaParser(context: MangaLoaderContext) : PagedMangaParser( name = if (name != null && name != title) "$title: $name" else title, number = href.substringAfterLast('/').toIntOrNull() ?: 0, url = href, - scanlator = div.selectFirst(".team_link-name")?.textOrNull().assertNotNull("scanlator"), + scanlator = div.selectFirst(".team_link-name")?.textOrNull(), uploadDate = div.selectFirst(".component-chapter-date") ?.ownTextOrNull() .parseDate(), From 20ad9d0472e356daa8506755a553983153c107b4 Mon Sep 17 00:00:00 2001 From: Koitharu Date: Sat, 13 May 2023 14:39:37 +0300 Subject: [PATCH 121/121] [ALlHentai] New parser --- .../parsers/site/grouple/AllHentaiParser.kt | 53 +++++++++++++++++++ .../parsers/site/grouple/GroupleParser.kt | 5 +- 2 files changed, 56 insertions(+), 2 deletions(-) create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/AllHentaiParser.kt diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/AllHentaiParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/AllHentaiParser.kt new file mode 100644 index 00000000..77808eff --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/AllHentaiParser.kt @@ -0,0 +1,53 @@ +package org.koitharu.kotatsu.parsers.site.grouple + +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.config.ConfigKey +import org.koitharu.kotatsu.parsers.exception.AuthRequiredException +import org.koitharu.kotatsu.parsers.exception.ParseException +import org.koitharu.kotatsu.parsers.model.MangaChapter +import org.koitharu.kotatsu.parsers.model.MangaPage +import org.koitharu.kotatsu.parsers.model.MangaSource +import org.koitharu.kotatsu.parsers.util.domain +import org.koitharu.kotatsu.parsers.util.parseFailed +import org.koitharu.kotatsu.parsers.util.parseHtml +import org.koitharu.kotatsu.parsers.util.urlEncoded + +@MangaSourceParser("ALLHENTAI", "ALlHentai", "ru") +internal class AllHentaiParser( + context: MangaLoaderContext, +) : GroupleParser(context, MangaSource.ALLHENTAI, 1) { + + override val configKeyDomain = ConfigKey.Domain( + "2023.allhen.online", + null, + ) + override val defaultIsNsfw = true + + override val authUrl: String + get() { + val targetUri = "https://${domain}/".urlEncoded() + return "https://qawa.org/internal/auth/login?targetUri=$targetUri&siteId=1" + } + + override suspend fun getPages(chapter: MangaChapter): List { + try { + return super.getPages(chapter) + } catch (e: ParseException) { + if (isAuthorized) { + throw e + } else { + throw AuthRequiredException(source) + } + } + } + + override suspend fun getUsername(): String { + val root = webClient.httpGet("https://qawa.org/").parseHtml().body() + val element = root.selectFirst("img.user-avatar") ?: throw AuthRequiredException(source) + val res = element.parent()?.text() + return if (res.isNullOrEmpty()) { + root.parseFailed("Cannot find username") + } else res + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt index cc85cd1d..12ab60b0 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt @@ -37,6 +37,7 @@ internal abstract class GroupleParser( @Volatile private var cachedPagesServer: String? = null + protected open val defaultIsNsfw = false private val userAgentKey = ConfigKey.UserAgent( "Mozilla/5.0 (X11; U; UNICOS lcLinux; en-US) Gecko/20140730 (KHTML, like Gecko, Safari/419.3) Arora/0.8.0", @@ -131,7 +132,7 @@ internal abstract class GroupleParser( ?.div(5f) }.getOrNull() ?: RATING_UNKNOWN, author = tileInfo?.selectFirst("a.person-link")?.text(), - isNsfw = false, + isNsfw = defaultIsNsfw, tags = runCatching { tileInfo?.select("a.element-link") ?.mapToSet { @@ -173,7 +174,7 @@ internal abstract class GroupleParser( ) }, author = root.selectFirst("a.person-link")?.text() ?: manga.author, - isNsfw = root.select(".alert-warning").any { it.ownText().contains(NSFW_ALERT) }, + isNsfw = manga.isNsfw || root.select(".alert-warning").any { it.ownText().contains(NSFW_ALERT) }, chapters = root.selectFirst("div.chapters-link")?.selectFirst("table") ?.select("tr:has(td > a)")?.mapChapters(reversed = true) { i, tr -> val a = tr.selectFirst("a.chapter-link") ?: return@mapChapters null