From b49aee318840ae04e3e71695e6102ea5ee44179c Mon Sep 17 00:00:00 2001 From: vianh Date: Sun, 16 Apr 2023 22:52:22 +0700 Subject: [PATCH 1/2] [DoujinDesu] New parser --- .../kotatsu/parsers/site/DoujinDesuParser.kt | 134 ++++++++++++++++++ 1 file changed, 134 insertions(+) create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/DoujinDesuParser.kt diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/DoujinDesuParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/DoujinDesuParser.kt new file mode 100644 index 00000000..04746949 --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/DoujinDesuParser.kt @@ -0,0 +1,134 @@ +package org.koitharu.kotatsu.parsers.site + +import org.koitharu.kotatsu.parsers.InternalParsersApi +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.PagedMangaParser +import org.koitharu.kotatsu.parsers.config.ConfigKey +import org.koitharu.kotatsu.parsers.model.* +import org.koitharu.kotatsu.parsers.util.* +import java.text.SimpleDateFormat +import java.util.* + +@MangaSourceParser("DOUJINDESU", "DoujinDesu", "id") +class DoujinDesuParser(context: MangaLoaderContext) : PagedMangaParser(context, MangaSource.DOUJINDESU, pageSize = 18) { + @InternalParsersApi + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("212.32.226.234", null) + + override val sortOrders: Set + get() = EnumSet.of(SortOrder.UPDATED, SortOrder.NEWEST, SortOrder.ALPHABETICAL, SortOrder.POPULARITY) + + override suspend fun getDetails(manga: Manga): Manga { + val docs = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml().selectFirstOrThrow("#archive") + val metadataEl = docs.selectFirst(".wrapper > .metadata tbody") + val state = when (metadataEl?.selectFirst("tr:contains(Status)")?.selectLast("td")?.text()) { + "Finished" -> MangaState.FINISHED + "Publishing" -> MangaState.ONGOING + else -> null + } + return manga.copy( + author = metadataEl?.selectFirst("tr:contains(Author)")?.selectLast("td")?.text(), + description = docs.selectFirst(".wrapper > .metadata > .pb-2")?.selectFirst("p")?.html(), + state = state, + rating = metadataEl?.selectFirst(".rating-prc")?.ownText()?.toFloatOrNull()?.div(10f) ?: RATING_UNKNOWN, + tags = docs.select(".tags > a").mapToSet { + MangaTag( + key = it.attr("title"), + title = it.text(), + source = source, + ) + }, + chapters = docs.select("#chapter_list > ul > li").mapChapters(reversed = true) { index, element -> + val titleTag = element.selectFirstOrThrow(".epsleft > .lchx > a") + val dateFormat = SimpleDateFormat("EEEE, dd MMMM yyyy", Locale("in")) + MangaChapter( + id = generateUid(titleTag.attrAsRelativeUrl("href")), + name = titleTag.text(), + number = index + 1, + url = titleTag.attrAsRelativeUrl("href"), + scanlator = null, + uploadDate = dateFormat.tryParse(element.select(".epsleft > .date").text()), + branch = null, + source = source, + ) + }, + ) + } + + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder + ): List { + val url = urlBuilder().apply { + addPathSegment("manga") + addPathSegment("page") + addPathSegment("$page/") + val order = when (sortOrder) { + SortOrder.UPDATED -> "update" + SortOrder.POPULARITY -> "popular" + SortOrder.ALPHABETICAL -> "title" + SortOrder.NEWEST -> "latest" + else -> throw IllegalArgumentException("Sort order not supported") + } + addQueryParameter("order", order) + addQueryParameter("title", query.orEmpty()) + tags?.forEach { + addEncodedQueryParameter("genre[]".urlEncoded(), it.key.urlEncoded()) + } + }.build() + + return webClient.httpGet(url).parseHtml().select("#archives > div.entries > .entry") + .map { + val titleTag = it.selectFirstOrThrow(".metadata > a") + val relativeUrl = titleTag.attrAsRelativeUrl("href") + Manga( + id = generateUid(relativeUrl), + title = titleTag.attr("title"), + altTitle = null, + url = relativeUrl, + publicUrl = relativeUrl.toAbsoluteUrl(domain), + rating = RATING_UNKNOWN, + isNsfw = true, + coverUrl = it.selectFirst(".thumbnail > img")?.attrAsAbsoluteUrl("src").orEmpty(), + tags = emptySet(), + state = null, + author = null, + largeCoverUrl = null, + description = null, + source = source, + ) + } + } + + override suspend fun getPages(chapter: MangaChapter): List { + val id = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + .selectFirstOrThrow("#reader") + .attr("data-id") + return webClient.httpPost("/themes/ajax/ch.php".toAbsoluteUrl(domain), "id=$id").parseHtml() + .select("img") + .map { + val url = it.attrAsRelativeUrl("src") + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } + + override suspend fun getTags(): Set { + return webClient.httpGet("/genre/".toAbsoluteUrl(domain)).parseHtml() + .select("#taxonomy .entries > .entry > a") + .mapToSet { + MangaTag( + key = it.attr("title"), + title = it.attr("title"), + source = source, + ) + } + } +} From 3112b7937cc0a2dd71ec6a7d7135622cb135ad17 Mon Sep 17 00:00:00 2001 From: vianh Date: Tue, 18 Apr 2023 23:49:46 +0700 Subject: [PATCH 2/2] [DoujinDesu] Apply suggesions from code review --- .../kotatsu/parsers/site/DoujinDesuParser.kt | 47 +++++++++++-------- 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/DoujinDesuParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/DoujinDesuParser.kt index 04746949..9e9faa7a 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/DoujinDesuParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/DoujinDesuParser.kt @@ -1,6 +1,5 @@ package org.koitharu.kotatsu.parsers.site -import org.koitharu.kotatsu.parsers.InternalParsersApi import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaSourceParser import org.koitharu.kotatsu.parsers.PagedMangaParser @@ -12,7 +11,7 @@ import java.util.* @MangaSourceParser("DOUJINDESU", "DoujinDesu", "id") class DoujinDesuParser(context: MangaLoaderContext) : PagedMangaParser(context, MangaSource.DOUJINDESU, pageSize = 18) { - @InternalParsersApi + override val configKeyDomain: ConfigKey.Domain get() = ConfigKey.Domain("212.32.226.234", null) @@ -21,6 +20,7 @@ class DoujinDesuParser(context: MangaLoaderContext) : PagedMangaParser(context, override suspend fun getDetails(manga: Manga): Manga { val docs = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml().selectFirstOrThrow("#archive") + val chapterDateFormat = SimpleDateFormat("EEEE, dd MMMM yyyy", sourceLocale) val metadataEl = docs.selectFirst(".wrapper > .metadata tbody") val state = when (metadataEl?.selectFirst("tr:contains(Status)")?.selectLast("td")?.text()) { "Finished" -> MangaState.FINISHED @@ -39,20 +39,22 @@ class DoujinDesuParser(context: MangaLoaderContext) : PagedMangaParser(context, source = source, ) }, - chapters = docs.select("#chapter_list > ul > li").mapChapters(reversed = true) { index, element -> - val titleTag = element.selectFirstOrThrow(".epsleft > .lchx > a") - val dateFormat = SimpleDateFormat("EEEE, dd MMMM yyyy", Locale("in")) - MangaChapter( - id = generateUid(titleTag.attrAsRelativeUrl("href")), - name = titleTag.text(), - number = index + 1, - url = titleTag.attrAsRelativeUrl("href"), - scanlator = null, - uploadDate = dateFormat.tryParse(element.select(".epsleft > .date").text()), - branch = null, - source = source, - ) - }, + chapters = docs.requireElementById("chapter_list") + .select("ul > li") + .mapChapters(reversed = true) { index, element -> + val titleTag = element.selectFirstOrThrow(".epsleft > .lchx > a") + val url = titleTag.attrAsRelativeUrl("href") + MangaChapter( + id = generateUid(url), + name = titleTag.text(), + number = index + 1, + url = url, + scanlator = null, + uploadDate = chapterDateFormat.tryParse(element.select(".epsleft > .date").text()), + branch = null, + source = source, + ) + }, ) } @@ -60,7 +62,7 @@ class DoujinDesuParser(context: MangaLoaderContext) : PagedMangaParser(context, page: Int, query: String?, tags: Set?, - sortOrder: SortOrder + sortOrder: SortOrder, ): List { val url = urlBuilder().apply { addPathSegment("manga") @@ -80,7 +82,10 @@ class DoujinDesuParser(context: MangaLoaderContext) : PagedMangaParser(context, } }.build() - return webClient.httpGet(url).parseHtml().select("#archives > div.entries > .entry") + return webClient.httpGet(url).parseHtml() + .requireElementById("archives") + .selectFirstOrThrow("div.entries") + .select(".entry") .map { val titleTag = it.selectFirstOrThrow(".metadata > a") val relativeUrl = titleTag.attrAsRelativeUrl("href") @@ -105,7 +110,7 @@ class DoujinDesuParser(context: MangaLoaderContext) : PagedMangaParser(context, override suspend fun getPages(chapter: MangaChapter): List { val id = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() - .selectFirstOrThrow("#reader") + .requireElementById("reader") .attr("data-id") return webClient.httpPost("/themes/ajax/ch.php".toAbsoluteUrl(domain), "id=$id").parseHtml() .select("img") @@ -122,7 +127,9 @@ class DoujinDesuParser(context: MangaLoaderContext) : PagedMangaParser(context, override suspend fun getTags(): Set { return webClient.httpGet("/genre/".toAbsoluteUrl(domain)).parseHtml() - .select("#taxonomy .entries > .entry > a") + .requireElementById("taxonomy") + .selectFirstOrThrow(".entries") + .select(".entry > a") .mapToSet { MangaTag( key = it.attr("title"),