From f10e6f021c5d9fe5eed45c1e12cf00530a86c473 Mon Sep 17 00:00:00 2001 From: Draken Date: Tue, 11 Feb 2025 13:25:33 +0000 Subject: [PATCH 1/5] Add base code for Hentai18VN --- .github/summary.yaml | 2 +- .../kotatsu/parsers/site/vi/Hentai18VN.kt | 37 +++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/Hentai18VN.kt diff --git a/.github/summary.yaml b/.github/summary.yaml index aa373c5b..50f9197d 100644 --- a/.github/summary.yaml +++ b/.github/summary.yaml @@ -1 +1 @@ -total: 1181 \ No newline at end of file +total: 1182 \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/Hentai18VN.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/Hentai18VN.kt new file mode 100644 index 00000000..a8550d25 --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/Hentai18VN.kt @@ -0,0 +1,37 @@ +package org.koitharu.kotatsu.parsers.site.vi + +import androidx.collection.arraySetOf +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.PagedMangaParser +import org.koitharu.kotatsu.parsers.network.UserAgents +import org.koitharu.kotatsu.parsers.config.ConfigKey +import org.koitharu.kotatsu.parsers.model.* +import org.koitharu.kotatsu.parsers.util.* +import java.util.* + +@MangaSourceParser("HENTAI18VN", "Hentai18VN", "vi", type = ContentType.HENTAI) +internal class Hentai18VN(context: MangaLoaderContext) : PagedMangaParser(context, MangaParserSource.HENTAI18VN, 60) { + + override val configKeyDomain = ConfigKey.Domain("hentai18vn.art") + + override fun onCreateConfig(keys: MutableCollection>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override val userAgentKey = ConfigKey.UserAgent(UserAgents.CHROME_DESKTOP) + override val availableSortOrders: Set = EnumSet.of(SortOrder.UPDATED) + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isSearchSupported = true, + isSearchWithFiltersSupported = false, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = fetchAvailableTags(), + ) + + +} \ No newline at end of file From fcd6e92dbaa5194060dd57690b87e6ce714616e5 Mon Sep 17 00:00:00 2001 From: Draken Date: Tue, 11 Feb 2025 16:08:46 +0000 Subject: [PATCH 2/5] [Hentai18VN] Add source --- .../kotatsu/parsers/site/vi/Hentai18VN.kt | 226 +++++++++++++++++- 1 file changed, 216 insertions(+), 10 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/Hentai18VN.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/Hentai18VN.kt index a8550d25..584a5a05 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/Hentai18VN.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/Hentai18VN.kt @@ -1,17 +1,21 @@ package org.koitharu.kotatsu.parsers.site.vi import androidx.collection.arraySetOf +import okhttp3.Headers +import okhttp3.HttpUrl.Companion.toHttpUrl +import org.json.JSONObject +import org.jsoup.nodes.Document import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaSourceParser import org.koitharu.kotatsu.parsers.PagedMangaParser -import org.koitharu.kotatsu.parsers.network.UserAgents import org.koitharu.kotatsu.parsers.config.ConfigKey import org.koitharu.kotatsu.parsers.model.* import org.koitharu.kotatsu.parsers.util.* +import java.text.SimpleDateFormat import java.util.* @MangaSourceParser("HENTAI18VN", "Hentai18VN", "vi", type = ContentType.HENTAI) -internal class Hentai18VN(context: MangaLoaderContext) : PagedMangaParser(context, MangaParserSource.HENTAI18VN, 60) { +internal class Hentai18VN(context: MangaLoaderContext) : PagedMangaParser(context, MangaParserSource.HENTAI18VN, 30) { override val configKeyDomain = ConfigKey.Domain("hentai18vn.art") @@ -20,18 +24,220 @@ internal class Hentai18VN(context: MangaLoaderContext) : PagedMangaParser(contex keys.add(userAgentKey) } - override val userAgentKey = ConfigKey.UserAgent(UserAgents.CHROME_DESKTOP) - override val availableSortOrders: Set = EnumSet.of(SortOrder.UPDATED) - override val filterCapabilities: MangaListFilterCapabilities get() = MangaListFilterCapabilities( isSearchSupported = true, - isSearchWithFiltersSupported = false, + isSearchWithFiltersSupported = false ) - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = fetchAvailableTags(), - ) + override suspend fun getFilterOptions() = MangaListFilterOptions(availableTags = fetchAvailableTags()) + override val availableSortOrders: Set = EnumSet.of( + SortOrder.NEWEST, + SortOrder.POPULARITY, + SortOrder.UPDATED + ) + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { + return when { + !filter.query.isNullOrEmpty() -> { + if (page > 1) { + return emptyList() + } + val url = "https://$domain/search/html/1".toHttpUrl() + val body = JSONObject().apply { + put("keyword", filter.query.urlEncoded()) + } + + val headers = Headers.Builder() + .add("X-Requested-With", "XMLHttpRequest") + .build() + + val response = webClient.httpPost(url, body, headers).parseHtml() + parseMangaSearch(response) + } + + !filter.tags.isNullOrEmpty() -> { + val tag = filter.tags.first() + val url = buildString { + append("https://") + append(domain) + append("/the-loai/") + append(tag.key) + if (page > 1) { + append("?page=") + append(page) + } + } + val response = webClient.httpGet(url).parseHtml() + parseMangaList(response) + } + + else -> { + val url = buildString { + append("https://") + append(domain) + append("/") + append( + when (order) { + SortOrder.NEWEST -> "danh-sach/truyen-hentai-moi" + SortOrder.POPULARITY -> "danh-sach/truyen-hentai-hot" + SortOrder.UPDATED -> "danh-sach/truyen-hentai-hoan-thanh" + else -> "danh-sach/truyen-hentai-hay" + } + ) + if (page > 1) { + append("?page=") + append(page) + } + } + + val response = webClient.httpGet(url).parseHtml() + parseMangaList(response) + } + } + } + + private fun parseMangaSearch(doc: Document): List { + return doc.select("ul li a.item").map { a -> + val href = a.attr("href") + val img = a.selectFirst("img")!! + Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + title = img.attr("alt"), + altTitle = null, + author = null, + tags = emptySet(), + rating = RATING_UNKNOWN, + state = null, + coverUrl = img.attr("src"), + isNsfw = isNsfwSource, + source = source + ) + } + } + + private fun parseMangaList(doc: Document): List { + return doc.select("div.visual").map { div -> + val a = div.selectFirst("div.main_text h3.title a")!! + val img = div.selectFirst("div.hentai-cover img")!! + val mangaUrl = a.attr("href") + Manga( + id = generateUid(mangaUrl), + publicUrl = mangaUrl, + url = mangaUrl.removePrefix("https://$domain"), + title = a.text(), + altTitle = null, + author = null, + description = null, + tags = emptySet(), + rating = RATING_UNKNOWN, + state = null, + coverUrl = img.attr("data-original").takeIf { it.isNotEmpty() } ?: img.attr("src"), + isNsfw = isNsfwSource, + source = source, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val tags = doc.select("div.hentai-info .line-content a.item-tag") + .mapNotNull { a -> + MangaTag( + title = a.text(), + key = a.attr("href").substringAfterLast("/"), + source = source + ) + }.toSet() + + val chapters = doc.select("ul#chapter-list li.citem").map { li -> + val a = li.selectFirst("a")!! + val number = a.text().substringAfter("Chap ").toFloatOrNull() ?: 0f + MangaChapter( + id = generateUid(a.attr("href")), + name = a.text(), + number = number, + url = a.attr("href").removePrefix("https://$domain"), + uploadDate = parseChapterDate(li.selectFirst(".time")?.text()), + source = source, + scanlator = null, + branch = null, + volume = 0 + ) + } + + val altTitle = doc.selectFirst("h2.alternative")?.text() + val author = doc.selectFirst("div.hentai-info .line:contains(Tác giả) .line-content")?.text() + val state = when(doc.selectFirst("div.hentai-info .line:contains(Tình trạng) .line-content")?.text()) { + "Đang cập nhật" -> MangaState.ONGOING + "Hoàn thành" -> MangaState.FINISHED + else -> null + } + + return manga.copy( + tags = tags, + author = author?.takeUnless { it == "Đang cập nhật" }, + altTitle = altTitle, + state = state, + chapters = chapters, + description = doc.select("div.about").text() + ) + } + + private fun parseChapterDate(date: String?): Long { + if (date == null) return 0 + return try { + val now = SimpleDateFormat("dd/MM/yyyy", Locale.US) + now.parse(date)?.time ?: 0L + } catch (e: Exception) { + 0L + } + } + + override suspend fun getPages(chapter: MangaChapter): List { + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + return doc.select("div.chapter-content div.item-photo img").mapNotNull { img -> + val url = img.requireSrc() + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source + ) + } + } + + private suspend fun fetchAvailableTags(): Set { + val tags = arraySetOf() + val firstPage = webClient.httpGet("https://$domain/tim-the-loai").parseHtml() + val lastPage = firstPage.selectFirst("a[aria-label=Last]") + ?.attr("href") + ?.substringAfter("page=") + ?.toIntOrNull() ?: 1 + + for (page in 1..lastPage) { + val doc = if (page == 1) { + firstPage + } else { + webClient.httpGet("https://$domain/tim-the-loai?page=$page").parseHtml() + } + + doc.select("ul.list-tags li").forEach { li -> + val a = li.selectFirst("a") ?: return@forEach + val title = a.selectFirst("h3.tag-name")?.text()?.trim() ?: return@forEach + val url = a.attr("href") + tags.add( + MangaTag( + title = title, + key = url.substringAfterLast("/"), + source = source + ) + ) + } + } + + return tags + } } \ No newline at end of file From 2760dc8a767088b109e92d132682557d1e36a186 Mon Sep 17 00:00:00 2001 From: Draken Date: Tue, 11 Feb 2025 17:23:30 +0000 Subject: [PATCH 3/5] [Hentai18VN] Fix attempt 1 --- .../kotatsu/parsers/site/vi/Hentai18VN.kt | 58 +++++++++---------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/Hentai18VN.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/Hentai18VN.kt index 584a5a05..a5e533b7 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/Hentai18VN.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/Hentai18VN.kt @@ -24,6 +24,10 @@ internal class Hentai18VN(context: MangaLoaderContext) : PagedMangaParser(contex keys.add(userAgentKey) } + override fun getRequestHeaders(): Headers = Headers.Builder() + .add("X-Requested-With", "XMLHttpRequest") + .build() + override val filterCapabilities: MangaListFilterCapabilities get() = MangaListFilterCapabilities( isSearchSupported = true, @@ -43,15 +47,16 @@ internal class Hentai18VN(context: MangaLoaderContext) : PagedMangaParser(contex if (page > 1) { return emptyList() } - val url = "https://$domain/search/html/1".toHttpUrl() + val url = buildString { + append("http://") + append(domain) + append("/search/html/1") + }.toHttpUrl() val body = JSONObject().apply { - put("keyword", filter.query.urlEncoded()) + put("keyword", filter.query) } - val headers = Headers.Builder() - .add("X-Requested-With", "XMLHttpRequest") - .build() - + val headers = Headers.Builder().add("X-Requested-With", "XMLHttpRequest").build() val response = webClient.httpPost(url, body, headers).parseHtml() parseMangaSearch(response) } @@ -186,16 +191,6 @@ internal class Hentai18VN(context: MangaLoaderContext) : PagedMangaParser(contex ) } - private fun parseChapterDate(date: String?): Long { - if (date == null) return 0 - return try { - val now = SimpleDateFormat("dd/MM/yyyy", Locale.US) - now.parse(date)?.time ?: 0L - } catch (e: Exception) { - 0L - } - } - override suspend fun getPages(chapter: MangaChapter): List { val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() return doc.select("div.chapter-content div.item-photo img").mapNotNull { img -> @@ -210,34 +205,39 @@ internal class Hentai18VN(context: MangaLoaderContext) : PagedMangaParser(contex } private suspend fun fetchAvailableTags(): Set { - val tags = arraySetOf() val firstPage = webClient.httpGet("https://$domain/tim-the-loai").parseHtml() val lastPage = firstPage.selectFirst("a[aria-label=Last]") ?.attr("href") ?.substringAfter("page=") ?.toIntOrNull() ?: 1 - for (page in 1..lastPage) { + return (1..lastPage).flatMap { page -> val doc = if (page == 1) { firstPage } else { webClient.httpGet("https://$domain/tim-the-loai?page=$page").parseHtml() } - doc.select("ul.list-tags li").forEach { li -> - val a = li.selectFirst("a") ?: return@forEach - val title = a.selectFirst("h3.tag-name")?.text()?.trim() ?: return@forEach + doc.select("ul.list-tags li").mapNotNull { li -> + val a = li.selectFirst("a") ?: return@mapNotNull null + val title = a.selectFirst("h3.tag-name")?.text()?.trim() ?: return@mapNotNull null val url = a.attr("href") - tags.add( - MangaTag( - title = title, - key = url.substringAfterLast("/"), - source = source - ) + MangaTag( + title = title, + key = url.substringAfterLast("/"), + source = source ) } - } + }.toSet() + } - return tags + private fun parseChapterDate(date: String?): Long { + if (date == null) return 0 + return try { + val now = SimpleDateFormat("dd/MM/yyyy", Locale.US) + now.parse(date)?.time ?: 0L + } catch (e: Exception) { + 0L + } } } \ No newline at end of file From d3858ff5ff913500e7f6c284a70fff62df1ec603 Mon Sep 17 00:00:00 2001 From: Draken Date: Tue, 11 Feb 2025 17:41:18 +0000 Subject: [PATCH 4/5] [Hentai18VN] Fix attempt 2 --- .../kotatsu/parsers/site/vi/Hentai18VN.kt | 21 ++++++------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/Hentai18VN.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/Hentai18VN.kt index a5e533b7..16123a83 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/Hentai18VN.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/Hentai18VN.kt @@ -47,17 +47,12 @@ internal class Hentai18VN(context: MangaLoaderContext) : PagedMangaParser(contex if (page > 1) { return emptyList() } - val url = buildString { - append("http://") - append(domain) - append("/search/html/1") - }.toHttpUrl() - val body = JSONObject().apply { - put("keyword", filter.query) - } - + + val keyword = filter.query + // val body = JSONObject().apply { put("keyword", filter.query) } + val url = "$domain/search/html/1".toHttpUrl() val headers = Headers.Builder().add("X-Requested-With", "XMLHttpRequest").build() - val response = webClient.httpPost(url, body, headers).parseHtml() + val response = webClient.httpPost(url, payload = "keyword=$keyword", headers).parseHtml() parseMangaSearch(response) } @@ -222,11 +217,7 @@ internal class Hentai18VN(context: MangaLoaderContext) : PagedMangaParser(contex val a = li.selectFirst("a") ?: return@mapNotNull null val title = a.selectFirst("h3.tag-name")?.text()?.trim() ?: return@mapNotNull null val url = a.attr("href") - MangaTag( - title = title, - key = url.substringAfterLast("/"), - source = source - ) + MangaTag( title = title, key = url.substringAfterLast("/"), source = source ) } }.toSet() } From 569aa73cd42fbf21ae99bcc48c506e0d018e7103 Mon Sep 17 00:00:00 2001 From: Draken Date: Tue, 11 Feb 2025 18:00:14 +0000 Subject: [PATCH 5/5] [Hentai18VN] Fix attempt 3 --- .../koitharu/kotatsu/parsers/site/vi/Hentai18VN.kt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/Hentai18VN.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/Hentai18VN.kt index 16123a83..faac7c26 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/Hentai18VN.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/Hentai18VN.kt @@ -50,9 +50,9 @@ internal class Hentai18VN(context: MangaLoaderContext) : PagedMangaParser(contex val keyword = filter.query // val body = JSONObject().apply { put("keyword", filter.query) } - val url = "$domain/search/html/1".toHttpUrl() + val url = "http://$domain/search/html/1" val headers = Headers.Builder().add("X-Requested-With", "XMLHttpRequest").build() - val response = webClient.httpPost(url, payload = "keyword=$keyword", headers).parseHtml() + val response = webClient.httpPost(url.toHttpUrl(), payload = "keyword=$keyword", headers).parseHtml() parseMangaSearch(response) } @@ -98,20 +98,20 @@ internal class Hentai18VN(context: MangaLoaderContext) : PagedMangaParser(contex } private fun parseMangaSearch(doc: Document): List { - return doc.select("ul li a.item").map { a -> + return doc.select("a.item").map { a -> val href = a.attr("href") - val img = a.selectFirst("img")!! + val mangaInfo = a.selectFirst("img") Manga( id = generateUid(href), url = href, publicUrl = href.toAbsoluteUrl(domain), - title = img.attr("alt"), + title = mangaInfo.attr("alt"), altTitle = null, author = null, tags = emptySet(), rating = RATING_UNKNOWN, state = null, - coverUrl = img.attr("src"), + coverUrl = mangaInfo.requireSrc(), isNsfw = isNsfwSource, source = source )