From 2da8a4af849d2b2bac2ef923c425c843201aaecf Mon Sep 17 00:00:00 2001 From: Draken Date: Mon, 17 Feb 2025 11:19:23 +0000 Subject: [PATCH 1/3] [viHentai] Add source --- .github/summary.yaml | 2 +- .../kotatsu/parsers/site/vi/LxManga.kt | 86 +------ .../kotatsu/parsers/site/vi/TruyenHentaiVN.kt | 24 +- .../kotatsu/parsers/site/vi/viHentai.kt | 238 ++++++++++++++++++ 4 files changed, 268 insertions(+), 82 deletions(-) create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/viHentai.kt diff --git a/.github/summary.yaml b/.github/summary.yaml index b504a883..983b6230 100644 --- a/.github/summary.yaml +++ b/.github/summary.yaml @@ -1 +1 @@ -total: 1186 \ No newline at end of file +total: 1187 \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/LxManga.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/LxManga.kt index 33707ef5..a1d76a4c 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/LxManga.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/LxManga.kt @@ -208,79 +208,19 @@ internal class LxManga(context: MangaLoaderContext) : PagedMangaParser(context, } } - private fun availableTags() = arraySetOf( - MangaTag("3D", "3d", source), - MangaTag("Adult", "adult", source), - MangaTag("Animal girl", "animal-girl", source), - MangaTag("Artist", "artist", source), - MangaTag("Bạo Dâm", "bao-dam", source), - MangaTag("CG", "cg", source), - MangaTag("Chơi Hai Lỗ", "choi-hai-lo", source), - MangaTag("Côn Trùng", "con-trung", source), - MangaTag("Cosplay", "cosplay", source), - MangaTag("Creampie", "creampie", source), - MangaTag("Doujinshi", "doujinshi", source), - MangaTag("Drama", "drama", source), - MangaTag("Đam Mỹ", "dam-my", source), - MangaTag("Đồ Bơi", "do-boi", source), - MangaTag("Elf", "elf", source), - MangaTag("Fantasy", "fantasy", source), - MangaTag("Furry", "furry", source), - MangaTag("Gangbang", "gangbang", source), - MangaTag("Gender Bender", "gender-bender", source), - MangaTag("Giáo Viên", "giao-vien", source), - MangaTag("Girl love", "girl-love", source), - MangaTag("Group", "group", source), - MangaTag("Guro", "guro", source), - MangaTag("Hài Hước", "hai-huoc", source), - MangaTag("Hãm Hiếp", "ham-hiep", source), - MangaTag("Harem", "harem", source), - MangaTag("Hầu Gái", "hau-gai", source), - MangaTag("Hậu Môn", "hau-mon", source), - MangaTag("Housewife", "housewife", source), - MangaTag("Kinh Dị", "kinh-di", source), - MangaTag("Kogal", "kogal", source), - MangaTag("Lão Già Dâm", "lao-gia-dam", source), - MangaTag("Lãng Mạn", "lang-man", source), - MangaTag("Loạn Luân", "loan-luan", source), - MangaTag("Loli", "loli", source), - MangaTag("LXHENTAI", "lxhentai", source), - MangaTag("Mang Thai", "mang-thai", source), - MangaTag("Manhwa", "manhwa", source), - MangaTag("Mắt Kính", "mat-kinh", source), - MangaTag("Mature", "mature", source), - MangaTag("Milf", "milf", source), - MangaTag("Mind Break", "mind-break", source), - MangaTag("Mind Control", "mind-control", source), - MangaTag("Monster Girl", "monster-girl", source), - MangaTag("Nàng Dâu", "nang-dau", source), - MangaTag("Netorare", "netorare", source), - MangaTag("Ngực Lớn", "nguc-lon", source), - MangaTag("Ngực Nhỏ", "nguc-nho", source), - MangaTag("Nô Lệ", "no-le", source), - MangaTag("NTR", "ntr", source), - MangaTag("Nữ Sinh", "nu-sinh", source), - MangaTag("Office lady", "office-lady", source), - MangaTag("OneShot", "oneshot", source), - MangaTag("Quái Vật", "quai-vat", source), - MangaTag("Series", "series", source), - MangaTag("Shota", "shota", source), - MangaTag("Supernatural", "supernatural", source), - MangaTag("Swinging", "swinging", source), - MangaTag("Thể Thao", "the-thao", source), - MangaTag("Three some", "three-some", source), - MangaTag("Thú Vật", "thu-vat", source), - MangaTag("Trap", "trap", source), - MangaTag("Truyện Comic", "truyen-comic", source), - MangaTag("Truyện Không Che", "truyen-khong-che", source), - MangaTag("Truyện Màu", "truyen-mau", source), - MangaTag("Truyện Ngắn", "truyen-ngan", source), - MangaTag("Virgin", "virgin", source), - MangaTag("Xúc Tua", "xuc-tua", source), - MangaTag("Y Tá", "y-ta", source), - MangaTag("Yaoi", "yaoi", source), - MangaTag("Yuri", "yuri", source), - ) + private suspend fun availableTags(): Set { + val url = "$domain/the-loai" + val doc = webClient.httpGet(url).parseHtml() + + return doc.select("nav.grid.grid-cols-3.md\\:grid-cols-8 button").map { button -> + val key = button.attr("wire:click").substringAfterLast("'").substringBeforeLast("'") + MangaTag( + key = key, + title = button.select("span.text-ellipsis").text(), + source = source + ) + }.toSet() + } private fun parseDateTime(dateStr: String): Long = runCatching { val parts = dateStr.split(' ') diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/TruyenHentaiVN.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/TruyenHentaiVN.kt index a39056a9..2c3bd5c9 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/TruyenHentaiVN.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/TruyenHentaiVN.kt @@ -1,18 +1,21 @@ package org.koitharu.kotatsu.parsers.site.vi import androidx.collection.arraySetOf +import androidx.collection.ArrayMap import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaSourceParser import org.koitharu.kotatsu.parsers.PagedMangaParser import org.koitharu.kotatsu.parsers.config.ConfigKey import org.koitharu.kotatsu.parsers.model.* import org.koitharu.kotatsu.parsers.util.* +import org.koitharu.kotatsu.parsers.util.suspendlazy.suspendLazy import java.util.* import java.text.SimpleDateFormat @MangaSourceParser("TRUYENHENTAIVN", "TruyenHentaiVN", "vi", type = ContentType.HENTAI) internal class TruyenHentaiVN(context: MangaLoaderContext) : PagedMangaParser(context, MangaParserSource.TRUYENHENTAIVN, 30) { + private var cacheTags = suspendLazy(initializer = ::fetchTags) override val configKeyDomain = ConfigKey.Domain("truyenhentaivn.live") override fun onCreateConfig(keys: MutableCollection>) { @@ -26,7 +29,7 @@ internal class TruyenHentaiVN(context: MangaLoaderContext) : PagedMangaParser(co isSearchWithFiltersSupported = false ) - override suspend fun getFilterOptions() = MangaListFilterOptions(availableTags = getAvailableTags()) + override suspend fun getFilterOptions() = MangaListFilterOptions(availableTags = cacheTags.get().values.toSet()) override val availableSortOrders: Set = EnumSet.of(SortOrder.UPDATED) @@ -38,7 +41,7 @@ internal class TruyenHentaiVN(context: MangaLoaderContext) : PagedMangaParser(co when { !filter.tags.isNullOrEmpty() -> { val tag = filter.tags.first() - append(tag.key) // Testing... + append(tag.key) if (page > 1) { append("?page=") append(page) @@ -156,12 +159,17 @@ internal class TruyenHentaiVN(context: MangaLoaderContext) : PagedMangaParser(co } } - private suspend fun getAvailableTags(): Set { + private suspend fun fetchTags(): Map { + // Remove cached tags, avoid "Connection errors" + // Remake this function from site/vi/BlogTruyenVN.kt val doc = webClient.httpGet("https://$domain").parseHtml() - return doc.select("a.py-2[href^=/the-loai-]").mapNotNull { a -> - val key = a.attr("href") - val title = a.text() - MangaTag( key = key, title = title, source = source ) - }.toSet() + val tagItems = doc.select("a.py-2[href^=/the-loai-]") + val tagMap = ArrayMap(tagItems.size) + for (tag in tagItems) { + val key = tag.attr("href") + val title = tag.text() + tagMap[key] = MangaTag(key = key, title = title, source = source) + } + return tagMap } } \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/viHentai.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/viHentai.kt new file mode 100644 index 00000000..aff01815 --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/viHentai.kt @@ -0,0 +1,238 @@ +package org.koitharu.kotatsu.parsers.site.vi + +import androidx.collection.arraySetOf +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.PagedMangaParser +import org.koitharu.kotatsu.parsers.config.ConfigKey +import org.koitharu.kotatsu.parsers.model.* +import org.koitharu.kotatsu.parsers.util.* +import java.util.* + +@MangaSourceParser("VIHENTAI", "viHentai", "vi", type = ContentType.HENTAI) +internal class viHentai(context: MangaLoaderContext) : PagedMangaParser(context, MangaParserSource.VIHENTAI, 60) { + + override val configKeyDomain = ConfigKey.Domain("vi-hentai.com") + + override fun onCreateConfig(keys: MutableCollection>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override val availableSortOrders: Set = EnumSet.of( + SortOrder.ALPHABETICAL, + SortOrder.ALPHABETICAL_DESC, + SortOrder.UPDATED, + SortOrder.NEWEST, + SortOrder.POPULARITY, + ) + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isSearchSupported = true, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = availableTags(), + availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED), + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { + val url = buildString { + append("https://") + append(domain) + + when { + + !filter.query.isNullOrEmpty() -> { + append("/tim-kiem") + append("?filter[name]=") + append(filter.query.urlEncoded()) + + if (page > 1) { + append("&page=") + append(page) + } + + append("&sort=") + append( + when (order) { + SortOrder.POPULARITY -> "-views" + SortOrder.UPDATED -> "-updated_at" + SortOrder.NEWEST -> "-created_at" + SortOrder.ALPHABETICAL -> "name" + SortOrder.ALPHABETICAL_DESC -> "-name" + else -> "-updated_at" + }, + ) + } + + filter.tags.isNotEmpty() -> { + val tag = filter.tags.first() + append("/the-loai/") + append(tag.key) + + append("?page=") + append(page) + } + + else -> { + append("/danh-sach") + append("?sort=") + append( + when (order) { + SortOrder.POPULARITY -> "-views" + SortOrder.UPDATED -> "-updated_at" + SortOrder.NEWEST -> "-created_at" + SortOrder.ALPHABETICAL -> "name" + SortOrder.ALPHABETICAL_DESC -> "-name" + else -> "-updated_at" + }, + ) + append("&page=") + append(page) + } + } + + if (filter.query.isNullOrEmpty()) { + append("&sort=") + when (order) { + SortOrder.POPULARITY -> append("-views") + SortOrder.UPDATED -> append("-updated_at") + SortOrder.NEWEST -> append("-created_at") + SortOrder.ALPHABETICAL -> append("name") + SortOrder.ALPHABETICAL_DESC -> append("-name") + else -> append("-updated_at") + } + } + + if (filter.states.isNotEmpty()) { + append("&filter[status]=") + filter.states.forEach { + append( + when (it) { + MangaState.ONGOING -> "2," + MangaState.FINISHED -> "1," + else -> "1,2" + }, + ) + } + } + } + + val doc = webClient.httpGet(url).parseHtml() + + return doc.select("div.grid div.relative").map { div -> + val href = div.selectFirst("a[href^=/truyen/]")?.attrOrNull("href") + ?: div.parseFailed("Không thể tìm thấy nguồn ảnh của Manga này!") + val coverUrl = div.selectFirstOrThrow("div.cover").let { + it.attrOrNull("data-bg") ?: it.attr("style").cssUrl()?.replace("s3.lxmanga.top", domain) + }.orEmpty() + + Manga( + id = generateUid(href), + title = div.select("div.p-2 a.text-ellipsis").text(), + altTitle = null, + url = href, + publicUrl = href.toAbsoluteUrl(domain), + rating = RATING_UNKNOWN, + isNsfw = true, + coverUrl = coverUrl, + tags = setOf(), + state = null, + author = null, + source = source, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val root = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + + return manga.copy( + altTitle = root.selectLast("div.grow div:contains(Tên khác) span")?.textOrNull(), + state = when (root.selectFirst("div.mt-2:contains(Tình trạng) span.text-blue-500")?.text()) { + "Đang tiến hành" -> MangaState.ONGOING + "Đã hoàn thành" -> MangaState.FINISHED + else -> null + }, + tags = root.select("div.mt-2:contains(Thể loại) a.bg-gray-500").mapToSet { a -> + MangaTag( + key = a.attr("href").removeSuffix('/').substringAfterLast('/'), + title = a.text(), + source = source, + ) + }, + author = root.selectFirst("div.mt-2:contains(Tác giả) span a")?.textOrNull(), + description = root.selectFirst("meta[name=description]")?.attrOrNull("content"), + chapters = root.select("div.justify-between ul.overflow-y-auto.overflow-x-hidden a") + .mapChapters(reversed = true) { i, a -> + val href = a.attrAsRelativeUrl("href") + val name = a.selectFirst("span.text-ellipsis")?.text().orEmpty() + val dateText = a.parent()?.selectFirst("span.timeago")?.attr("datetime").orEmpty() + val scanlator = root.selectFirst("div.mt-2:contains(Nhóm dịch) span a")?.textOrNull() + + MangaChapter( + id = generateUid(href), + name = name, + number = i.toFloat(), + volume = 0, + url = href, + scanlator = scanlator, + uploadDate = parseDateTime(dateText), + branch = null, + source = source, + ) + }, + ) + } + + override suspend fun getPages(chapter: MangaChapter): List { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + return doc.select("div.text-center div.lazy") + .mapNotNull { div -> + val url = div.attr("data-src") + if (url.endsWith(".jpg", ignoreCase = true) || + url.endsWith(".png", ignoreCase = true) + ) { + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } else { + null + } + } + } + + private fun parseDateTime(dateStr: String): Long = runCatching { + val parts = dateStr.split(' ') + val dateParts = parts[0].split('-') + val timeParts = parts[1].split(':') + + val calendar = Calendar.getInstance() + calendar.set( + dateParts[0].toInt(), + dateParts[1].toInt() - 1, + dateParts[2].toInt(), + timeParts[0].toInt(), + timeParts[1].toInt(), + timeParts[2].toInt(), + ) + calendar.timeInMillis + }.getOrDefault(0L) + + private suspend fun availableTags(): Set { + val doc = webClient.httpGet("https://$domain").parseHtml() + return doc.select("ul.grid.grid-cols-2 a").mapToSet { a -> + MangaTag( + key = a.attr("href").removeSuffix('/').substringAfterLast('/'), + title = a.text(), + source = source, + ) + } + } +} From 8c4a93d1dc829a6235dc30a050abe93464abf03b Mon Sep 17 00:00:00 2001 From: Draken Date: Mon, 17 Feb 2025 11:57:03 +0000 Subject: [PATCH 2/3] [viHentai] Fix attempt 1 --- .../kotatsu/parsers/site/vi/LxManga.kt | 2 +- .../kotatsu/parsers/site/vi/viHentai.kt | 32 +++++++------------ 2 files changed, 13 insertions(+), 21 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/LxManga.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/LxManga.kt index a1d76a4c..88c6e9fe 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/LxManga.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/LxManga.kt @@ -209,7 +209,7 @@ internal class LxManga(context: MangaLoaderContext) : PagedMangaParser(context, } private suspend fun availableTags(): Set { - val url = "$domain/the-loai" + val url = "https://$domain/the-loai" val doc = webClient.httpGet(url).parseHtml() return doc.select("nav.grid.grid-cols-3.md\\:grid-cols-8 button").map { button -> diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/viHentai.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/viHentai.kt index aff01815..1f1b9828 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/viHentai.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/viHentai.kt @@ -125,9 +125,8 @@ internal class viHentai(context: MangaLoaderContext) : PagedMangaParser(context, return doc.select("div.grid div.relative").map { div -> val href = div.selectFirst("a[href^=/truyen/]")?.attrOrNull("href") ?: div.parseFailed("Không thể tìm thấy nguồn ảnh của Manga này!") - val coverUrl = div.selectFirstOrThrow("div.cover").let { - it.attrOrNull("data-bg") ?: it.attr("style").cssUrl()?.replace("s3.lxmanga.top", domain) - }.orEmpty() + val coverUrl = div.selectFirst("div.cover")?.attr("style") + ?.substringAfter("url('")?.substringBefore("')") Manga( id = generateUid(href), @@ -137,7 +136,7 @@ internal class viHentai(context: MangaLoaderContext) : PagedMangaParser(context, publicUrl = href.toAbsoluteUrl(domain), rating = RATING_UNKNOWN, isNsfw = true, - coverUrl = coverUrl, + coverUrl = coverUrl.orEmpty(), tags = setOf(), state = null, author = null, @@ -190,22 +189,15 @@ internal class viHentai(context: MangaLoaderContext) : PagedMangaParser(context, override suspend fun getPages(chapter: MangaChapter): List { val fullUrl = chapter.url.toAbsoluteUrl(domain) val doc = webClient.httpGet(fullUrl).parseHtml() - return doc.select("div.text-center div.lazy") - .mapNotNull { div -> - val url = div.attr("data-src") - if (url.endsWith(".jpg", ignoreCase = true) || - url.endsWith(".png", ignoreCase = true) - ) { - MangaPage( - id = generateUid(url), - url = url, - preview = null, - source = source, - ) - } else { - null - } - } + return doc.select("div.text-center img.lazy").mapNotNull { img -> + val url = img.attr("data-src") + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } } private fun parseDateTime(dateStr: String): Long = runCatching { From 09ff9c0a7d3f4efff9ece31e2b60bff545fa27d2 Mon Sep 17 00:00:00 2001 From: Draken Date: Mon, 17 Feb 2025 12:16:47 +0000 Subject: [PATCH 3/3] [viHentai] Fix attempt 2 --- src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/LxManga.kt | 2 +- .../kotlin/org/koitharu/kotatsu/parsers/site/vi/viHentai.kt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/LxManga.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/LxManga.kt index 88c6e9fe..4287cd03 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/LxManga.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/LxManga.kt @@ -213,7 +213,7 @@ internal class LxManga(context: MangaLoaderContext) : PagedMangaParser(context, val doc = webClient.httpGet(url).parseHtml() return doc.select("nav.grid.grid-cols-3.md\\:grid-cols-8 button").map { button -> - val key = button.attr("wire:click").substringAfterLast("'").substringBeforeLast("'") + val key = button.attr("wire:click").substringAfterLast(", '").substringBeforeLast("')") MangaTag( key = key, title = button.select("span.text-ellipsis").text(), diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/viHentai.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/viHentai.kt index 1f1b9828..355bdaee 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/viHentai.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/viHentai.kt @@ -190,7 +190,7 @@ internal class viHentai(context: MangaLoaderContext) : PagedMangaParser(context, val fullUrl = chapter.url.toAbsoluteUrl(domain) val doc = webClient.httpGet(fullUrl).parseHtml() return doc.select("div.text-center img.lazy").mapNotNull { img -> - val url = img.attr("data-src") + val url = img.requireSrc() MangaPage( id = generateUid(url), url = url,