KotatsuApp
/
kotatsu-parsers
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

GocTruyenTranhVui + KuroNeko: Fixes (#2165)

Browse Source 
Co-authored-by: Draken <131387159+dragonx943@users.noreply.github.com>
master
Hoàng Phi Hùng 8 months ago committed by GitHub
parent b8d3124227
commit 8573921243
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 274 additions and 282 deletions
Show Stats Download Patch File Download Diff File

38
src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/GocTruyenTranhVui.kt
Unescape Escape View File

@ -14,30 +14,29 @@ import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.* import org.koitharu.kotatsu.parsers.util.*
import org.json.JSONObject import org.json.JSONObject
import java.util.* import java.util.*
import org.koitharu.kotatsu.parsers.Broken
@Broken("Need to clean code + Testing")
@MangaSourceParser("GOCTRUYENTRANHVUI", "Góc Truyện Tranh Vui", "vi") @MangaSourceParser("GOCTRUYENTRANHVUI", "Góc Truyện Tranh Vui", "vi")
internal class GocTruyenTranhVui(context: MangaLoaderContext): internal class GocTruyenTranhVui(context: MangaLoaderContext) : PagedMangaParser(context, MangaParserSource.GOCTRUYENTRANHVUI, 50) {
PagedMangaParser(context, MangaParserSource.GOCTRUYENTRANHVUI, 50) {
override val configKeyDomain = ConfigKey.Domain("goctruyentranhvui17.com") override val configKeyDomain = ConfigKey.Domain("goctruyentranhvui17.com")
private val apiUrl by lazy { "https://$domain/api/v2" }
private val apiUrl = "https://$domain/api/v2"
private val requestMutex = Mutex() private val requestMutex = Mutex()
private var lastRequestTime = 0L private var lastRequestTime = 0L
override fun getRequestHeaders(): Headers = Headers.Builder() private val apiHeaders by lazy {
Headers.Builder()
.add("Authorization", TOKEN_KEY) .add("Authorization", TOKEN_KEY)
.add("Referer", "https://$domain/") .add("Referer", "https://$domain/")
.add("X-Requested-With", "XMLHttpRequest") .add("X-Requested-With", "XMLHttpRequest")
.build() .build()
}
override val availableSortOrders: Set<SortOrder> = EnumSet.of( override val availableSortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.UPDATED, SortOrder.UPDATED,
SortOrder.POPULARITY, SortOrder.POPULARITY,
SortOrder.NEWEST, SortOrder.NEWEST,
SortOrder.RATING, SortOrder.RATING
) )
override val filterCapabilities = MangaListFilterCapabilities( override val filterCapabilities = MangaListFilterCapabilities(
@ -47,18 +46,14 @@ internal class GocTruyenTranhVui(context: MangaLoaderContext):
override suspend fun getFilterOptions() = MangaListFilterOptions( override suspend fun getFilterOptions() = MangaListFilterOptions(
availableTags = availableTags(), availableTags = availableTags(),
availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED), availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED)
) )
init {
setFirstPage(0)
}
override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> {
enforceRateLimit() enforceRateLimit()
val url = buildString { val url = buildString {
append(apiUrl) append(apiUrl)
append("/search?p=$page") append("/search?p=${page - 1}")
if (!filter.query.isNullOrBlank()) { if (!filter.query.isNullOrBlank()) {
append("&searchValue=${filter.query.urlEncoded()}") append("&searchValue=${filter.query.urlEncoded()}")
} }
@ -83,7 +78,7 @@ internal class GocTruyenTranhVui(context: MangaLoaderContext):
} }
} }
val json = webClient.httpGet(url).parseJson() val json = webClient.httpGet(url, extraHeaders = apiHeaders).parseJson()
val result = json.optJSONObject("result") ?: return emptyList() val result = json.optJSONObject("result") ?: return emptyList()
val data = result.optJSONArray("data") ?: return emptyList() val data = result.optJSONArray("data") ?: return emptyList()
@ -129,7 +124,7 @@ internal class GocTruyenTranhVui(context: MangaLoaderContext):
val chapters = try { val chapters = try {
enforceRateLimit() enforceRateLimit()
val chapterApiUrl = "https://$domain/api/comic/$comicId/chapter?limit=-1" val chapterApiUrl = "https://$domain/api/comic/$comicId/chapter?limit=-1"
val chapterJson = webClient.httpGet(chapterApiUrl).parseJson() val chapterJson = webClient.httpGet(chapterApiUrl, extraHeaders = apiHeaders).parseJson()
val chaptersData = chapterJson.getJSONObject("result").getJSONArray("chapters") val chaptersData = chapterJson.getJSONObject("result").getJSONArray("chapters")
List(chaptersData.length()) { i -> List(chaptersData.length()) { i ->
@ -205,19 +200,14 @@ internal class GocTruyenTranhVui(context: MangaLoaderContext):
"nameEn" to nameEn "nameEn" to nameEn
) )
val authApiUrl = "$apiUrl/chapter/auth".toHttpUrl() val authApiUrl = "$apiUrl/chapter/auth".toHttpUrl()
val authResponse = webClient.httpPost(url = authApiUrl, form = formBody).parseJson() val authResponse = webClient.httpPost(url = authApiUrl, form = formBody, extraHeaders = apiHeaders).parseJson()
val data = authResponse.getJSONObject("result").getJSONArray("data") val data = authResponse.getJSONObject("result").getJSONArray("data")
imageUrls = List(data.length()) { i -> data.getString(i) } imageUrls = List(data.length()) { i -> data.getString(i) }
} }
return imageUrls.map { url -> return imageUrls.map { url ->
val finalUrl = if (url.startsWith("/image/")) "https://$domain$url" else url val finalUrl = if (url.startsWith("/image/")) "https://$domain$url" else url
MangaPage( MangaPage(id = generateUid(finalUrl), url = finalUrl, preview = null, source = source)
id = generateUid(finalUrl),
url = finalUrl,
preview = null,
source = source
)
} }
} }
@ -225,7 +215,7 @@ internal class GocTruyenTranhVui(context: MangaLoaderContext):
requestMutex.withLock { requestMutex.withLock {
val currentTime = System.currentTimeMillis() val currentTime = System.currentTimeMillis()
val timeSinceLastRequest = currentTime - lastRequestTime val timeSinceLastRequest = currentTime - lastRequestTime
if (timeSinceLastRequest < REQUEST_DELAY_MS) { if (timeSinceLastRequest < REQUEST_DELAY_MS) { // Vẫn truy cập được REQUEST_DELAY_MS
delay(REQUEST_DELAY_MS - timeSinceLastRequest) delay(REQUEST_DELAY_MS - timeSinceLastRequest)
} }
lastRequestTime = System.currentTimeMillis() lastRequestTime = System.currentTimeMillis()
@ -277,7 +267,7 @@ internal class GocTruyenTranhVui(context: MangaLoaderContext):
MangaTag("Gender Bender", "GDB", source), MangaTag("Gender Bender", "GDB", source),
MangaTag("Murim", "MRR", source), MangaTag("Murim", "MRR", source),
MangaTag("Leo Tháp", "LTT", source), MangaTag("Leo Tháp", "LTT", source),
MangaTag("Nấu Ăn", "COO", source), MangaTag("Nấu Ăn", "COO", source)
) )
companion object { companion object {

44
src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/KuroNeko.kt
Unescape Escape View File

@ -1,6 +1,8 @@
package org.koitharu.kotatsu.parsers.site.vi package org.koitharu.kotatsu.parsers.site.vi
import org.jsoup.nodes.Document import kotlinx.coroutines.delay
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.config.ConfigKey import org.koitharu.kotatsu.parsers.config.ConfigKey
@ -10,11 +12,13 @@ import org.koitharu.kotatsu.parsers.util.*
import java.util.* import java.util.*
@MangaSourceParser("KURONEKO", "Kuro Neko / vi-Hentai", "vi", type = ContentType.HENTAI) @MangaSourceParser("KURONEKO", "Kuro Neko / vi-Hentai", "vi", type = ContentType.HENTAI)
internal class KuroNeko(context: MangaLoaderContext): internal class KuroNeko(context: MangaLoaderContext) : PagedMangaParser(context, MangaParserSource.KURONEKO, 30) {
PagedMangaParser(context, MangaParserSource.KURONEKO, 30) {
override val configKeyDomain = ConfigKey.Domain("vi-hentai.moe") override val configKeyDomain = ConfigKey.Domain("vi-hentai.moe")
private val pagesRequestMutex = Mutex()
private var lastPagesRequestTime = 0L
override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) {
super.onCreateConfig(keys) super.onCreateConfig(keys)
keys.add(userAgentKey) keys.add(userAgentKey)
@ -129,9 +133,9 @@ internal class KuroNeko(context: MangaLoaderContext):
} }
} }
val doc = parseHttp(url) { it } val doc = webClient.httpGet(url).parseHtml()
return doc.select("div.grid div.relative")
return doc.select("div.grid div.relative").map { div -> .map { div ->
val href = div.selectFirst("a[href^=/truyen/]")?.attrOrNull("href") val href = div.selectFirst("a[href^=/truyen/]")?.attrOrNull("href")
?: div.parseFailed("Không thể tìm thấy nguồn ảnh của Manga này!") ?: div.parseFailed("Không thể tìm thấy nguồn ảnh của Manga này!")
val coverUrl = div.selectFirst("div.cover")?.attr("style") val coverUrl = div.selectFirst("div.cover")?.attr("style")
@ -155,7 +159,7 @@ internal class KuroNeko(context: MangaLoaderContext):
} }
override suspend fun getDetails(manga: Manga): Manga { override suspend fun getDetails(manga: Manga): Manga {
val root = parseHttp(manga.url.toAbsoluteUrl(domain)) { it } val root = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml()
val author = root.selectFirst("div.mt-2:contains(Tác giả) span a")?.textOrNull() val author = root.selectFirst("div.mt-2:contains(Tác giả) span a")?.textOrNull()
return manga.copy( return manga.copy(
@ -180,7 +184,6 @@ internal class KuroNeko(context: MangaLoaderContext):
val name = a.selectFirst("span.text-ellipsis")?.text().orEmpty() val name = a.selectFirst("span.text-ellipsis")?.text().orEmpty()
val dateText = a.parent()?.selectFirst("span.timeago")?.attr("datetime").orEmpty() val dateText = a.parent()?.selectFirst("span.timeago")?.attr("datetime").orEmpty()
val scanlator = root.selectFirst("div.mt-2:contains(Nhóm dịch) span a")?.textOrNull() val scanlator = root.selectFirst("div.mt-2:contains(Nhóm dịch) span a")?.textOrNull()
MangaChapter( MangaChapter(
id = generateUid(href), id = generateUid(href),
title = name, title = name,
@ -197,12 +200,18 @@ internal class KuroNeko(context: MangaLoaderContext):
} }
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val doc = parseHttp(chapter.url.toAbsoluteUrl(domain)) { it } pagesRequestMutex.withLock {
return doc.select("div.text-center img.lazy").mapNotNull { img -> val currentTime = System.currentTimeMillis()
val url = img.attr("src").takeIf { it.isNotBlank() } val timeSinceLastRequest = currentTime - lastPagesRequestTime
?: img.attr("data-src").takeIf { it.isNotBlank() } if (timeSinceLastRequest < PAGES_REQUEST_DELAY_MS) {
?: return@mapNotNull null delay(PAGES_REQUEST_DELAY_MS - timeSinceLastRequest)
}
lastPagesRequestTime = System.currentTimeMillis()
}
val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml()
return doc.select("div.text-center img").mapNotNull { img ->
val url = img.requireSrc()
MangaPage( MangaPage(
id = generateUid(url), id = generateUid(url),
url = url, url = url,
@ -243,15 +252,8 @@ internal class KuroNeko(context: MangaLoaderContext):
calendar.timeInMillis calendar.timeInMillis
}.getOrDefault(0L) }.getOrDefault(0L)
private suspend fun <T> parseHttp(url: String, block: (Document) -> T): T {
// 15 reqs / minute
kotlinx.coroutines.delay(4000L)
val doc = webClient.httpGet(url).parseHtml()
return block(doc)
}
companion object { companion object {
private const val PAGES_REQUEST_DELAY_MS = 5000L
const val PATH = "AxsAEQdJWk4YDUkHDgcVEwxaBQoHShIXHwYbD1seHAwHOwAKCAYFFw==\n" const val PATH = "AxsAEQdJWk4YDUkHDgcVEwxaBQoHShIXHwYbD1seHAwHOwAKCAYFFw==\n"
} }
} }

Write Preview
Loading…
Cancel
Save