From a235f6281c74048dcee565c264c9d953d8043f9c Mon Sep 17 00:00:00 2001 From: vianh Date: Mon, 17 Jul 2023 14:52:09 +0700 Subject: [PATCH] [BlogTruyen] Update domain, user agent --- .../kotatsu/parsers/site/BlogTruyenParser.kt | 55 +++++++++---------- 1 file changed, 26 insertions(+), 29 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BlogTruyenParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BlogTruyenParser.kt index 18aac9b1..c820f77c 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BlogTruyenParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BlogTruyenParser.kt @@ -1,8 +1,7 @@ package org.koitharu.kotatsu.parsers.site import androidx.collection.ArrayMap -import kotlinx.coroutines.sync.Mutex -import kotlinx.coroutines.sync.withLock +import okhttp3.Headers import org.json.JSONArray import org.jsoup.nodes.Document import org.jsoup.nodes.Element @@ -18,6 +17,8 @@ import org.koitharu.kotatsu.parsers.model.MangaState import org.koitharu.kotatsu.parsers.model.MangaTag import org.koitharu.kotatsu.parsers.model.RATING_UNKNOWN import org.koitharu.kotatsu.parsers.model.SortOrder +import org.koitharu.kotatsu.parsers.network.UserAgents +import org.koitharu.kotatsu.parsers.util.SuspendLazy import org.koitharu.kotatsu.parsers.util.attrAsAbsoluteUrlOrNull import org.koitharu.kotatsu.parsers.util.attrAsRelativeUrl import org.koitharu.kotatsu.parsers.util.domain @@ -40,14 +41,17 @@ class BlogTruyenParser(context: MangaLoaderContext) : PagedMangaParser(context, MangaSource.BLOGTRUYEN, pageSize = 20) { override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("blogtruyen.vn") + get() = ConfigKey.Domain("blogtruyenmoi.com") override val sortOrders: Set get() = EnumSet.of(SortOrder.UPDATED) - private val mutex = Mutex() + override val headers: Headers = Headers.Builder() + .add("User-Agent", UserAgents.CHROME_DESKTOP) + .build() + private val dateFormat = SimpleDateFormat("dd/MM/yyyy HH:mm", Locale.US) - private var cacheTags: ArrayMap? = null + private var cacheTags = SuspendLazy(::fetchTags) override suspend fun getDetails(manga: Manga): Manga { val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() @@ -75,14 +79,15 @@ class BlogTruyenParser(context: MangaLoaderContext) : } } - val tagMap = getOrCreateTagMap() - val tags = descriptionElement.select("p > span.category").mapNotNullToSet { - val tagName = it.selectFirst("a")?.text()?.trim() ?: return@mapNotNullToSet null - tagMap[tagName] + val tags = cacheTags.tryGet().getOrNull()?.let { tagMap -> + descriptionElement.select("p > span.category").mapNotNullToSet { + val tagName = it.selectFirst("a")?.text()?.trim() ?: return@mapNotNullToSet null + tagMap[tagName] + } } return manga.copy( - tags = tags, + tags = tags ?: emptySet(), author = descriptionElement.selectFirst("p:contains(Tác giả) > a")?.text(), description = doc.selectFirst(".detail .content")?.html(), chapters = parseChapterList(doc), @@ -146,24 +151,25 @@ class BlogTruyenParser(context: MangaLoaderContext) : val listElements = doc.selectFirstOrThrow("section.list-mainpage.listview") .select("div.bg-white.storyitem") - return listElements.mapNotNull { - val linkTag = it.selectFirst("div.fl-l > a") ?: return@mapNotNull null + return listElements.mapNotNull { el -> + val linkTag = el.selectFirst("div.fl-l > a") ?: return@mapNotNull null val relativeUrl = linkTag.attrAsRelativeUrl("href") - val tagMap = getOrCreateTagMap() - val tags = it.select("footer > div.category > a").mapNotNullToSet { a -> - tagMap[a.text()] + val tags = cacheTags.tryGet().getOrNull()?.let { tagMap -> + el.select("footer > div.category > a").mapNotNullToSet { a -> + tagMap[a.text()] + } } Manga( id = generateUid(relativeUrl), title = linkTag.attr("title"), altTitle = null, - description = it.selectFirst("p.al-j.break.line-height-15")?.text(), + description = el.selectFirst("p.al-j.break.line-height-15")?.text(), url = relativeUrl, publicUrl = relativeUrl.toAbsoluteUrl(domain), coverUrl = linkTag.selectLast("img")?.imageUrl().orEmpty(), source = source, - tags = tags, + tags = tags ?: emptySet(), isNsfw = false, rating = RATING_UNKNOWN, author = null, @@ -235,18 +241,11 @@ class BlogTruyenParser(context: MangaLoaderContext) : } override suspend fun getTags(): Set { - val map = getOrCreateTagMap() - val tags = HashSet(map.size) - for (entry in map) { - tags.add(entry.value) - } - - return tags + return cacheTags.get().values.toSet() } - private suspend fun getOrCreateTagMap(): ArrayMap = mutex.withLock { - cacheTags?.let { return@withLock it } + private suspend fun fetchTags(): Map { val doc = webClient.httpGet("/timkiem/nangcao".toAbsoluteUrl(domain)).parseHtml() val tagItems = doc.select("li[data-id]") val tagMap = ArrayMap(tagItems.size) @@ -258,9 +257,7 @@ class BlogTruyenParser(context: MangaLoaderContext) : source = source, ) } - - cacheTags = tagMap - tagMap + return tagMap } private fun Element.imageUrl(): String {