[Nhentai World + MeHentaiVN] Fixes (#1769)

* [HentaiRead] Fix lines

* [MeHentaiVN] Soft re-write due to source changed (#1604)

* [NHentaiWorld] Fixes fetchTags (due to source changed) and pass MangaParserTest:link (#1604)

* [MeHentaiVN] Small changes

* [MeHentaiVN] Update domain

---------

Solve task #1604

Co-authored-by: Draken <131387159+dragonx943@users.noreply.github.com>

src/main/kotlin/org/koitharu/kotatsu/parsers/site/galleryadults/all/HentaiRead.kt

@@ -227,7 +227,7 @@ internal class HentaiRead(context: MangaLoaderContext) :
 
 	override suspend fun getDetails(manga: Manga): Manga {
 		val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml()
-        	val dateFormat = SimpleDateFormat("MMMM d, yyyy h:mm a", Locale.ENGLISH)
+
 		val title = buildString {
 			val mangaTitle = doc.selectFirst(selectTitle)?.text()?.cleanupTitle()
 			val parody = doc.selectFirst(selectParody)?.nextElementSibling()?.select("span:first-child")?.text()
@@ -255,7 +255,8 @@ internal class HentaiRead(context: MangaLoaderContext) :
 			}
 		}
 
-		val uploadedDateString = doc.selectFirst(selectUploadedDate)?.nextElementSibling()?.text()
+		val dateFormat = SimpleDateFormat("MMMM d, yyyy h:mm a", Locale.ENGLISH)
+		val uploadDateString = doc.selectFirst(selectUploadedDate)?.nextElementSibling()?.text()
 
 		var tags = manga.tags
 		if (tags.count() == 0) {
@@ -281,7 +282,7 @@ internal class HentaiRead(context: MangaLoaderContext) :
 					volume = 0,
 					url = manga.url,
 					scanlator = null,
-					uploadDate = dateFormat.tryParse(uploadedDateString),
+					uploadDate = dateFormat.tryParse(uploadDateString),
 					branch = "English",
 					source = source,
 				)

src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/NhentaiWorld.kt

@@ -1,8 +1,8 @@
 package org.koitharu.kotatsu.parsers.site.vi
 
 import okhttp3.Headers
+import okio.ByteString.Companion.encode
 import org.json.JSONArray
-import org.json.JSONObject
 import org.koitharu.kotatsu.parsers.MangaLoaderContext
 import org.koitharu.kotatsu.parsers.MangaSourceParser
 import org.koitharu.kotatsu.parsers.config.ConfigKey
@@ -148,6 +148,7 @@ internal class NhentaiWorld(context: MangaLoaderContext) :
 		val chapters = parseChapterList(scriptTag, manga, chapterDateFormat)
 
 		return manga.copy(
+			title = doc.selectFirst("h1")!!.text(),
 			tags = tags,
 			state = state,
 			description = description,
@@ -156,7 +157,7 @@ internal class NhentaiWorld(context: MangaLoaderContext) :
 		)
 	}
 
- 	private suspend fun parseChapterList(scriptTag: String?, manga: Manga, chapterDateFormat: SimpleDateFormat): List<MangaChapter> {
+	private fun parseChapterList(scriptTag: String?, manga: Manga, chapterDateFormat: SimpleDateFormat): List<MangaChapter> {
 		val idManga = manga.url.substringAfter("detail/").toIntOrNull() ?: return emptyList()
 
 		val chapters = ArrayList<MangaChapter>()
@@ -264,40 +265,30 @@ internal class NhentaiWorld(context: MangaLoaderContext) :
 	}
 
 	private suspend fun fetchTags(): Set<MangaTag> {
-		val doc = webClient.httpGet(
+		val doc = webClient.httpGet("https://$domain").parseHtml()
-			urlBuilder()
+		val scriptSrc = doc.select("script")[7].src()!!
-				.addPathSegment("genre")
+		val docJS = webClient.httpGet(scriptSrc).parseRaw()
-				.addPathSegment("all")
-				.build(),
-		).parseHtml()
 
-		val scriptTag = doc.select("script").firstOrNull { script ->
+		val optionsStart = docJS.indexOf("genres:[{")
-			val data = script.data()
-			data.contains("buildId") && data.contains("options")
-		}?.data() ?: return emptySet()
-		
-		val cleanedScript = scriptTag.replace("\\", "")
-		
-		val optionsPrefix = "\"options\":"
-		val optionsStart = cleanedScript.indexOf(optionsPrefix)
 		if (optionsStart == -1) return emptySet()
 
-		val optionsEnd = cleanedScript.indexOf("\"zombie\"}]", optionsStart) + "\"zombie\"}]".length
+		val optionsEnd = docJS.indexOf("}]", optionsStart)
 		if (optionsEnd == -1) return emptySet()
 
-		val optionsStr = cleanedScript.substring(optionsStart + optionsPrefix.length, optionsEnd)
+		val optionsStr = docJS.substring(optionsStart + 7, optionsEnd + 2)
 
-		val optionsArray = try {
+		val optionsArray = JSONArray(
-			JSONArray(optionsStr)
+			optionsStr
-		} catch (e: Exception) {
+				.replace(Regex(",description:\\s*\"[^\"]*\"(,?)"), "")
-			return emptySet()
+				.replace(Regex("(\\w+):"), "\"$1\":")
-		}
+		)
 
 		return buildSet {
 			for (i in 0 until optionsArray.length()) {
+				// {"label":"Ahegao","href":"/genre/ahegao"}
 				val option = optionsArray.getJSONObject(i)
-				val title = option.getStringOrNull("label")?.toTitleCase(sourceLocale) ?: continue
+				val title = option.getStringOrNull("label")!!.toTitleCase(sourceLocale)
-				val key = option.getStringOrNull("value") ?: continue
+				val key = option.getStringOrNull("href")!!.split("/")[2]
 				if (title.isNotEmpty() && key.isNotEmpty()) {
 					if (title != "Tất cả" || key != "all") { // remove "All" tags, default list = all
 						add(MangaTag(title = title, key = key, source = source))

src/main/kotlin/org/koitharu/kotatsu/parsers/site/wpcomics/vi/MeHentaiVN.kt

@@ -13,25 +13,30 @@ import org.koitharu.kotatsu.parsers.site.wpcomics.WpComicsParser
 import org.koitharu.kotatsu.parsers.exception.NotFoundException
 import org.koitharu.kotatsu.parsers.model.*
 import org.koitharu.kotatsu.parsers.util.*
+import java.lang.NullPointerException
+import java.net.URL
 import java.util.*
 
 @MangaSourceParser("MEHENTAIVN", "MeHentaiVN", "vi", ContentType.HENTAI)
 internal class MeHentaiVN(context: MangaLoaderContext) :
 	WpComicsParser(context, MangaParserSource.MEHENTAIVN, "www.mehentaivn.xyz", 44) {
 
-	override val configKeyDomain: ConfigKey.Domain = ConfigKey.Domain("www.mehentaivn.xyz", "www.hentaivnx.autos")
+	override val configKeyDomain: ConfigKey.Domain = ConfigKey.Domain(
-
+		"www.mehentaivn.xyz",
-	override val userAgentKey = ConfigKey.UserAgent(UserAgents.CHROME_DESKTOP)
+		"www.hentaivnx.autos",
-
+		"www.hentaivnx.com"
-	override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) {
+	)
-		super.onCreateConfig(keys)
-		keys.add(userAgentKey)
-	}
 
 	override fun getRequestHeaders() = super.getRequestHeaders().newBuilder()
-		.add("referer", "no-referrer")
+		.add("referer", "https://$domain/")
 		.build()
 
+	override val filterCapabilities: MangaListFilterCapabilities
+		get() = super.filterCapabilities.copy(
+			isMultipleTagsSupported = true,
+			isTagsExclusionSupported = true
+		)
+
 	override suspend fun getFilterOptions() = MangaListFilterOptions(
 		availableTags = fetchTags(),
 		availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED),
@@ -40,6 +45,7 @@ internal class MeHentaiVN(context: MangaLoaderContext) :
 	override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> {
 		val response =
 			when {
+				// url template: https://www.mehentaivn.xyz/tim-truyen?keyword=${query}
 				!filter.query.isNullOrEmpty() -> {
 					val url = buildString {
 						append("https://")
@@ -47,11 +53,12 @@ internal class MeHentaiVN(context: MangaLoaderContext) :
 						append(listUrl)
 						append("?keyword=")
 						append(filter.query.urlEncoded())
-						append("&page=")
+						if (page > 1) {
-						append(page.toString())
+							append("&page=$page")
+						}
 					}
 
-					val result = runCatchingCancellable { webClient.httpGet(url) }
+					val result = runCatchingCancellable { webClient.httpGet(url) } // execute
 					val exception = result.exceptionOrNull()
 					if (exception is NotFoundException) {
 						return emptyList()
@@ -59,54 +66,66 @@ internal class MeHentaiVN(context: MangaLoaderContext) :
 					result.getOrThrow()
 				}
 
+				// url tempalte: https://www.mehentaivn.xyz/tim-truyen-nang-cao?{query}
+				// Query Structure:
+				// genres=19775801,1&                /* tags include */
+				// notgenres=19776383,19777327&      /* tags exclude */
+				// minchapter=0&                     /* chapter count. Leaves 0 to get everything */
+				// sort=15&                          /* Sort order */
+				// contain=                          /* Not supported */
 				else -> {
+					val queries = mutableListOf<String>()
+
+					// tags
+					queries.add("genres=${filter.tags.joinToString (",") { it.key }}")
+
+					// tags exclude
+					queries.add("notgenres=${filter.tagsExclude.joinToString (",") { it.key }}")
+
+					if (filter.tags.isNotEmpty() or filter.tagsExclude.isNotEmpty()) {
+						// This means our query is not empty!
 						val url = buildString {
-						append("https://")
+							append("http://$domain/tim-truyen-nang-cao?")
-						append(domain)
+							append(queries.joinToString("&"))
-						append(listUrl)
+
-						if (filter.tags.isNotEmpty()) {
+							// order
-							append('/')
-							filter.tags.oneOrThrowIfMany()?.let {
-								append(it.key)
-							}
-						}
-						append("?sort=")
-						append(
 							when (order) {
-								SortOrder.UPDATED -> 0
+								SortOrder.NEWEST ->     append("&sort=15")      // Truyện mới
-								SortOrder.POPULARITY -> 10
+								SortOrder.POPULARITY -> append("&sort=10")      // Top all
-								SortOrder.NEWEST -> 15
+								SortOrder.UPDATED ->    append("&sort=0")       // Truyện mới
-								SortOrder.RATING -> 20
+								SortOrder.RATING ->     append("&sort=20")      // Theo dõi
 								else -> throw IllegalArgumentException("Sort order ${order.name} not supported")
-							},
-						)
-						filter.states.oneOrThrowIfMany()?.let {
-							append("&status=")
-							append(
-								when (it) {
-									MangaState.ONGOING -> "1"
-									MangaState.FINISHED -> "2"
-									else -> "-1"
-								},
-							)
 							}
-						append("&page=")
+
-						append(page.toString())
+							if (page > 1) {
+								append("&page=$page")
+							}
+						}
+
+						webClient.httpGet(url) // execute
+						
+					} else {
+						val url = buildString {
+							append("https://$domain/")
+							if (page > 1) {
+								append("?page=$page")
+							}
 						}
 
 						webClient.httpGet(url)
 					}
 				}
+			}
 
 		val tagMap = getOrCreateTagMap()
 		return parseSearchList(response.parseHtml(), tagMap)
 	}
 
-	private suspend fun parseSearchList(doc: Document, tagMap: ArrayMap<String, MangaTag>): List<Manga> {
+	private fun parseSearchList(doc: Document, tagMap: ArrayMap<String, MangaTag>): List<Manga> {
 		return doc.select("div.items div.item").mapNotNull { item ->
 			val tooltipElement = item.selectFirst("div.box_tootip")
 			val absUrl = item.selectFirst("div.image > a")?.attrAsAbsoluteUrlOrNull("href") ?: return@mapNotNull null
-			val slug = absUrl.substringAfterLast('/')
+			val url = absUrl.toRelativeUrl(domain)
 			val mangaState =
 				when (tooltipElement?.selectFirst("div.message_main > p:contains(Tình trạng)")?.ownText()) {
 					in ongoing -> MangaState.ONGOING
@@ -117,13 +136,12 @@ internal class MeHentaiVN(context: MangaLoaderContext) :
 				tooltipElement?.selectFirst("div.message_main > p:contains(Thể loại)")?.ownText().orEmpty()
 			val mangaTags = tagsElement.split(',').mapNotNullToSet { tagMap[it.trim()] }
 			val author = tooltipElement?.selectFirst("div.message_main > p:contains(Tác giả)")?.ownText()
-			val coverUrl = item.selectFirst("div.image a img")?.requireSrc()
+			val coverUrl = checkImgUrl(item.selectFirst("div.image a img")?.requireSrc())
-			val largeCoverUrl = null
 			Manga(
-				id = generateUid(slug),
+				id = generateUid(url),
 				title = item.selectFirst("div.box_tootip div.title, h3 a")?.text().orEmpty(),
 				altTitles = emptySet(),
-				url = absUrl.toRelativeUrl(domain),
+				url = url,
 				publicUrl = absUrl,
 				rating = RATING_UNKNOWN,
 				contentRating = null,
@@ -157,7 +175,8 @@ internal class MeHentaiVN(context: MangaLoaderContext) :
 		val author = doc.body().selectFirst(selectAut)?.textOrNull()
 
 		manga.copy(
-			description = doc.selectFirst(selectDesc)?.html(),
+			title = doc.select("h1.title-detail").text(),
+			description = "", // no more description for manga on this source
 			altTitles = setOfNotNull(doc.selectFirst("h2.other-name")?.textOrNull()),
 			authors = setOfNotNull(author),
 			state = doc.selectFirst(selectState)?.let {
@@ -175,24 +194,9 @@ internal class MeHentaiVN(context: MangaLoaderContext) :
 	}
 
 	override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
-		val fullUrl = chapter.url.toAbsoluteUrl(domain)
+		val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml()
-		val doc = webClient.httpGet(fullUrl).parseHtml()
+		return doc.select(".page-chapter img").map {
-		
+			val url = checkImgUrl(it.requireSrc())
-		val imageUrls = doc.select("div.page-chapter").flatMap { div ->
-			div.select("img").mapNotNull { img ->
-				val src = img.attr("src").takeIf { it.isNotEmpty() }
-				val dataSrc = img.attr("data-src").takeIf { it.isNotEmpty() }
-				val imageUrl = src ?: dataSrc
-				
-				if (imageUrl != null && checkMangaImgs(imageUrl)) {
-					imageUrl
-				} else {
-					null
-				}
-			}
-		}
-
-		return imageUrls.map { url ->
 			MangaPage(
 				id = generateUid(url),
 				url = url,
@@ -202,14 +206,14 @@ internal class MeHentaiVN(context: MangaLoaderContext) :
 		}
 	}
 
-	private suspend fun checkMangaImgs(url: String): Boolean {
+	private fun checkImgUrl (url: String?) : String {
-		return try {
+		if (url.isNullOrEmpty()) return ""
-			val response = webClient.httpHead(url)
+		val urlImage = URL(url)
-			val contentType = response.header("Content-Type") ?: ""
+
-			contentType.startsWith("image/")
+		// Need updating frequently
-		} catch (e: Exception) {
+		if (urlImage.host.contains("duckduckgo.com")) return url.split("?u=")[1]
-			false
+
-		}
+		return url
 	}
 
 	private suspend fun fetchTags(): Set<MangaTag> {