[madtheme/en + wpcomics/vi] Small fixes (#1830)

* [WpComics] Split NetTruyenX + Small improvements

* [MangaBuddy] Add subdomain

* [MangaPuma] Add subdomain

* [MangaForest] Add subdomain

* [MangaCute] Add subdomain

* [Manhuagui] Change UA to `CHROME_DESKTOP`

* [Mangaxyz] Add subdomain

.github/summary.yaml

@@ -1 +1 @@
-total: 1233
+total: 1234

src/main/kotlin/org/koitharu/kotatsu/parsers/site/madtheme/en/MangaBuddy.kt

@@ -2,9 +2,36 @@ package org.koitharu.kotatsu.parsers.site.madtheme.en
 
 import org.koitharu.kotatsu.parsers.MangaLoaderContext
 import org.koitharu.kotatsu.parsers.MangaSourceParser
+import org.koitharu.kotatsu.parsers.model.MangaChapter
+import org.koitharu.kotatsu.parsers.model.MangaPage
 import org.koitharu.kotatsu.parsers.model.MangaParserSource
 import org.koitharu.kotatsu.parsers.site.madtheme.MadthemeParser
+import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl
+import org.koitharu.kotatsu.parsers.util.generateUid
+import org.koitharu.kotatsu.parsers.util.parseHtml
 
 @MangaSourceParser("MANGABUDDY", "MangaBuddy", "en")
 internal class MangaBuddy(context: MangaLoaderContext) :
-	MadthemeParser(context, MangaParserSource.MANGABUDDY, "mangabuddy.com")
+	MadthemeParser(context, MangaParserSource.MANGABUDDY, "mangabuddy.com") {
+
+	private val subDomain = "sb.mbcdn.xyz"
+
+	override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
+		val fullUrl = chapter.url.toAbsoluteUrl(domain)
+		val doc = webClient.httpGet(fullUrl).parseHtml()
+		val regexPages = Regex("chapImages\\s*=\\s*['\"](.*?)['\"]")
+		val pages = doc.select("script").firstNotNullOfOrNull { script ->
+			regexPages.find(script.html())?.groupValues?.getOrNull(1)
+		}?.split(',')
+
+		return pages?.map { url ->
+			val cleanUrl = url.substringAfter("/manga")
+			MangaPage(
+				id = generateUid(url),
+				url = "https://$subDomain/manga$cleanUrl",
+				preview = null,
+				source = source,
+			)
+		} ?: emptyList()
+	}
+}

src/main/kotlin/org/koitharu/kotatsu/parsers/site/madtheme/en/MangaCute.kt

@@ -2,9 +2,36 @@ package org.koitharu.kotatsu.parsers.site.madtheme.en
 
 import org.koitharu.kotatsu.parsers.MangaLoaderContext
 import org.koitharu.kotatsu.parsers.MangaSourceParser
+import org.koitharu.kotatsu.parsers.model.MangaChapter
+import org.koitharu.kotatsu.parsers.model.MangaPage
 import org.koitharu.kotatsu.parsers.model.MangaParserSource
 import org.koitharu.kotatsu.parsers.site.madtheme.MadthemeParser
+import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl
+import org.koitharu.kotatsu.parsers.util.generateUid
+import org.koitharu.kotatsu.parsers.util.parseHtml
 
 @MangaSourceParser("MANGACUTE", "MangaCute", "en")
 internal class MangaCute(context: MangaLoaderContext) :
-	MadthemeParser(context, MangaParserSource.MANGACUTE, "mangacute.com")
+	MadthemeParser(context, MangaParserSource.MANGACUTE, "mangacute.com") {
+
+	private val subDomain = "sb.mbcdn.xyz"
+
+	override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
+		val fullUrl = chapter.url.toAbsoluteUrl(domain)
+		val doc = webClient.httpGet(fullUrl).parseHtml()
+		val regexPages = Regex("chapImages\\s*=\\s*['\"](.*?)['\"]")
+		val pages = doc.select("script").firstNotNullOfOrNull { script ->
+			regexPages.find(script.html())?.groupValues?.getOrNull(1)
+		}?.split(',')
+
+		return pages?.map { url ->
+			val cleanUrl = url.substringAfter("/manga")
+			MangaPage(
+				id = generateUid(url),
+				url = "https://$subDomain/manga$cleanUrl",
+				preview = null,
+				source = source,
+			)
+		} ?: emptyList()
+	}
+}

src/main/kotlin/org/koitharu/kotatsu/parsers/site/madtheme/en/MangaForest.kt

@@ -2,9 +2,36 @@ package org.koitharu.kotatsu.parsers.site.madtheme.en
 
 import org.koitharu.kotatsu.parsers.MangaLoaderContext
 import org.koitharu.kotatsu.parsers.MangaSourceParser
+import org.koitharu.kotatsu.parsers.model.MangaChapter
+import org.koitharu.kotatsu.parsers.model.MangaPage
 import org.koitharu.kotatsu.parsers.model.MangaParserSource
 import org.koitharu.kotatsu.parsers.site.madtheme.MadthemeParser
+import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl
+import org.koitharu.kotatsu.parsers.util.generateUid
+import org.koitharu.kotatsu.parsers.util.parseHtml
 
 @MangaSourceParser("MANGAFOREST", "MangaForest", "en")
 internal class MangaForest(context: MangaLoaderContext) :
-	MadthemeParser(context, MangaParserSource.MANGAFOREST, "mangaforest.me")
+	MadthemeParser(context, MangaParserSource.MANGAFOREST, "mangaforest.me") {
+	
+	private val subDomain = "sb.mbcdn.xyz"
+
+	override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
+		val fullUrl = chapter.url.toAbsoluteUrl(domain)
+		val doc = webClient.httpGet(fullUrl).parseHtml()
+		val regexPages = Regex("chapImages\\s*=\\s*['\"](.*?)['\"]")
+		val pages = doc.select("script").firstNotNullOfOrNull { script ->
+			regexPages.find(script.html())?.groupValues?.getOrNull(1)
+		}?.split(',')
+
+		return pages?.map { url ->
+			val cleanUrl = url.substringAfter("/manga")
+			MangaPage(
+				id = generateUid(url),
+				url = "https://$subDomain/manga$cleanUrl",
+				preview = null,
+				source = source,
+			)
+		} ?: emptyList()
+	}
+}

src/main/kotlin/org/koitharu/kotatsu/parsers/site/madtheme/en/MangaPuma.kt

@@ -2,9 +2,37 @@ package org.koitharu.kotatsu.parsers.site.madtheme.en
 
 import org.koitharu.kotatsu.parsers.MangaLoaderContext
 import org.koitharu.kotatsu.parsers.MangaSourceParser
+import org.koitharu.kotatsu.parsers.model.MangaChapter
+import org.koitharu.kotatsu.parsers.model.MangaPage
 import org.koitharu.kotatsu.parsers.model.MangaParserSource
 import org.koitharu.kotatsu.parsers.site.madtheme.MadthemeParser
+import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl
+import org.koitharu.kotatsu.parsers.util.generateUid
+import org.koitharu.kotatsu.parsers.util.parseHtml
 
 @MangaSourceParser("MANGAPUMA", "MangaPuma", "en")
 internal class MangaPuma(context: MangaLoaderContext) :
-	MadthemeParser(context, MangaParserSource.MANGAPUMA, "mangapuma.com")
+	MadthemeParser(context, MangaParserSource.MANGAPUMA, "mangapuma.com") {
+
+	private val subDomain = "sb.mbcdn.xyz"
+
+	override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
+		val fullUrl = chapter.url.toAbsoluteUrl(domain)
+		val doc = webClient.httpGet(fullUrl).parseHtml()
+		val regexPages = Regex("chapImages\\s*=\\s*['\"](.*?)['\"]")
+		val pages = doc.select("script").firstNotNullOfOrNull { script ->
+			regexPages.find(script.html())?.groupValues?.getOrNull(1)
+		}?.split(',')
+
+		return pages?.map { url ->
+			val cleanUrl = url.substringAfter("/manga")
+			MangaPage(
+				id = generateUid(url),
+				url = "https://$subDomain/manga$cleanUrl",
+				preview = null,
+				source = source,
+			)
+		} ?: emptyList()
+	}
+
+}

src/main/kotlin/org/koitharu/kotatsu/parsers/site/madtheme/en/Mangaxyz.kt

@@ -2,9 +2,37 @@ package org.koitharu.kotatsu.parsers.site.madtheme.en
 
 import org.koitharu.kotatsu.parsers.MangaLoaderContext
 import org.koitharu.kotatsu.parsers.MangaSourceParser
+import org.koitharu.kotatsu.parsers.model.MangaChapter
+import org.koitharu.kotatsu.parsers.model.MangaPage
 import org.koitharu.kotatsu.parsers.model.MangaParserSource
 import org.koitharu.kotatsu.parsers.site.madtheme.MadthemeParser
+import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl
+import org.koitharu.kotatsu.parsers.util.generateUid
+import org.koitharu.kotatsu.parsers.util.parseHtml
 
 @MangaSourceParser("MANGAXYZ", "MangaXyz", "en")
 internal class Mangaxyz(context: MangaLoaderContext) :
-	MadthemeParser(context, MangaParserSource.MANGAXYZ, "mangaxyz.com")
+	MadthemeParser(context, MangaParserSource.MANGAXYZ, "mangaxyz.com") {
+
+    private val subDomain = "sb.mbcdn.xyz"
+
+	override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
+		val fullUrl = chapter.url.toAbsoluteUrl(domain)
+		val doc = webClient.httpGet(fullUrl).parseHtml()
+		val regexPages = Regex("chapImages\\s*=\\s*['\"](.*?)['\"]")
+		val pages = doc.select("script").firstNotNullOfOrNull { script ->
+			regexPages.find(script.html())?.groupValues?.getOrNull(1)
+		}?.split(',')
+
+		return pages?.map { url ->
+			val cleanUrl = url.substringAfter("/manga")
+			MangaPage(
+				id = generateUid(url),
+				url = "https://$subDomain/manga$cleanUrl",
+				preview = null,
+				source = source,
+			)
+		} ?: emptyList()
+	}
+
+}

src/main/kotlin/org/koitharu/kotatsu/parsers/site/wpcomics/WpComicsParser.kt

@@ -145,6 +145,8 @@ internal abstract class WpComicsParser(
 		return parseMangaList(response.parseHtml(), tagMap)
 	}
 
+	protected open val coverDiv = "div.image a img"
+
 	protected open fun parseMangaList(doc: Document, tagMap: ArrayMap<String, MangaTag>): List<Manga> {
 		return doc.select("div.items div.item").mapNotNull { item ->
 			val tooltipElement = item.selectFirst("div.box_tootip")
@@ -168,7 +170,7 @@ internal abstract class WpComicsParser(
 				publicUrl = absUrl,
 				rating = RATING_UNKNOWN,
 				contentRating = null,
-				coverUrl = item.selectFirst("div.image a img")?.findImageUrl().orEmpty(),
+				coverUrl = item.selectFirst(coverDiv)?.findImageUrl().orEmpty(),
 				largeCoverUrl = null,
 				tags = mangaTags,
 				state = mangaState,

src/main/kotlin/org/koitharu/kotatsu/parsers/site/wpcomics/vi/NetTruyen.kt

@@ -15,11 +15,6 @@ import java.text.SimpleDateFormat
 internal class NetTruyen(context: MangaLoaderContext) :
 	WpComicsParser(context, MangaParserSource.NETTRUYEN, "nettruyener.com", 36) {
 
-	override val configKeyDomain: ConfigKey.Domain = ConfigKey.Domain(
-		"nettruyener.com",
-		"nettruyenx.net",
-	)
-
 	override suspend fun getDetails(manga: Manga): Manga = coroutineScope {
 		val fullUrl = manga.url.toAbsoluteUrl(domain)
 		val docDeferred = async { webClient.httpGet(fullUrl).parseHtml() }

src/main/kotlin/org/koitharu/kotatsu/parsers/site/wpcomics/vi/NetTruyenX.kt

@@ -0,0 +1,77 @@
+package org.koitharu.kotatsu.parsers.site.wpcomics.vi
+
+import kotlinx.coroutines.async
+import kotlinx.coroutines.coroutineScope
+import org.koitharu.kotatsu.parsers.MangaLoaderContext
+import org.koitharu.kotatsu.parsers.MangaSourceParser
+import org.koitharu.kotatsu.parsers.config.ConfigKey
+import org.koitharu.kotatsu.parsers.model.*
+import org.koitharu.kotatsu.parsers.site.wpcomics.WpComicsParser
+import org.koitharu.kotatsu.parsers.util.*
+import org.koitharu.kotatsu.parsers.util.json.getStringOrNull
+import java.text.SimpleDateFormat
+
+@MangaSourceParser("NETTRUYENX", "NetTruyenX", "vi")
+internal class NetTruyenX(context: MangaLoaderContext) :
+	WpComicsParser(context, MangaParserSource.NETTRUYENX, "nettruyenx.net", 36) {
+
+    override fun getRequestHeaders() = super.getRequestHeaders().newBuilder()
+		.add("referer", "https://$domain/")
+		.build()
+
+    override val selectDesc = "div.detail-content div.shortened"
+    override val selectState = "li.status p.col-xs-8"
+    override val selectAut = "li.author p.col-xs-8"
+
+	override suspend fun getDetails(manga: Manga): Manga = coroutineScope {
+		val fullUrl = manga.url.toAbsoluteUrl(domain)
+		val docDeferred = async { webClient.httpGet(fullUrl).parseHtml() }
+		val chaptersDeferred = async { fetchChapters(manga.url) }
+		val tagMap = getOrCreateTagMap()
+		val doc = docDeferred.await()
+		val tagsElement = doc.select("li.kind p.col-xs-8 a")
+		val mangaTags = tagsElement.mapNotNullToSet { tagMap[it.text()] }
+		val author = doc.body().select(selectAut).textOrNull()
+		manga.copy(
+			description = doc.selectFirst(selectDesc)?.html(),
+			altTitles = setOfNotNull(doc.selectFirst("h2.other-name")?.textOrNull()),
+			authors = setOfNotNull(author),
+			state = doc.selectFirst(selectState)?.let {
+				when (it.text()) {
+					in ongoing -> MangaState.ONGOING
+					in finished -> MangaState.FINISHED
+					else -> null
+				}
+			},
+			tags = mangaTags,
+			rating = doc.selectFirst("div.star input[name=score]")?.attr("value")?.toFloatOrNull()?.div(5f) ?: RATING_UNKNOWN,
+			chapters = chaptersDeferred.await(),
+		)
+	}
+
+	private suspend fun fetchChapters(mangaUrl: String): List<MangaChapter> {
+		val slug = mangaUrl.substringAfterLast('/')
+		val chaptersUrl = "/Comic/Services/ComicService.asmx/ChapterList?slug=$slug".toAbsoluteUrl(domain)
+		val df = SimpleDateFormat("yyyy-MM-dd HH:mm:ss")
+
+		val data = webClient.httpGet(chaptersUrl).parseJson().getJSONArray("data")
+		return List(data.length()) { i ->
+			val jo = data.getJSONObject(data.length() - 1 - i)
+			val chapterSlug = jo.getString("chapter_slug")
+			val chapterId = jo.getString("chapter_id")
+			val chapterUrl = "/truyen-tranh/$slug/$chapterSlug/$chapterId"
+
+			MangaChapter(
+				id = generateUid(chapterUrl),
+				title = jo.getStringOrNull("chapter_name"),
+				number = i + 1f,
+				volume = 0,
+				url = chapterUrl,
+				scanlator = null,
+				uploadDate = df.tryParse(jo.getString("updated_at")),
+				branch = null,
+				source = source,
+			)
+		}
+	}
+}