KomikTap parser #40

[NetTruyen] New source

.gitignore

@@ -75,4 +75,5 @@ build/
 .idea/**/ktlint.xml
 .idea/codeStyles/
 
-src/test/resources/cookies.txt
+src/test/resources/cookies.txt
+local.properties

build.gradle

@@ -20,6 +20,7 @@ compileKotlin {
 		freeCompilerArgs += [
 				'-opt-in=kotlin.RequiresOptIn',
 				'-opt-in=kotlin.contracts.ExperimentalContracts',
+				'-opt-in=kotlinx.coroutines.ExperimentalCoroutinesApi',
 				'-opt-in=org.koitharu.kotatsu.parsers.InternalParsersApi',
 		]
 	}
@@ -62,9 +63,9 @@ dependencies {
 
 	ksp project(':kotatsu-parsers-ksp')
 
-	testImplementation 'org.junit.jupiter:junit-jupiter-api:5.8.2'
-	testImplementation 'org.junit.jupiter:junit-jupiter-engine:5.8.2'
-	testImplementation 'org.junit.jupiter:junit-jupiter-params:5.8.2'
+	testImplementation 'org.junit.jupiter:junit-jupiter-api:5.9.0'
+	testImplementation 'org.junit.jupiter:junit-jupiter-engine:5.9.0'
+	testImplementation 'org.junit.jupiter:junit-jupiter-params:5.9.0'
 	testImplementation 'org.jetbrains.kotlinx:kotlinx-coroutines-test:1.6.4'
 	testImplementation 'io.webfolder:quickjs:1.1.0'
 }

src/main/kotlin/org/koitharu/kotatsu/parsers/site/BlogTruyenParser.kt

@@ -0,0 +1,248 @@
+package org.koitharu.kotatsu.parsers.site
+
+import androidx.collection.ArrayMap
+import kotlinx.coroutines.sync.Mutex
+import kotlinx.coroutines.sync.withLock
+import org.json.JSONArray
+import org.jsoup.nodes.Document
+import org.jsoup.nodes.Element
+import org.koitharu.kotatsu.parsers.MangaLoaderContext
+import org.koitharu.kotatsu.parsers.MangaSourceParser
+import org.koitharu.kotatsu.parsers.PagedMangaParser
+import org.koitharu.kotatsu.parsers.config.ConfigKey
+import org.koitharu.kotatsu.parsers.model.*
+import org.koitharu.kotatsu.parsers.util.*
+import java.text.SimpleDateFormat
+import java.util.*
+import kotlin.collections.HashSet
+
+@MangaSourceParser("BLOGTRUYEN", "BlogTruyen", "vi")
+class BlogTruyenParser(override val context: MangaLoaderContext) :
+	PagedMangaParser(MangaSource.BLOGTRUYEN, pageSize = 20) {
+
+	override val configKeyDomain: ConfigKey.Domain
+		get() = ConfigKey.Domain("blogtruyen.vn", null)
+
+	override val sortOrders: Set<SortOrder>
+		get() = EnumSet.of(SortOrder.UPDATED)
+
+	private val mutex = Mutex()
+	private val dateFormat = SimpleDateFormat("dd/MM/yyyy HH:mm", Locale.US)
+	private var cacheTags: ArrayMap<String, MangaTag>? = null
+
+	override suspend fun getDetails(manga: Manga): Manga {
+		val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml()
+		val descriptionElement = doc.selectFirstOrThrow("div.description")
+		val statusText = descriptionElement
+			.selectFirst("p:contains(Trạng thái) > span.color-red")
+			?.text()
+
+		val state = when (statusText) {
+			"Đang tiến hành" -> MangaState.ONGOING
+			"Đã hoàn thành" -> MangaState.FINISHED
+			else -> null
+		}
+
+		val rating = doc.selectFirst("span.total-vote")?.attr("ng-init")?.let { text ->
+			val like = text.substringAfter("TotalLike=")
+				.substringBefore(';')
+				.toIntOrNull() ?: return@let RATING_UNKNOWN
+			val dislike = text.substringAfter("TotalDisLike=")
+				.toIntOrNull() ?: return@let RATING_UNKNOWN
+
+			when {
+				like == 0 && dislike == 0 -> RATING_UNKNOWN
+				else -> like.toFloat() / (like + dislike)
+			}
+		}
+
+		val tagMap = getOrCreateTagMap()
+		val tags = descriptionElement.select("p > span.category").mapNotNullToSet {
+			val tagName = it.selectFirst("a")?.text()?.trim() ?: return@mapNotNullToSet null
+			tagMap[tagName]
+		}
+
+		return manga.copy(
+			tags = tags,
+			author = descriptionElement.selectFirst("p:contains(Tác giả) > a")?.text(),
+			description = doc.selectFirst(".detail .content")?.html(),
+			chapters = parseChapterList(doc),
+			largeCoverUrl = doc.selectLast("div.thumbnail > img")?.attrAsAbsoluteUrlOrNull("src"),
+			state = state,
+			rating = rating ?: RATING_UNKNOWN,
+			isNsfw = doc.getElementById("warningCategory") != null
+		)
+	}
+
+	private fun parseChapterList(doc: Document): List<MangaChapter> {
+		val chapterList = doc.select("#list-chapters > p")
+		return chapterList.asReversed().mapChapters { index, element ->
+			val titleElement = element.selectFirst("span.title > a") ?: return@mapChapters null
+			val name = titleElement.text()
+			val relativeUrl = titleElement.attrAsRelativeUrl("href")
+			val id = relativeUrl.substringAfter('/').substringBefore('/')
+			val uploadDate = dateFormat.tryParse(element.select("span.publishedDate").text())
+			MangaChapter(
+				id = generateUid(id),
+				name = name,
+				number = index + 1,
+				url = relativeUrl,
+				scanlator = null,
+				uploadDate = uploadDate,
+				branch = null,
+				source = source
+			)
+		}
+	}
+
+	override suspend fun getListPage(
+		page: Int,
+		query: String?,
+		tags: Set<MangaTag>?,
+		sortOrder: SortOrder,
+	): List<Manga> {
+		return when {
+			!query.isNullOrEmpty() -> {
+				val searchUrl = "https://${getDomain()}/timkiem/nangcao/1/0/-1/-1?txt=${query.urlEncoded()}&p=$page"
+				val searchContent = context.httpGet(searchUrl).parseHtml()
+					.selectFirst("section.list-manga-bycate > div.list")
+				parseMangaList(searchContent)
+			}
+
+			!tags.isNullOrEmpty() -> {
+				val tag = tags.oneOrThrowIfMany()!!
+				val categoryAjax = "https://${getDomain()}/ajax/Category/AjaxLoadMangaByCategory?id=${tag.key}&orderBy=5&p=$page"
+				val listContent = context.httpGet(categoryAjax).parseHtml().selectFirst("div.list")
+				parseMangaList(listContent)
+			}
+			else -> getNormalList(page)
+		}
+	}
+
+	private suspend fun getNormalList(page: Int): List<Manga> {
+		val pageLink = "https://${getDomain()}/page-$page"
+		val doc = context.httpGet(pageLink).parseHtml()
+		val listElements = doc.selectFirstOrThrow("section.list-mainpage.listview")
+			.select("div.bg-white.storyitem")
+
+		return listElements.mapNotNull {
+			val linkTag = it.selectFirst("div.fl-l > a") ?: return@mapNotNull null
+			val relativeUrl = linkTag.attrAsRelativeUrl("href")
+			val tagMap = getOrCreateTagMap()
+			val tags = it.select("footer > div.category > a").mapNotNullToSet { a ->
+				tagMap[a.text()]
+			}
+
+			Manga(
+				id = generateUid(relativeUrl),
+				title = linkTag.attr("title"),
+				altTitle = null,
+				description = it.selectFirst("p.al-j.break.line-height-15")?.text(),
+				url = relativeUrl,
+				publicUrl = relativeUrl.toAbsoluteUrl(getDomain()),
+				coverUrl = linkTag.selectLast("img")?.attr("src").orEmpty(),
+				source = source,
+				tags = tags,
+				isNsfw = false,
+				rating = RATING_UNKNOWN,
+				author = null,
+				state = null,
+			)
+		}
+	}
+
+	private fun parseMangaList(listElement: Element?): List<Manga> {
+		listElement ?: return emptyList()
+
+		return listElement.select("span.tiptip[data-tiptip]").mapNotNull {
+			val mangaInfo = listElement.getElementById(it.attr("data-tiptip")) ?: return@mapNotNull null
+			val a = it.selectFirst("a") ?: return@mapNotNull null
+			val relativeUrl = a.attrAsRelativeUrl("href")
+			Manga(
+				id = generateUid(relativeUrl),
+				title = a.text(),
+				altTitle = null,
+				description = mangaInfo.select("div.al-j.fs-12").text(),
+				url = relativeUrl,
+				publicUrl = relativeUrl.toAbsoluteUrl(getDomain()),
+				coverUrl = mangaInfo.selectFirst("div > img.img")?.absUrl("src").orEmpty(),
+				isNsfw = false,
+				rating = RATING_UNKNOWN,
+				tags = emptySet(),
+				author = null,
+				state = null,
+				source = source,
+			)
+		}
+	}
+
+	override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
+		fun generateImageId(index: Int) = generateUid("${chapter.url}/$index")
+
+		val doc = context.httpGet(chapter.url.toAbsoluteUrl(getDomain())).parseHtml()
+		val pages = ArrayList<MangaPage>()
+		val referer = chapter.url.toAbsoluteUrl(getDomain())
+		doc.select("#content > img").forEach { img ->
+			val url = img.attrAsRelativeUrl("src")
+			pages.add(
+				MangaPage(
+					id = generateImageId(pages.lastIndex),
+					url = url,
+					referer = referer,
+					preview = null,
+					source = source,
+				)
+			)
+		}
+
+		// Some chapters use js script to render images
+		val script = doc.selectLast("#content > script")
+		if (script != null && script.data().contains("listImageCaption")) {
+			val imagesStr = script.data().substringBefore(';').substringAfterLast('=').trim()
+			val imageArr = JSONArray(imagesStr)
+			for (i in 0 until imageArr.length()) {
+				val imageUrl = imageArr.getJSONObject(i).getString("url")
+				pages.add(
+					MangaPage(
+						id = generateImageId(pages.lastIndex),
+						url = imageUrl,
+						referer = referer,
+						preview = null,
+						source = source
+					)
+				)
+			}
+		}
+
+		return pages
+	}
+
+	override suspend fun getTags(): Set<MangaTag> {
+		val map = getOrCreateTagMap()
+		val tags = HashSet<MangaTag>(map.size)
+		for (entry in map) {
+			tags.add(entry.value)
+		}
+
+		return tags
+	}
+
+
+	private suspend fun getOrCreateTagMap(): ArrayMap<String, MangaTag> = mutex.withLock {
+		cacheTags?.let { return@withLock it }
+		val doc = context.httpGet("/timkiem/nangcao".toAbsoluteUrl(getDomain())).parseHtml()
+		val tagItems = doc.select("li[data-id]")
+		val tagMap = ArrayMap<String, MangaTag>(tagItems.size)
+		for (tag in tagItems) {
+			val title = tag.text().trim()
+			tagMap[tag.text().trim()] = MangaTag(
+				title = title,
+				key = tag.attr("data-id"),
+				source = source
+			)
+		}
+
+		cacheTags = tagMap
+		tagMap
+	}
+}

src/main/kotlin/org/koitharu/kotatsu/parsers/site/KomikTapParser.kt

@@ -0,0 +1,189 @@
+package org.koitharu.kotatsu.parsers.site
+
+import org.json.JSONObject
+import org.jsoup.nodes.Element
+import org.koitharu.kotatsu.parsers.MangaLoaderContext
+import org.koitharu.kotatsu.parsers.MangaSourceParser
+import org.koitharu.kotatsu.parsers.PagedMangaParser
+import org.koitharu.kotatsu.parsers.config.ConfigKey
+import org.koitharu.kotatsu.parsers.model.*
+import org.koitharu.kotatsu.parsers.util.*
+import org.koitharu.kotatsu.parsers.util.json.associateByKey
+import org.koitharu.kotatsu.parsers.util.json.stringIterator
+import java.text.SimpleDateFormat
+import java.util.*
+
+@MangaSourceParser("KOMIKTAP", "KomikTap", "id")
+class KomikTapParser(
+	override val context: MangaLoaderContext,
+) : PagedMangaParser(MangaSource.KOMIKTAP, pageSize = 25, searchPageSize = 10) {
+
+	override val configKeyDomain = ConfigKey.Domain("194.233.66.232", arrayOf("194.233.66.232", "komiktap.in"))
+
+	override val sortOrders = EnumSet.of(
+		SortOrder.ALPHABETICAL,
+		SortOrder.POPULARITY,
+		SortOrder.UPDATED,
+		SortOrder.NEWEST,
+	)
+
+	override suspend fun getListPage(
+		page: Int,
+		query: String?,
+		tags: Set<MangaTag>?,
+		sortOrder: SortOrder,
+	): List<Manga> {
+		val url = urlBuilder()
+		if (query.isNullOrEmpty()) {
+			url.addPathSegment("manga")
+				.addQueryParameter("page", page.toString())
+				.addQueryParameter("status", "")
+				.addQueryParameter("type", "")
+				.addQueryParameter("order", sortOrder.asQueryParameter())
+			tags?.forEach {
+				url.addQueryParameter("genre[]", it.key)
+			}
+		} else {
+			url.addPathSegment("page")
+				.addPathSegment(page.toString())
+				.addQueryParameter("s", query)
+		}
+		val root = context.httpGet(url.build()).parseHtml().body()
+			.requireElementById("content")
+			.selectFirstOrThrow(".listupd")
+		return root.select("div.bs").map { div ->
+			val a = div.selectFirstOrThrow("a")
+			val img = div.selectFirstOrThrow("img")
+			val href = a.attrAsRelativeUrl("href")
+			val bigor = div.selectFirstOrThrow(".bigor")
+			Manga(
+				id = generateUid(href),
+				title = bigor.selectFirstOrThrow(".tt").text(),
+				altTitle = null,
+				url = href,
+				publicUrl = a.attrAsAbsoluteUrl("href"),
+				rating = bigor.selectFirst(".rating .numscore")
+					?.text()?.toFloatOrNull()?.div(10f)
+					.assertNotNull("rating") ?: RATING_UNKNOWN,
+				isNsfw = true,
+				coverUrl = img.attrAsAbsoluteUrl("src"),
+				tags = emptySet(),
+				state = when (div.selectFirst("span.status")?.text()) {
+					"Completed" -> MangaState.FINISHED
+					else -> null
+				},
+				author = null,
+				source = source,
+			)
+		}
+	}
+
+	override suspend fun getDetails(manga: Manga): Manga {
+		val root = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml().body()
+			.requireElementById("content")
+			.selectFirstOrThrow("article")
+		val table = root.selectFirstOrThrow(".infotable")
+		val dateFormat = SimpleDateFormat("MMM d, yyyy", checkNotNull(sourceLocale))
+		val chapters = root.requireElementById("chapterlist")
+			.select("li")
+			.mapChapters { index, li ->
+				val a = li.selectFirstOrThrow("a")
+				val href = a.attrAsRelativeUrl("href")
+				MangaChapter(
+					id = generateUid(href),
+					name = a.selectFirstOrThrow(".chapternum").text(),
+					number = index + 1,
+					url = href,
+					scanlator = null,
+					uploadDate = dateFormat.tryParse(a.selectFirst(".chapterdate")?.text()),
+					branch = null,
+					source = source,
+				)
+			}
+
+		return manga.copy(
+			largeCoverUrl = root.selectFirst("div.thumb")
+				?.selectFirst("img")
+				?.attrAsAbsoluteUrlOrNull("src")
+				.assertNotNull("largeCoverUrl"),
+			author = table.tableValue("Author").assertNotNull("author")?.takeUnless { it == "N/A" },
+			state = when (table.tableValue("Status").assertNotNull("status")) {
+				"Completed" -> MangaState.FINISHED
+				"Ongoing" -> MangaState.ONGOING
+				else -> null
+			},
+			tags = root.selectFirstOrThrow(".seriestugenre")
+				.select("a")
+				.mapToSet { a ->
+					MangaTag(
+						title = a.text().toTitleCase(Locale.ENGLISH),
+						key = a.attr("href").removeSuffix('/').substringAfterLast('/'),
+						source = manga.source,
+					)
+				},
+			description = root.selectFirstOrThrow("[itemprop=\"description\"]").html(),
+			chapters = chapters,
+		)
+	}
+
+	override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
+		val fullUrl = chapter.url.toAbsoluteUrl(getDomain())
+		val doc = context.httpGet(fullUrl).parseHtml()
+		val scripts = doc.select("script")
+		for (script in scripts) {
+			val content = script.html()
+			val pos = content.indexOf("ts_reader.run")
+			if (pos < 0) {
+				continue
+			}
+			val json = JSONObject(content.substringBetween("(", ")"))
+			val sources = json.getJSONArray("sources").associateByKey("source")
+			val images = (sources[json.optString("defaultSource")] ?: sources.values.first()).getJSONArray("images")
+			val result = ArrayList<MangaPage>(images.length())
+			images.stringIterator().forEach {
+				result += MangaPage(
+					id = generateUid(it),
+					url = it,
+					referer = fullUrl,
+					preview = null,
+					source = source,
+				)
+			}
+			return result
+		}
+		doc.parseFailed("Script with pages not found")
+	}
+
+	override suspend fun getTags(): Set<MangaTag> {
+		val root = context.httpGet("https://${getDomain()}/manga/").parseHtml()
+			.selectFirstOrThrow("form.filters")
+			.selectFirstOrThrow("ul.genrez")
+		return root.select("li").mapNotNullToSet { li ->
+			val input = li.selectFirstOrThrow("input")
+			if (input.attr("name") != "genre[]") {
+				return@mapNotNullToSet null
+			}
+			MangaTag(
+				title = li.selectFirstOrThrow("label").text().toTitleCase(sourceLocale ?: Locale.ENGLISH),
+				key = input.attrOrNull("value") ?: return@mapNotNullToSet null,
+				source = source,
+			)
+		}
+	}
+
+	override fun getFaviconUrl(): String {
+		return "https://${getDomain()}/wp-content/uploads/2020/09/cropped-LOGOa-180x180.png"
+	}
+
+	private fun SortOrder.asQueryParameter() = when (this) {
+		SortOrder.UPDATED -> "update"
+		SortOrder.POPULARITY -> "popular"
+		SortOrder.NEWEST -> "latest"
+		SortOrder.ALPHABETICAL -> "title"
+		else -> ""
+	}
+
+	private fun Element.tableValue(key: String): String? {
+		return getElementsMatchingOwnText(key).singleOrNull()?.parent()?.selectLast("td")?.text()
+	}
+}

src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaDexParser.kt

@@ -1,12 +1,15 @@
 package org.koitharu.kotatsu.parsers.site
 
+import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.async
+import kotlinx.coroutines.awaitAll
 import kotlinx.coroutines.coroutineScope
 import org.json.JSONObject
 import org.koitharu.kotatsu.parsers.MangaLoaderContext
 import org.koitharu.kotatsu.parsers.MangaParser
 import org.koitharu.kotatsu.parsers.MangaSourceParser
 import org.koitharu.kotatsu.parsers.config.ConfigKey
+import org.koitharu.kotatsu.parsers.exception.ParseException
 import org.koitharu.kotatsu.parsers.model.*
 import org.koitharu.kotatsu.parsers.util.*
 import org.koitharu.kotatsu.parsers.util.json.*
@@ -14,7 +17,9 @@ import java.text.SimpleDateFormat
 import java.util.*
 
 private const val PAGE_SIZE = 20
-private const val CHAPTERS_PAGE_SIZE = 60
+private const val CHAPTERS_FIRST_PAGE_SIZE = 120
+private const val CHAPTERS_MAX_PAGE_SIZE = 500
+private const val CHAPTERS_PARALLELISM = 3
 private const val CONTENT_RATING =
 	"contentRating[]=safe&contentRating[]=suggestive&contentRating[]=erotica&contentRating[]=pornographic"
 private const val LOCALE_FALLBACK = "en"
@@ -62,6 +67,7 @@ internal class MangaDexParser(override val context: MangaLoaderContext) : MangaP
 				when (sortOrder) {
 					SortOrder.UPDATED,
 					-> "[latestUploadedChapter]=desc"
+
 					SortOrder.ALPHABETICAL -> "[title]=asc"
 					SortOrder.NEWEST -> "[createdAt]=desc"
 					SortOrder.POPULARITY -> "[followedCount]=desc"
@@ -118,12 +124,13 @@ internal class MangaDexParser(override val context: MangaLoaderContext) : MangaP
 
 	override suspend fun getDetails(manga: Manga): Manga = coroutineScope {
 		val domain = getDomain()
+		val mangaId = manga.url.removePrefix("/")
 		val attrsDeferred = async {
 			context.httpGet(
-				"https://api.$domain/manga/${manga.url}?includes[]=artist&includes[]=author&includes[]=cover_art",
+				"https://api.$domain/manga/${mangaId}?includes[]=artist&includes[]=author&includes[]=cover_art",
 			).parseJson().getJSONObject("data").getJSONObject("attributes")
 		}
-		val feedDeferred = async { loadChapters(manga) }
+		val feedDeferred = async { loadChapters(mangaId) }
 		val mangaAttrs = attrsDeferred.await()
 		val feed = feedDeferred.await()
 		// 2022-01-02T00:27:11+00:00
@@ -134,11 +141,11 @@ internal class MangaDexParser(override val context: MangaLoaderContext) : MangaP
 		manga.copy(
 			description = mangaAttrs.getJSONObject("description").selectByLocale()
 				?: manga.description,
-			chapters = feed.mapNotNull { jo ->
+			chapters = feed.mapChapters { _, jo ->
 				val id = jo.getString("id")
 				val attrs = jo.getJSONObject("attributes")
 				if (!attrs.isNull("externalUrl")) {
-					return@mapNotNull null
+					return@mapChapters null
 				}
 				val locale = attrs.getStringOrNull("translatedLanguage")?.let { Locale.forLanguageTag(it) }
 				val relations = jo.getJSONArray("relationships").associateByKey("type")
@@ -201,30 +208,62 @@ internal class MangaDexParser(override val context: MangaLoaderContext) : MangaP
 		return getStringOrNull(LOCALE_FALLBACK) ?: values().nextOrNull() as? String
 	}
 
-	private suspend fun loadChapters(manga: Manga): List<JSONObject> {
-		val domain = getDomain()
-		val result = ArrayList<JSONObject>()
-		while (true) {
-			val url = buildString {
-				append("https://api.")
-				append(domain)
-				append("/manga/")
-				append(manga.url)
-				append("/feed")
-				append("?limit=")
-				append(CHAPTERS_PAGE_SIZE)
-				append("&includes[]=scanlation_group&order[volume]=asc&order[chapter]=asc&offset=")
-				append(result.size)
-				append('&')
-				append(CONTENT_RATING)
-			}
-			val json = context.httpGet(url).parseJson()
-			val data = json.optJSONArray("data")
-			if (data.isNullOrEmpty()) {
-				break
-			}
-			data.mapJSONTo(result) { it }
+	private suspend fun loadChapters(mangaId: String): List<JSONObject> {
+		val firstPage = loadChapters(mangaId, offset = 0, limit = CHAPTERS_FIRST_PAGE_SIZE)
+		if (firstPage.size >= firstPage.total) {
+			return firstPage.data
 		}
+		val tail = coroutineScope {
+			val leftCount = firstPage.total - firstPage.size
+			val pages = (leftCount / CHAPTERS_MAX_PAGE_SIZE.toFloat()).toIntUp()
+			val dispatcher = Dispatchers.Default.limitedParallelism(CHAPTERS_PARALLELISM)
+			List(pages) { page ->
+				val offset = page * CHAPTERS_MAX_PAGE_SIZE + firstPage.size
+				async(dispatcher) {
+					loadChapters(mangaId, offset, CHAPTERS_MAX_PAGE_SIZE)
+				}
+			}.awaitAll()
+		}
+		val result = ArrayList<JSONObject>(firstPage.total)
+		result += firstPage.data
+		tail.flatMapTo(result) { it.data }
 		return result
 	}
+
+	private suspend fun loadChapters(mangaId: String, offset: Int, limit: Int): Chapters {
+		val url = buildString {
+			append("https://api.")
+			append(getDomain())
+			append("/manga/")
+			append(mangaId)
+			append("/feed")
+			append("?limit=")
+			append(limit)
+			append("&includes[]=scanlation_group&order[volume]=asc&order[chapter]=asc&offset=")
+			append(offset)
+			append('&')
+			append(CONTENT_RATING)
+		}
+		val json = context.httpGet(url).parseJson()
+		if (json.getString("result") == "ok") {
+			return Chapters(
+				data = json.optJSONArray("data")?.toJSONList().orEmpty(),
+				total = json.getInt("total"),
+			)
+		} else {
+			val error = json.optJSONArray("errors").mapJSON { jo ->
+				jo.getString("detail")
+			}.joinToString("\n")
+			throw ParseException(error, url)
+		}
+	}
+
+	private class Chapters(
+		val data: List<JSONObject>,
+		val total: Int,
+	) {
+
+		val size: Int
+			get() = data.size
+	}
 }

src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaInUaParser.kt

@@ -83,9 +83,7 @@ class MangaInUaParser(override val context: MangaLoaderContext) : PagedMangaPars
 		val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml()
 		val root = doc.body().requireElementById("dle-content")
 		val dateFormat = SimpleDateFormat("dd.MM.yyyy", Locale.US)
-		val chapterNodes = root.selectFirstOrThrow(".linkstocomics")
-			.select(".ltcitems")
-			.filterNot { x -> x.selectFirst(".foruserreadedbar") != null }
+		val chapterNodes = root.selectFirstOrThrow(".linkstocomics").select(".ltcitems")
 		var prevChapterName: String? = null
 		var i = 0
 		return manga.copy(

src/main/kotlin/org/koitharu/kotatsu/parsers/site/NetTruyenParser.kt

@@ -0,0 +1,221 @@
+package org.koitharu.kotatsu.parsers.site
+
+import androidx.collection.ArrayMap
+import androidx.collection.ArraySet
+import kotlinx.coroutines.sync.Mutex
+import kotlinx.coroutines.sync.withLock
+import org.koitharu.kotatsu.parsers.*
+import org.koitharu.kotatsu.parsers.config.ConfigKey
+import org.koitharu.kotatsu.parsers.exception.NotFoundException
+import org.koitharu.kotatsu.parsers.model.*
+import org.koitharu.kotatsu.parsers.util.*
+import java.text.SimpleDateFormat
+import java.util.*
+
+@MangaSourceParser("NETTRUYEN", "NetTruyen", "vi")
+class NetTruyenParser(override val context: MangaLoaderContext) :
+	PagedMangaParser(MangaSource.NETTRUYEN, pageSize = 36) {
+
+	override val configKeyDomain: ConfigKey.Domain
+		get() = ConfigKey.Domain("www.nettruyenme.com", null)
+
+	override val sortOrders: Set<SortOrder>
+		get() = EnumSet.of(SortOrder.UPDATED, SortOrder.POPULARITY, SortOrder.NEWEST, SortOrder.RATING)
+
+	private val mutex = Mutex()
+	private val dateFormat = SimpleDateFormat("dd/MM/yy", Locale.US)
+	private var tagCache: ArrayMap<String, MangaTag>? = null
+
+	override suspend fun getDetails(manga: Manga): Manga {
+		val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml()
+		val rating = doc.selectFirst("span[itemprop=ratingValue]")
+			?.ownText()
+			?.toFloatOrNull() ?: 0f
+
+		val chapterElements = doc.getElementById("nt_listchapter")?.select("ul > li") ?: doc.parseFailed()
+		val chapters = chapterElements.asReversed().mapChapters { index, element ->
+			val a = element.selectFirst("div.chapter > a") ?: return@mapChapters null
+			val relativeUrl = a.attrAsRelativeUrlOrNull("href") ?: return@mapChapters null
+			val timeText = element.selectFirst("div.col-xs-4.text-center.no-wrap.small")?.text()
+
+			MangaChapter(
+				id = generateUid(relativeUrl),
+				name = a.text(),
+				number = index + 1,
+				url = relativeUrl,
+				scanlator = null,
+				uploadDate = parseChapterTime(timeText),
+				branch = null,
+				source = source,
+			)
+		}
+
+		return manga.copy(
+			rating = rating / 5,
+			chapters = chapters,
+			description = doc.selectFirst("div.detail-content > p")?.html(),
+			isNsfw = doc.selectFirst("div.alert.alert-danger > strong:contains(Cảnh báo độ tuổi)") != null,
+		)
+	}
+
+	// 20 giây trước
+	// 52 phút trước
+	// 6 giờ trước
+	// 2 ngày trước
+	// 19:09 30/07
+	// 23/12/21
+	private fun parseChapterTime(timeText: String?): Long {
+		if (timeText.isNullOrEmpty()) {
+			return 0L
+		}
+
+		val timeWords = arrayOf("giây", "phút", "giờ", "ngày")
+		val calendar = Calendar.getInstance()
+		val timeArr = timeText.split(' ')
+		if (WordSet(*timeWords).anyWordIn(timeText)) {
+			val timeSuffix = timeArr.getOrNull(1)
+			val timeDiff = timeArr.getOrNull(0)?.toIntOrNull() ?: return 0L
+			when (timeSuffix) {
+				timeWords[0] -> calendar.add(Calendar.SECOND, -timeDiff)
+				timeWords[1] -> calendar.add(Calendar.MINUTE, -timeDiff)
+				timeWords[2] -> calendar.add(Calendar.HOUR, -timeDiff)
+				timeWords[3] -> calendar.add(Calendar.DATE, -timeDiff)
+				else -> return 0L
+			}
+		} else {
+			val relativeDate = timeArr.lastOrNull() ?: return 0L
+			val dateString = when (relativeDate.split('/').size) {
+				2 -> {
+					val currentYear = calendar.get(Calendar.YEAR).toString().takeLast(2)
+					"$relativeDate/$currentYear"
+				}
+				3 -> relativeDate
+				else -> return 0L
+			}
+
+			calendar.timeInMillis = dateFormat.tryParse(dateString)
+		}
+
+
+		return calendar.time.time
+	}
+
+	override suspend fun getListPage(
+		page: Int,
+		query: String?,
+		tags: Set<MangaTag>?,
+		sortOrder: SortOrder,
+	): List<Manga> {
+		val isSearching = !query.isNullOrEmpty()
+		val url = buildString {
+			append("https://")
+			append(getDomain())
+			if (isSearching) {
+				append("/tim-truyen?keyword=")
+				append(query!!.urlEncoded())
+				append("&page=")
+				append(page)
+			} else {
+				val tagQuery = tags.orEmpty().joinToString(",") { it.key }
+				append("/tim-truyen-nang-cao?genres=$tagQuery")
+				append("&notgenres=&gender=-1&status=-1&minchapter=1&sort=${getSortOrderKey(sortOrder)}")
+				append("&page=$page")
+			}
+		}
+
+		val response = if (isSearching) {
+			val result = runCatching { context.httpGet(url) }
+			val exception = result.exceptionOrNull()
+			if (exception is NotFoundException) {
+				return emptyList()
+			}
+
+			result.getOrThrow()
+		} else {
+			context.httpGet(url)
+		}
+
+		val itemsElements = response.parseHtml()
+			.select("div.ModuleContent > div.items")
+			.select("div.item")
+		return itemsElements.mapNotNull { item ->
+			val tooltipElement = item.selectFirst("div.box_tootip") ?: return@mapNotNull null
+			val absUrl = item.selectFirst("div.image > a")?.attrAsAbsoluteUrlOrNull("href") ?: return@mapNotNull null
+			val slug = absUrl.substringAfterLast('/')
+			val mangaState = when (tooltipElement.selectFirst("div.message_main > p:contains(Tình trạng)")?.ownText()) {
+				"Đang tiến hành" -> MangaState.ONGOING
+				"Hoàn thành" -> MangaState.FINISHED
+				else -> null
+			}
+
+			val tagMap = getOrCreateTagMap()
+			val tagsElement = tooltipElement.selectFirst("div.message_main > p:contains(Thể loại)")?.ownText().orEmpty()
+			val mangaTags = tagsElement.split(',').mapNotNullToSet { tagMap[it.trim()] }
+			Manga(
+				id = generateUid(slug),
+				title = tooltipElement.selectFirst("div.title")?.text().orEmpty(),
+				altTitle = null,
+				url = absUrl.toRelativeUrl(getDomain()),
+				publicUrl = absUrl,
+				rating = RATING_UNKNOWN,
+				isNsfw = false,
+				coverUrl = item.selectFirst("div.image a img")?.absUrl("data-original").orEmpty(),
+				largeCoverUrl = null,
+				tags = mangaTags,
+				state = mangaState,
+				author = tooltipElement.selectFirst("div.message_main > p:contains(Tác giả)")?.ownText(),
+				description = tooltipElement.selectFirst("div.box_text")?.text(),
+				chapters = null,
+				source = source,
+			)
+		}
+	}
+
+
+	override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
+		val pageElements = context.httpGet(chapter.url.toAbsoluteUrl(getDomain())).parseHtml()
+			.select("div.reading-detail.box_doc > div img")
+		return pageElements.map { element ->
+			val url = element.attrAsAbsoluteUrl("data-original")
+			MangaPage(
+				id = generateUid(url),
+				url = url,
+				referer = getDomain(),
+				preview = null,
+				source = source,
+			)
+		}
+	}
+
+	override suspend fun getTags(): Set<MangaTag> {
+		val map = getOrCreateTagMap()
+		val tagSet = ArraySet<MangaTag>(map.size)
+		for (entry in map) {
+			tagSet.add(entry.value)
+		}
+
+		return tagSet
+	}
+
+	private suspend fun getOrCreateTagMap(): ArrayMap<String, MangaTag> = mutex.withLock {
+		tagCache?.let { return@withLock it }
+		val doc = context.httpGet("/tim-truyen-nang-cao".toAbsoluteUrl(getDomain())).parseHtml()
+		val tagItems = doc.select("div.genre-item")
+		val result = ArrayMap<String, MangaTag>(tagItems.size)
+		for (item in tagItems) {
+			val title = item.text().trim()
+			val key = item.select("span[data-id]").attr("data-id")
+			result[title] = MangaTag(title = title, key = key, source = source)
+		}
+		tagCache = result
+		result
+	}
+
+	private fun getSortOrderKey(sortOrder: SortOrder) = when (sortOrder) {
+		SortOrder.UPDATED -> 0
+		SortOrder.POPULARITY -> 10
+		SortOrder.NEWEST -> 15
+		SortOrder.RATING -> 20
+		else -> throw IllegalArgumentException("Sort order ${sortOrder.name} not supported")
+	}
+}

src/main/kotlin/org/koitharu/kotatsu/parsers/site/TuMangaOnlineParser.kt

@@ -1,116 +0,0 @@
-package org.koitharu.kotatsu.parsers.site
-
-import org.koitharu.kotatsu.parsers.MangaLoaderContext
-import org.koitharu.kotatsu.parsers.MangaSourceParser
-import org.koitharu.kotatsu.parsers.PagedMangaParser
-import org.koitharu.kotatsu.parsers.config.ConfigKey
-import org.koitharu.kotatsu.parsers.model.*
-import org.koitharu.kotatsu.parsers.util.*
-import java.text.SimpleDateFormat
-import java.util.*
-
-@MangaSourceParser("TUMANGAONLINE", "TuMangaOnline", "es")
-class TuMangaOnlineParser(override val context: MangaLoaderContext) : PagedMangaParser(
-	source = MangaSource.TUMANGAONLINE,
-	pageSize = 24,
-) {
-
-	override val configKeyDomain = ConfigKey.Domain("lectortmo.com", null)
-
-	override val sortOrders: EnumSet<SortOrder> = EnumSet.of(
-		SortOrder.POPULARITY,
-		SortOrder.ALPHABETICAL,
-		SortOrder.RATING,
-		SortOrder.NEWEST,
-		SortOrder.UPDATED,
-	)
-
-	init {
-		paginator.firstPage = 1
-		searchPaginator.firstPage = 0
-	}
-
-	override suspend fun getListPage(
-		page: Int,
-		query: String?,
-		tags: Set<MangaTag>?,
-		sortOrder: SortOrder,
-	): List<Manga> {
-		val url = (
-				"/library" +
-						"?order_item=likes_count" +
-						"&order_dir=desc" +
-						"&filter_by=title" +
-						"&_pg=1" +
-						"&page=$page"
-				).toAbsoluteUrl(getDomain())
-		val doc = context.httpGet(url).parseHtml()
-		val items = doc.body().select("div.element")
-		return items.mapNotNull { item ->
-			val href = item.selectFirst("a")?.attrAsRelativeUrl("href")?.substringAfter(" ") ?: return@mapNotNull null
-			Manga(
-				id = generateUid(href),
-				title = item.selectFirst("h4.text-truncate")?.text() ?: return@mapNotNull null,
-				coverUrl = item.select("style").toString().substringAfter("('").substringBeforeLast("')"),
-				altTitle = null,
-				author = null,
-				rating = item.selectFirst("span.score")
-					?.text()
-					?.toFloatOrNull()
-					?.div(10F) ?: RATING_UNKNOWN,
-				url = href,
-				isNsfw = item.select("i").hasClass("fas fa-heartbeat fa-2x"),
-				tags = emptySet(),
-				state = null,
-				publicUrl = href.toAbsoluteUrl(doc.host ?: getDomain()),
-				source = source,
-			)
-		}
-	}
-
-	override suspend fun getDetails(manga: Manga): Manga {
-		val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml()
-		val contents = doc.body().selectFirstOrThrow("section.element-header-content")
-		return manga.copy(
-			description = contents.selectFirst("p.element-description")?.html(),
-			largeCoverUrl = contents.selectFirst(".book-thumbnail")?.attrAsAbsoluteUrlOrNull("src"),
-			state = parseStatus(contents.select("span.book-status").text().orEmpty()),
-			author = contents.selectFirst("h5.card-title")?.attr("title")?.substringAfter(", "),
-			chapters = doc.body().select("div.chapters > ul.list-group li.p-0.list-group-item").asReversed()
-				.mapChapters { i, li ->
-					val href = li.selectFirst("div.row > .text-right > a")
-						?.attrAsRelativeUrl("href") ?: li.parseFailed()
-					MangaChapter(
-						id = generateUid(href),
-						name = li.select("div.col-10.text-truncate").text().substringAfter(": ").trim(),
-						number = i + 1,
-						url = href,
-						scanlator = li.select("div.col-md-6.text-truncate").text(),
-						branch = null,
-						uploadDate = li.select("span.badge.badge-primary.p-2").first()?.text()
-							?.let { parseChapterDate(it) }
-							?: 0,
-						source = source,
-					)
-				},
-		)
-	}
-
-	override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
-		TODO("Not yet implemented")
-	}
-
-	override suspend fun getTags(): Set<MangaTag> {
-		TODO("Not yet implemented")
-	}
-
-	private fun parseStatus(status: String) = when {
-		status.contains("Publicándose") -> MangaState.ONGOING
-		status.contains("Finalizado") -> MangaState.FINISHED
-		else -> null
-	}
-
-	private fun parseChapterDate(date: String): Long =
-		SimpleDateFormat("yyyy-MM-dd", Locale.getDefault()).parse(date)?.time ?: 0
-
-}

src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt

@@ -111,11 +111,10 @@ internal abstract class GroupleParser(
 				altTitle = descDiv.selectFirst("h4")?.text(),
 				coverUrl = imgDiv.selectFirst("img.lazy")?.attr("data-original")?.replace("_p.", ".").orEmpty(),
 				rating = runCatching {
-					node.selectFirst("div.rating")
+					node.selectFirst(".compact-rate")
 						?.attr("title")
-						?.substringBefore(' ')
 						?.toFloatOrNull()
-						?.div(10f)
+						?.div(5f)
 				}.getOrNull() ?: RATING_UNKNOWN,
 				author = tileInfo?.selectFirst("a.person-link")?.text(),
 				isNsfw = false,

src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/IsekaiScanEu.kt

@@ -0,0 +1,45 @@
+package org.koitharu.kotatsu.parsers.site.madara
+
+import org.jsoup.nodes.Document
+import org.koitharu.kotatsu.parsers.MangaLoaderContext
+import org.koitharu.kotatsu.parsers.MangaSourceParser
+import org.koitharu.kotatsu.parsers.model.Manga
+import org.koitharu.kotatsu.parsers.model.MangaChapter
+import org.koitharu.kotatsu.parsers.model.MangaSource
+import org.koitharu.kotatsu.parsers.util.*
+import java.text.SimpleDateFormat
+import java.util.*
+
+@MangaSourceParser("ISEKAISCAN_EU", "IsekaiScan (eu)", "en")
+internal class IsekaiScanEuParser(context: MangaLoaderContext) :
+	MadaraParser(context, MangaSource.ISEKAISCAN_EU, "isekaiscan.eu") {
+
+	override suspend fun getChapters(manga: Manga, doc: Document): List<MangaChapter> {
+		val mangaId = doc.body().requireElementById("manga-chapters-holder").attr("data-id")
+		val ul = context.httpPost(
+			"https://${getDomain()}/wp-admin/admin-ajax.php",
+			mapOf(
+				"action" to "manga_get_chapters",
+				"manga" to mangaId,
+			),
+		).parseHtml().body().selectFirstOrThrow("ul")
+		val dateFormat = SimpleDateFormat(datePattern, Locale.US)
+		return ul.select("li").asReversed().mapChapters { i, li ->
+			val a = li.selectFirst("a")
+			val href = a?.attrAsRelativeUrlOrNull("href") ?: li.parseFailed("Link is missing")
+			MangaChapter(
+				id = generateUid(href),
+				name = a.ownText(),
+				number = i + 1,
+				url = href,
+				uploadDate = parseChapterDate(
+					dateFormat,
+					li.selectFirst("span.chapter-release-date i")?.text(),
+				),
+				source = source,
+				scanlator = null,
+				branch = null,
+			)
+		}
+	}
+}

src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt

@@ -1,5 +1,8 @@
 package org.koitharu.kotatsu.parsers.site.madara
 
+import kotlinx.coroutines.async
+import kotlinx.coroutines.coroutineScope
+import org.jsoup.nodes.Document
 import org.jsoup.nodes.Element
 import org.koitharu.kotatsu.parsers.MangaLoaderContext
 import org.koitharu.kotatsu.parsers.MangaSourceParser
@@ -116,9 +119,10 @@ internal abstract class MadaraParser(
 		}
 	}
 
-	override suspend fun getDetails(manga: Manga): Manga {
+	override suspend fun getDetails(manga: Manga): Manga = coroutineScope {
 		val fullUrl = manga.url.toAbsoluteUrl(getDomain())
 		val doc = context.httpGet(fullUrl).parseHtml()
+		val chaptersDeferred = async { getChapters(manga, doc) }
 		val root = doc.body().selectFirst("div.profile-manga")
 			?.selectFirst("div.summary_content")
 			?.selectFirst("div.post-content")
@@ -126,8 +130,7 @@ internal abstract class MadaraParser(
 		val root2 = doc.body().selectFirst("div.content-area")
 			?.selectFirst("div.c-page")
 			?: throw ParseException("Root2 not found", fullUrl)
-		val dateFormat = SimpleDateFormat(datePattern, Locale.US)
-		return manga.copy(
+		manga.copy(
 			tags = root.selectFirst("div.genres-content")?.select("a")
 				?.mapNotNullToSet { a ->
 					MangaTag(
@@ -141,26 +144,33 @@ internal abstract class MadaraParser(
 				?.select("p")
 				?.filterNot { it.ownText().startsWith("A brief description") }
 				?.joinToString { it.html() },
-			chapters = root2.select("li").asReversed().mapChapters { i, li ->
-				val a = li.selectFirst("a")
-				val href = a?.attrAsRelativeUrlOrNull("href") ?: li.parseFailed("Link is missing")
-				MangaChapter(
-					id = generateUid(href),
-					name = a.ownText(),
-					number = i + 1,
-					url = href,
-					uploadDate = parseChapterDate(
-						dateFormat,
-						li.selectFirst("span.chapter-release-date i")?.text(),
-					),
-					source = source,
-					scanlator = null,
-					branch = null,
-				)
-			},
+			chapters = chaptersDeferred.await(),
 		)
 	}
 
+	protected open suspend fun getChapters(manga: Manga, doc: Document): List<MangaChapter> {
+		val root2 = doc.body().selectFirstOrThrow("div.content-area")
+			.selectFirstOrThrow("div.c-page")
+		val dateFormat = SimpleDateFormat(datePattern, Locale.US)
+		return root2.select("li").asReversed().mapChapters { i, li ->
+			val a = li.selectFirst("a")
+			val href = a?.attrAsRelativeUrlOrNull("href") ?: li.parseFailed("Link is missing")
+			MangaChapter(
+				id = generateUid(href),
+				name = a.ownText(),
+				number = i + 1,
+				url = href,
+				uploadDate = parseChapterDate(
+					dateFormat,
+					li.selectFirst("span.chapter-release-date i")?.text(),
+				),
+				source = source,
+				scanlator = null,
+				branch = null,
+			)
+		}
+	}
+
 	override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
 		val fullUrl = chapter.url.toAbsoluteUrl(getDomain())
 		val doc = context.httpGet(fullUrl).parseHtml()
@@ -180,7 +190,7 @@ internal abstract class MadaraParser(
 		}
 	}
 
-	private fun parseChapterDate(dateFormat: DateFormat, date: String?): Long {
+	protected fun parseChapterDate(dateFormat: DateFormat, date: String?): Long {
 		date ?: return 0
 		return when {
 			date.endsWith(" ago", ignoreCase = true) -> {
@@ -355,9 +365,6 @@ internal abstract class MadaraParser(
 	@MangaSourceParser("MANGAROCK", "MangaRock", "en")
 	class MangaRock(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGAROCK, "mangarockteam.com")
 
-	@MangaSourceParser("ISEKAISCAN_EU", "IsekaiScan (eu)", "en")
-	class IsekaiScanEu(context: MangaLoaderContext) : MadaraParser(context, MangaSource.ISEKAISCAN_EU, "isekaiscan.eu")
-
 	@MangaSourceParser("FREEMANGA", "FreeManga", "en")
 	class FreeManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.FREEMANGA, "freemanga.me")
 

src/main/kotlin/org/koitharu/kotatsu/parsers/util/Assert.kt

@@ -0,0 +1,8 @@
+package org.koitharu.kotatsu.parsers.util
+
+fun <T : Any> T?.assertNotNull(name: String): T? {
+	assert(this != null) {
+		"Value $name is null"
+	}
+	return this
+}

src/main/kotlin/org/koitharu/kotatsu/parsers/util/Parse.kt

@@ -68,10 +68,13 @@ fun String.toAbsoluteUrl(domain: String): String = when {
 }
 
 fun DateFormat.tryParse(str: String?): Long = if (str.isNullOrEmpty()) {
+	assert(false) { "Date string is null or empty" }
 	0L
 } else {
 	runCatching {
 		parse(str)?.time ?: 0L
+	}.onFailure {
+		assert(false) { "Cannot parse date $str: ${it.message}" }
 	}.getOrDefault(0L)
 }
 

src/main/kotlin/org/koitharu/kotatsu/parsers/util/json/JsonExt.kt

@@ -136,4 +136,8 @@ fun JSONArray?.isNullOrEmpty(): Boolean {
 	}
 
 	return this == null || this.length() == 0
+}
+
+fun JSONArray.toJSONList(): List<JSONObject> {
+	return List(length()) { i -> getJSONObject(i) }
 }