[MadaraParser] auto fix url manga on

fix query ( withoutAjax
[MangaPark] fix close #1152
close #1145
[AdultWebtoon] fix close #1139
[ComicExtra] change type close #1130
[liliana] fix and add ManhuaPlusorg, Raw1001, MangaSect, MangaKoma01, ManhuaGold

.github/summary.yaml

@@ -1 +1 @@
-total: 1122
+total: 1127

src/main/kotlin/org/koitharu/kotatsu/parsers/model/Manga.kt

@@ -82,6 +82,7 @@ public class Manga(
 
 	@InternalParsersApi
 	public fun copy(
+		url: String = this.url,
 		title: String = this.title,
 		altTitle: String? = this.altTitle,
 		publicUrl: String = this.publicUrl,

src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/MangaPark.kt

@@ -236,18 +236,24 @@ internal class MangaPark(context: MangaLoaderContext) :
 		val number = Regex("""(\d+)""").find(date)?.value?.toIntOrNull() ?: return 0
 		val cal = Calendar.getInstance()
 		return when {
-			WordSet("second").anyWordIn(date) -> cal.apply { add(Calendar.SECOND, -number) }.timeInMillis
+			WordSet("second")
-			WordSet("minute", "minutes", "mins", "min").anyWordIn(date) -> cal.apply {
+				.anyWordIn(date) -> cal.apply { add(Calendar.SECOND, -number) }.timeInMillis
-				add(
+
-					Calendar.MINUTE,
+			WordSet("minute", "minutes", "mins", "min")
-					-number,
+				.anyWordIn(date) -> cal.apply { add(Calendar.MINUTE, -number) }.timeInMillis
-				)
+
-			}.timeInMillis
+			WordSet("hour", "hours")
+				.anyWordIn(date) -> cal.apply { add(Calendar.HOUR, -number) }.timeInMillis
+
+			WordSet("day", "days")
+				.anyWordIn(date) -> cal.apply { add(Calendar.DAY_OF_MONTH, -number) }.timeInMillis
+
+			WordSet("month", "months")
+				.anyWordIn(date) -> cal.apply { add(Calendar.MONTH, -number) }.timeInMillis
+
+			WordSet("year")
+				.anyWordIn(date) -> cal.apply { add(Calendar.YEAR, -number) }.timeInMillis
 
-			WordSet("hour", "hours").anyWordIn(date) -> cal.apply { add(Calendar.HOUR, -number) }.timeInMillis
-			WordSet("day", "days").anyWordIn(date) -> cal.apply { add(Calendar.DAY_OF_MONTH, -number) }.timeInMillis
-			WordSet("month", "months").anyWordIn(date) -> cal.apply { add(Calendar.MONTH, -number) }.timeInMillis
-			WordSet("year").anyWordIn(date) -> cal.apply { add(Calendar.YEAR, -number) }.timeInMillis
 			else -> 0
 		}
 	}
@@ -255,18 +261,23 @@ internal class MangaPark(context: MangaLoaderContext) :
 
 	override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
 		val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml()
-		val script = if (doc.selectFirst("script:containsData(comic-)") != null) {
+		val id = chapter.url.removeSuffix('/').substringAfterLast('/').substringBefore('-')
-			doc.selectFirstOrThrow("script:containsData(comic-)").data()
+		val s = doc.selectFirstOrThrow("script:containsData($id)").data()
-				.substringAfterLast("\"comic-")
+
+		val script = if (s.contains("\"comic-")) {
+			s.substringAfterLast("\"comic-")
 		} else {
-			doc.selectFirstOrThrow("script:containsData(manga-)").data()
+			s.substringAfterLast("\"manga-")
-				.substringAfterLast("\"manga-")
 		}
+
 		return Regex("\"(https?:.+?)\"")
 			.findAll(script)
-			.mapNotNullTo(ArrayList()) {
+			.mapIndexedNotNullTo(ArrayList()) { i, it ->
-				val url = it.groupValues.getOrNull(1) ?: return@mapNotNullTo null
+				val url = it.groupValues.getOrNull(1) ?: return@mapIndexedNotNullTo null
-				if (url.contains("/comic/") || url.contains("/manga/") || url.contains("/image/mpup/")) {
+				if (url.contains(".jpg") || url.contains(".jpeg") || url.contains(".jfif") || url.contains(".pjpeg") ||
+					url.contains(".pjp") || url.contains(".png") || url.contains(".webp") || url.contains(".avif") ||
+					url.contains(".gif")
+				) {
 					MangaPage(
 						id = generateUid(url),
 						url = url,
@@ -274,7 +285,7 @@ internal class MangaPark(context: MangaLoaderContext) :
 						source = source,
 					)
 				} else {
-					return@mapNotNullTo null
+					return@mapIndexedNotNullTo null
 				}
 			}
 	}

src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/ComicExtra.kt

@@ -11,7 +11,7 @@ import org.koitharu.kotatsu.parsers.util.*
 import java.text.SimpleDateFormat
 import java.util.*
 
-@MangaSourceParser("COMICEXTRA", "ComicExtra", "en")
+@MangaSourceParser("COMICEXTRA", "ComicExtra", "en", ContentType.COMICS)
 internal class ComicExtra(context: MangaLoaderContext) : PagedMangaParser(context, MangaParserSource.COMICEXTRA, 25) {
 
 	override val configKeyDomain = ConfigKey.Domain("comixextra.com")

src/main/kotlin/org/koitharu/kotatsu/parsers/site/liliana/LilianaParser.kt

@@ -1,5 +1,6 @@
 package org.koitharu.kotatsu.parsers.site.liliana
 
+import androidx.collection.scatterSetOf
 import kotlinx.coroutines.coroutineScope
 import org.jsoup.Jsoup
 import org.jsoup.nodes.Element
@@ -27,16 +28,26 @@ internal abstract class LilianaParser(
 	override val availableSortOrders: Set<SortOrder> = EnumSet.of(
 		SortOrder.UPDATED,
 		SortOrder.POPULARITY,
+		SortOrder.POPULARITY_MONTH,
+		SortOrder.POPULARITY_WEEK,
+		SortOrder.POPULARITY_TODAY,
 		SortOrder.ALPHABETICAL,
+		SortOrder.ALPHABETICAL_DESC,
 		SortOrder.NEWEST,
-		SortOrder.RATING_ASC,
+		SortOrder.NEWEST_ASC,
+		SortOrder.RATING,
+	)
+
+	override suspend fun getFilterOptions(): MangaListFilterOptions = MangaListFilterOptions(
+		availableTags = getAvailableTags(),
+		availableStates = setOf(MangaState.ONGOING, MangaState.FINISHED, MangaState.PAUSED, MangaState.ABANDONED),
 	)
 
 	override val filterCapabilities: MangaListFilterCapabilities
 		get() = MangaListFilterCapabilities(
 			isMultipleTagsSupported = true,
+			isTagsExclusionSupported = true,
 			isSearchSupported = true,
-			isSearchWithFiltersSupported = true,
 		)
 
 	override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> {
@@ -45,32 +56,36 @@ internal abstract class LilianaParser(
 			append(domain)
 			when {
 				!filter.query.isNullOrEmpty() -> {
-					append("/search")
+					append("/search/")
-					append("?keyword=")
+					append(page)
+					append("/?keyword=")
 					append(filter.query.urlEncoded())
 				}
 
 				else -> {
-					append("/filter")
+					append("/filter/")
-				}
-			}
-			append("/")
 					append(page)
-			append("/")
 
+					append("/?sort=")
 					when (order) {
-				SortOrder.UPDATED -> append("?sort=latest-updated")
+						SortOrder.UPDATED -> append("latest-updated")
-				SortOrder.POPULARITY -> append("?sort=views")
+						SortOrder.POPULARITY -> append("views")
-				SortOrder.ALPHABETICAL -> append("?sort=az")
+						SortOrder.POPULARITY_MONTH -> append("views_month")
-				SortOrder.NEWEST -> append("?sort=new")
+						SortOrder.POPULARITY_WEEK -> append("views_week")
-				SortOrder.RATING_ASC -> append("?sort=score")
+						SortOrder.POPULARITY_TODAY -> append("views_day")
-				else -> append("?sort=default")
+						SortOrder.ALPHABETICAL -> append("az")
+						SortOrder.ALPHABETICAL_DESC -> append("za")
+						SortOrder.NEWEST -> append("new")
+						SortOrder.NEWEST_ASC -> append("old")
+						SortOrder.RATING -> append("score")
+						else -> append("latest-updated")
 					}
 
-			filter.tags.forEach { tag ->
 					append("&genres=")
-				append(tag.key)
+					filter.tags.joinTo(this, ",") { it.key }
-			}
+
+					append("&notGenres=")
+					filter.tagsExclude.joinTo(this, ",") { it.key }
 
 					if (filter.states.isNotEmpty()) {
 						append("&status=")
@@ -85,6 +100,8 @@ internal abstract class LilianaParser(
 						)
 					}
 				}
+			}
+		}
 
 		val doc = webClient.httpGet(url).parseHtml()
 		return doc.select("div#main div.grid > div").map { parseSearchManga(it) }
@@ -104,10 +121,30 @@ internal abstract class LilianaParser(
 			author = null,
 			state = null,
 			source = source,
-			isNsfw = false,
+			isNsfw = isNsfwSource,
 		)
 	}
 
+	@JvmField
+	protected val ongoing = scatterSetOf(
+		"on-going", "đang tiến hành", "進行中",
+	)
+
+	@JvmField
+	protected val finished = scatterSetOf(
+		"completed", "hoàn thành", "完了",
+	)
+
+	@JvmField
+	protected val abandoned = scatterSetOf(
+		"canceled", "đã huỷ bỏ", "キャンセル",
+	)
+
+	@JvmField
+	protected val paused = scatterSetOf(
+		"on-hold", "tạm dừng", "一時停止",
+	)
+
 	override suspend fun getDetails(manga: Manga): Manga {
 		val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml()
 		return manga.copy(
@@ -123,26 +160,27 @@ internal abstract class LilianaParser(
 			author = doc.selectFirst("div.y6x11p i.fas.fa-user + span.dt")?.text()?.takeUnless {
 				it.equals("updating", true)
 			},
-			state = when (doc.selectFirst("div.y6x11p i.fas.fa-rss + span.dt")?.text()?.lowercase()) {
+			state = when (doc.selectFirst("div.y6x11p i.fas.fa-rss + span.dt")?.text()?.lowercase().orEmpty()) {
-				"on-going", "đang tiến hành", "進行中" -> MangaState.ONGOING
+				in ongoing -> MangaState.ONGOING
-				"completed", "hoàn thành", "完了" -> MangaState.FINISHED
+				in finished -> MangaState.FINISHED
-				"on-hold", "tạm dừng", "一時停止" -> MangaState.PAUSED
+				in paused -> MangaState.PAUSED
-				"canceled", "đã huỷ bỏ", "キャンセル" -> MangaState.ABANDONED
+				in abandoned -> MangaState.ABANDONED
 				else -> null
 			},
 			chapters = doc.select("ul > li.chapter").mapChapters(reversed = true) { i, element ->
 				val href = element.selectFirstOrThrow("a").attrAsRelativeUrl("href")
 				MangaChapter(
 					id = generateUid(href),
-					name = element.selectFirst("a")?.text().orEmpty(),
+					name = element.selectFirst("a")?.text() ?: "Chapter : ${i + 1f}",
 					number = i + 1f,
+					volume = 0,
 					url = href,
 					scanlator = null,
 					uploadDate = element.selectFirst("time[datetime]")?.attr("datetime")?.toLongOrNull()?.times(1000)
 						?: 0L,
 					branch = null,
 					source = source,
-					volume = 0,
+
 					)
 			},
 		)
@@ -170,13 +208,10 @@ internal abstract class LilianaParser(
 			throw Exception(responseJson.optString("msg"))
 		}
 
-		val pageListHtml = responseJson.getString("html")
+		val pageListDoc = responseJson.getString("html").let(Jsoup::parse)
-		val pageListDoc = Jsoup.parse(pageListHtml)
-
-		return pageListDoc.select("div.iv-card").mapIndexed { index, div ->
-			val img = div.selectFirst("img")
-			val url = img?.attr("data-src") ?: img?.attr("src") ?: throw Exception("Failed to get image url")
 
+		return pageListDoc.select("div").map {
+			val url = it.selectFirstOrThrow("img").attr("src")
 			MangaPage(
 				id = generateUid(url),
 				url = url,
@@ -196,9 +231,4 @@ internal abstract class LilianaParser(
 			)
 		}
 	}
-
-	override suspend fun getFilterOptions(): MangaListFilterOptions = MangaListFilterOptions(
-		availableTags = getAvailableTags(),
-		availableStates = setOf(MangaState.ONGOING, MangaState.FINISHED, MangaState.PAUSED, MangaState.ABANDONED),
-	)
 }

src/main/kotlin/org/koitharu/kotatsu/parsers/site/liliana/en/MangaSect.kt

@@ -0,0 +1,10 @@
+package org.koitharu.kotatsu.parsers.site.liliana.en
+
+import org.koitharu.kotatsu.parsers.MangaLoaderContext
+import org.koitharu.kotatsu.parsers.MangaSourceParser
+import org.koitharu.kotatsu.parsers.model.MangaParserSource
+import org.koitharu.kotatsu.parsers.site.liliana.LilianaParser
+
+@MangaSourceParser("MANGASECT", "MangaSect", "en")
+internal class MangaSect(context: MangaLoaderContext) :
+	LilianaParser(context, MangaParserSource.MANGASECT, "mangasect.net")

src/main/kotlin/org/koitharu/kotatsu/parsers/site/liliana/en/ManhuaGold.kt

@@ -0,0 +1,10 @@
+package org.koitharu.kotatsu.parsers.site.liliana.en
+
+import org.koitharu.kotatsu.parsers.MangaLoaderContext
+import org.koitharu.kotatsu.parsers.MangaSourceParser
+import org.koitharu.kotatsu.parsers.model.MangaParserSource
+import org.koitharu.kotatsu.parsers.site.liliana.LilianaParser
+
+@MangaSourceParser("MANHUAGOLD", "ManhuaGold", "en")
+internal class ManhuaGold(context: MangaLoaderContext) :
+	LilianaParser(context, MangaParserSource.MANHUAGOLD, "manhuagold.top")

src/main/kotlin/org/koitharu/kotatsu/parsers/site/liliana/en/ManhuaPlusOrg.kt

@@ -0,0 +1,10 @@
+package org.koitharu.kotatsu.parsers.site.liliana.en
+
+import org.koitharu.kotatsu.parsers.MangaLoaderContext
+import org.koitharu.kotatsu.parsers.MangaSourceParser
+import org.koitharu.kotatsu.parsers.model.MangaParserSource
+import org.koitharu.kotatsu.parsers.site.liliana.LilianaParser
+
+@MangaSourceParser("MANHUAPLUSORG", "ManhuaPlus.org", "en")
+internal class ManhuaPlusOrg(context: MangaLoaderContext) :
+	LilianaParser(context, MangaParserSource.MANHUAPLUSORG, "manhuaplus.org")

src/main/kotlin/org/koitharu/kotatsu/parsers/site/liliana/ja/MangaKoma01.kt

@@ -0,0 +1,10 @@
+package org.koitharu.kotatsu.parsers.site.liliana.ja
+
+import org.koitharu.kotatsu.parsers.MangaLoaderContext
+import org.koitharu.kotatsu.parsers.MangaSourceParser
+import org.koitharu.kotatsu.parsers.model.MangaParserSource
+import org.koitharu.kotatsu.parsers.site.liliana.LilianaParser
+
+@MangaSourceParser("MANGAKOMA01", "MangaKoma01", "ja")
+internal class MangaKoma01(context: MangaLoaderContext) :
+	LilianaParser(context, MangaParserSource.MANGAKOMA01, "mangakoma01.net")

src/main/kotlin/org/koitharu/kotatsu/parsers/site/liliana/ja/Raw1001.kt

@@ -0,0 +1,10 @@
+package org.koitharu.kotatsu.parsers.site.liliana.ja
+
+import org.koitharu.kotatsu.parsers.MangaLoaderContext
+import org.koitharu.kotatsu.parsers.MangaSourceParser
+import org.koitharu.kotatsu.parsers.model.MangaParserSource
+import org.koitharu.kotatsu.parsers.site.liliana.LilianaParser
+
+@MangaSourceParser("RAW1001", "Raw1001", "ja")
+internal class Raw1001(context: MangaLoaderContext) :
+	LilianaParser(context, MangaParserSource.RAW1001, "raw1001.net")

src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt

@@ -233,7 +233,9 @@ internal abstract class MadaraParser(
 				}
 				append("/?s=")
 
-				append(filter.query?.urlEncoded())
+				filter.query?.let {
+					append(filter.query.urlEncoded())
+				}
 
 				append("&post_type=wp-manga")
 
@@ -536,10 +538,10 @@ internal abstract class MadaraParser(
 		val fullUrl = manga.url.toAbsoluteUrl(domain)
 		val doc = webClient.httpGet(fullUrl).parseHtml()
 
+		val href = doc.selectFirst("head meta[property='og:url']")?.attr("content")?.toRelativeUrl(domain) ?: manga.url
 		val testCheckAsync = doc.select(selectTestAsync)
-
 		val chaptersDeferred = if (testCheckAsync.isNullOrEmpty()) {
-			async { loadChapters(manga.url, doc) }
+			async { loadChapters(href, doc) }
 		} else {
 			async { getChapters(manga, doc) }
 		}
@@ -561,6 +563,8 @@ internal abstract class MadaraParser(
 		val alt = doc.body().select(selectAlt).firstOrNull()?.tableValue()?.text()?.trim()
 
 		manga.copy(
+			url = href,
+			publicUrl = href.toAbsoluteUrl(domain),
 			tags = doc.body().select(selectGenre).mapNotNullToSet { a ->
 				MangaTag(
 					key = a.attr("href").removeSuffix("/").substringAfterLast('/'),

src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/en/AdultWebtoon.kt

@@ -5,13 +5,17 @@ import okhttp3.Headers
 import okhttp3.Request
 import okhttp3.RequestBody
 import okhttp3.RequestBody.Companion.toRequestBody
+import org.json.JSONObject
 import org.jsoup.nodes.Document
 import org.koitharu.kotatsu.parsers.MangaLoaderContext
 import org.koitharu.kotatsu.parsers.MangaSourceParser
+import org.koitharu.kotatsu.parsers.exception.AuthRequiredException
+import org.koitharu.kotatsu.parsers.exception.ParseException
 import org.koitharu.kotatsu.parsers.model.*
 import org.koitharu.kotatsu.parsers.site.madara.MadaraParser
 import org.koitharu.kotatsu.parsers.util.*
 import java.text.SimpleDateFormat
+import java.util.Base64
 
 @MangaSourceParser("ADULT_WEBTOON", "AdultWebtoon", "en", ContentType.HENTAI)
 internal class AdultWebtoon(context: MangaLoaderContext) :
@@ -150,6 +154,68 @@ internal class AdultWebtoon(context: MangaLoaderContext) :
 		}
 	}
 
+	override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
+		val fullUrl = chapter.url.toAbsoluteUrl(domain)
+		val doc = webClient.httpGet(fullUrl).parseHtml()
+		val chapterProtector = doc.getElementById("chapter-protector-data")
+		if (chapterProtector == null) {
+			throw if (doc.selectFirst(selectRequiredLogin) != null) {
+				AuthRequiredException(source)
+			} else {
+				val root = doc.body().selectFirst(selectBodyPage) ?: throw ParseException(
+					"No image found, try to log in",
+					fullUrl,
+				)
+				return root.select(selectPage).map { div ->
+					val img = div.selectFirstOrThrow("img")
+					val url = img.src()?.toRelativeUrl(domain) ?: div.parseFailed("Image src not found")
+					MangaPage(
+						id = generateUid(url),
+						url = url.replace("http:", "https:"),
+						preview = null,
+						source = source,
+					)
+				}
+			}
+		} else {
+
+			val chapterProtectorHtml = chapterProtector.attr("src")
+				.takeIf { it.startsWith("data:text/javascript;base64,") }
+				?.substringAfter("data:text/javascript;base64,")
+				?.let {
+					Base64.getDecoder().decode(it).decodeToString()
+				}
+				?: chapterProtector.html()
+
+			val password = chapterProtectorHtml.substringAfter("wpmangaprotectornonce='").substringBefore("';")
+			val chapterData = JSONObject(
+				chapterProtectorHtml.substringAfter("chapter_data='").substringBefore("';").replace("\\/", "/"),
+			)
+			val unsaltedCiphertext = context.decodeBase64(chapterData.getString("ct"))
+			val salt = chapterData.getString("s").toString().decodeHex()
+			val ciphertext = "Salted__".toByteArray(Charsets.UTF_8) + salt + unsaltedCiphertext
+
+			val rawImgArray = CryptoAES(context).decrypt(context.encodeBase64(ciphertext), password)
+			val imgArrayString = rawImgArray.filterNot { c -> c == '[' || c == ']' || c == '\\' || c == '"' }
+
+			return imgArrayString.split(",").map { url ->
+				MangaPage(
+					id = generateUid(url),
+					url = url.replace("http:", "https:"),
+					preview = null,
+					source = source,
+				)
+			}
+
+		}
+	}
+
+	private fun String.decodeHex(): ByteArray {
+		check(length % 2 == 0) { "Must have an even length" }
+
+		return chunked(2).map { it.toInt(16).toByte() }.toByteArray()
+	}
+
 	private suspend fun makeRequest(url: String, payload: RequestBody, headers: Headers): Document {
 		var retryCount = 0
 		val backoffDelay = 2000L // Initial delay (milliseconds)

src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/BlogTruyenParser.kt

@@ -1,6 +1,5 @@
 package org.koitharu.kotatsu.parsers.site.vi
 
-import androidx.collection.ArrayMap
 import org.json.JSONArray
 import org.jsoup.nodes.Document
 import org.jsoup.nodes.Element
@@ -53,7 +52,8 @@ internal class BlogTruyenParser(context: MangaLoaderContext) :
 
 			filter.tags.isNotEmpty() -> {
 				filter.tags.oneOrThrowIfMany()?.let { tag ->
-					val categoryAjax = "https://${domain}/ajax/Category/AjaxLoadMangaByCategory?id=${tag.key}&orderBy=5&p=$page"
+					val categoryAjax =
+						"https://${domain}/ajax/Category/AjaxLoadMangaByCategory?id=${tag.key}&orderBy=5&p=$page"
 					val listContent = webClient.httpGet(categoryAjax).parseHtml().selectFirst("div.list")
 					parseMangaList(listContent) ?: emptyList()
 				} ?: emptyList()
@@ -248,6 +248,6 @@ internal class BlogTruyenParser(context: MangaLoaderContext) :
 		MangaTag("Soft Yuri", "36", source),
 		MangaTag("Live action", "16", source),
 		MangaTag("Tu chân - tu tiên", "66", source),
-		MangaTag("Ngôn tình", "65", source)
+		MangaTag("Ngôn tình", "65", source),
 	)
 }

src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/BlogTruyenVN.kt

@@ -17,7 +17,7 @@ import org.koitharu.kotatsu.parsers.Broken
 
 @Broken
 @MangaSourceParser("BLOGTRUYENVN", "BlogTruyenVN", "vi")
-internal class BlogTruyenVNParser(context: MangaLoaderContext) :
+internal class BlogTruyenVN(context: MangaLoaderContext) :
 	PagedMangaParser(context, MangaParserSource.BLOGTRUYENVN, pageSize = 20) {
 
 	override val configKeyDomain: ConfigKey.Domain