Merge branch 'master' of github.com:KotatsuApp/kotatsu-parsers into feature/advanced_filter

src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/MangaDexParser.kt

@@ -22,15 +22,26 @@ private const val CHAPTERS_MAX_PAGE_SIZE = 500
 private const val CHAPTERS_PARALLELISM = 3
 private const val CHAPTERS_MAX_COUNT = 10_000 // strange api behavior, looks like a bug
 private const val LOCALE_FALLBACK = "en"
+private const val SERVER_DATA = "data"
+private const val SERVER_DATA_SAVER = "data-saver"
 
 @MangaSourceParser("MANGADEX", "MangaDex")
 internal class MangaDexParser(context: MangaLoaderContext) : MangaParser(context, MangaParserSource.MANGADEX) {
 
 	override val configKeyDomain = ConfigKey.Domain("mangadex.org")
 
+	private val preferredServerKey = ConfigKey.PreferredImageServer(
+		presetValues = mapOf(
+			SERVER_DATA to "Original quality",
+			SERVER_DATA_SAVER to "Compressed quality",
+		),
+		defaultValue = SERVER_DATA,
+	)
+
 	override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) {
 		super.onCreateConfig(keys)
 		keys.add(userAgentKey)
+		keys.add(preferredServerKey)
 	}
 
 	override val filterCapabilities: MangaListFilterCapabilities
@@ -231,12 +242,15 @@ internal class MangaDexParser(context: MangaLoaderContext) : MangaParser(context
 	}
 
 	override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
-		val domain = domain
-		val chapterJson = webClient.httpGet("https://api.$domain/at-home/server/${chapter.url}?forcePort443=false")
-			.parseJson()
-			.getJSONObject("chapter")
-		val pages = chapterJson.getJSONArray("data")
-		val prefix = "https://uploads.$domain/data/${chapterJson.getString("hash")}/"
+		val json = webClient.httpGet(
+			"https://api.$domain/at-home/server/${chapter.url}?forcePort443=false",
+		).parseJson()
+		val chapterJson = json.getJSONObject("chapter")
+		val server = config[preferredServerKey] ?: SERVER_DATA
+		val pages = chapterJson.getJSONArray(
+			if (server == SERVER_DATA_SAVER) "dataSaver" else "data",
+		)
+		val prefix = "${json.getString("baseUrl")}/$server/${chapterJson.getString("hash")}/"
 		return List(pages.length()) { i ->
 			val url = prefix + pages.getString(i)
 			MangaPage(

src/main/kotlin/org/koitharu/kotatsu/parsers/site/heancms/HeanCms.kt

@@ -64,7 +64,7 @@ internal abstract class HeanCms(
 		val url = buildString {
 			append("https://")
 			append(apiPath)
-			append("/query?query_string=")
+			append("/query?query_string=&series_type=Comic&perPage=$pageSize")
 			when {
 				!filter.query.isNullOrEmpty() -> {
 					append(filter.query.urlEncoded())
@@ -97,13 +97,10 @@ internal abstract class HeanCms(
 						SortOrder.ALPHABETICAL_DESC -> append("title&order=desc")
 						else -> append("latest&order=desc")
 					}
-
-					append("&series_type=Comic&perPage=20")
 					append("&tags_ids=")
 					append("[".urlEncoded())
 					append(filter.tags.joinToString(",") { it.key })
 					append("]".urlEncoded())
-
 				}
 			}
 			append("&page=")
@@ -195,10 +192,16 @@ internal abstract class HeanCms(
 
 	private suspend fun fetchAvailableTags(): Set<MangaTag> {
 		val doc = webClient.httpGet("https://$domain/comics").parseHtml()
-		val regex = Regex("\"tags\\\\.*?(\\[.+?])")
-		val tags = doc.select("script").firstNotNullOf { script ->
-			regex.find(script.html())?.groupValues?.getOrNull(1)
-		}.unescapeJson()
+		val regex = Regex("\"tags\\\\?\":\\s*\\[(.+?)]\\s*[},]")
+		val tags = doc.select("script").joinToString("") { it.html() }
+			.let { fullHtml ->
+				regex.find(fullHtml)?.groupValues?.getOrNull(1)
+			}
+			?.unescapeJson()
+			?.replace(Regex(""""]\)\s*self\.__next_f\.push\(\[\d+,""""), "")
+			?.let { "[$it]" }
+			?: return emptySet()
+
 		return JSONArray(tags).mapJSON {
 			MangaTag(
 				key = it.getInt("id").toString(),
@@ -207,4 +210,5 @@ internal abstract class HeanCms(
 			)
 		}.toSet()
 	}
+
 }

src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/vi/HentaiCube.kt

@@ -12,7 +12,7 @@ import org.koitharu.kotatsu.parsers.util.*
 
 @MangaSourceParser("HENTAICUBE", "HentaiCube", "vi", ContentType.HENTAI)
 internal class HentaiCube(context: MangaLoaderContext) :
-	MadaraParser(context, MangaParserSource.HENTAICUBE, "hentaicb.mom") {
+	MadaraParser(context, MangaParserSource.HENTAICUBE, "hentaicb.lol") {
 
 	override val datePattern = "dd/MM/yyyy"
 	override val tagPrefix = "the-loai/"

src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/vi/HentaiVnFit.kt

@@ -6,9 +6,9 @@ import org.koitharu.kotatsu.parsers.model.ContentType
 import org.koitharu.kotatsu.parsers.model.MangaParserSource
 import org.koitharu.kotatsu.parsers.site.madara.MadaraParser
 
-@MangaSourceParser("HENTAIVNCAFE", "HentaiVnCafe", "vi", ContentType.HENTAI)
+@MangaSourceParser("HENTAIVNFIT", "HentaiVnFit", "vi", ContentType.HENTAI)
 internal class HentaiVnCafe(context: MangaLoaderContext) :
-	MadaraParser(context, MangaParserSource.HENTAIVNCAFE, "hentaivn.cafe", 24) {
+	MadaraParser(context, MangaParserSource.HENTAIVNFIT, "hentaivn.fit", 24) {
 	override val listUrl = "truyen-hentai/"
 	override val tagPrefix = "the-loai/"
 	override val datePattern = "dd/MM/yyyy"

src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/vi/PinkTeaComic.kt

@@ -4,7 +4,9 @@ import org.koitharu.kotatsu.parsers.MangaLoaderContext
 import org.koitharu.kotatsu.parsers.MangaSourceParser
 import org.koitharu.kotatsu.parsers.model.MangaParserSource
 import org.koitharu.kotatsu.parsers.site.madara.MadaraParser
+import org.koitharu.kotatsu.parsers.Broken
 
+@Broken
 @MangaSourceParser("PINKTEACOMIC", "PinkTeaComic", "vi")
 internal class PinkTeaComic(context: MangaLoaderContext) :
 	MadaraParser(context, MangaParserSource.PINKTEACOMIC, "pinkteacomics.com") {

src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/vi/TruyenVn.kt

@@ -8,7 +8,7 @@ import org.koitharu.kotatsu.parsers.site.madara.MadaraParser
 
 @MangaSourceParser("TRUYENVN", "TruyenVn", "vi", ContentType.HENTAI)
 internal class TruyenVn(context: MangaLoaderContext) :
-	MadaraParser(context, MangaParserSource.TRUYENVN, "truyenvn.mobi", 20) {
+	MadaraParser(context, MangaParserSource.TRUYENVN, "truyenvn.fit", 20) {
 	override val listUrl = "truyen-tranh/"
 	override val tagPrefix = "the-loai/"
 	override val datePattern = "dd/MM/yyyy"

src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/HentaiVNParser.kt

@@ -21,7 +21,7 @@ private const val SEARCH_PAGE_SIZE = 10
 @MangaSourceParser("HENTAIVN", "HentaiVN", "vi", type = ContentType.HENTAI)
 internal class HentaiVNParser(context: MangaLoaderContext) : MangaParser(context, MangaParserSource.HENTAIVN) {
 
-	override val configKeyDomain: ConfigKey.Domain = ConfigKey.Domain("hentaiayame.com")
+	override val configKeyDomain: ConfigKey.Domain = ConfigKey.Domain("hentaiayame.net")
 
 	override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) {
 		super.onCreateConfig(keys)

src/main/kotlin/org/koitharu/kotatsu/parsers/site/wpcomics/vi/NetTruyen.kt

@@ -10,6 +10,6 @@ import org.koitharu.kotatsu.parsers.site.wpcomics.WpComicsParser
 internal class NetTruyen(context: MangaLoaderContext) :
 	WpComicsParser(context, MangaParserSource.NETTRUYEN, "www.nettruyenupp.com", 44) {
 	override val configKeyDomain: ConfigKey.Domain = ConfigKey.Domain(
-		"www.nettruyenupp.com", "nettruyenaa.com", "nettruyenx.com",
+		"www.nettruyenupp.com", "nettruyenww.com", "nettruyenx.com",
 	)
 }

src/main/kotlin/org/koitharu/kotatsu/parsers/site/wpcomics/vi/NetTruyenLL.kt

@@ -8,8 +8,10 @@ import org.koitharu.kotatsu.parsers.exception.NotFoundException
 import org.koitharu.kotatsu.parsers.model.*
 import org.koitharu.kotatsu.parsers.site.wpcomics.WpComicsParser
 import org.koitharu.kotatsu.parsers.util.*
-import java.util.*
+import java.util.EnumSet
+import org.koitharu.kotatsu.parsers.Broken
 
+@Broken
 @MangaSourceParser("NETTRUYENLL", "NetTruyenLL", "vi")
 internal class NetTruyenLL(context: MangaLoaderContext) :
 	WpComicsParser(context, MangaParserSource.NETTRUYENLL, "nettruyenll.com", 20) {

src/main/kotlin/org/koitharu/kotatsu/parsers/site/wpcomics/vi/NetTruyenSSR.kt

@@ -8,8 +8,10 @@ import org.koitharu.kotatsu.parsers.exception.NotFoundException
 import org.koitharu.kotatsu.parsers.model.*
 import org.koitharu.kotatsu.parsers.site.wpcomics.WpComicsParser
 import org.koitharu.kotatsu.parsers.util.*
-import java.util.*
+import org.koitharu.kotatsu.parsers.Broken
+import java.util.EnumSet
 
+@Broken
 @MangaSourceParser("NETTRUYENSSR", "NetTruyenSSR", "vi")
 internal class NetTruyenSSR(context: MangaLoaderContext) :
 	WpComicsParser(context, MangaParserSource.NETTRUYENSSR, "nettruyenssr.com", 20) {