fix and add updated on template Mmrcms

2023-07-27 16:42:28 +02:00
parent f6dc4f9a00
commit 7e2a00520b
6 changed files with 110 additions and 91 deletions
--- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/manga18/Manga18Parser.kt
+++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/manga18/Manga18Parser.kt
@@ -27,8 +27,8 @@ internal abstract class Manga18Parser(
 		SortOrder.ALPHABETICAL,
 	)

-	protected open val listeurl = "list-manga"
-	protected open val tagUrl = "manga-list"
+	protected open val listeurl = "list-manga/"
+	protected open val tagUrl = "manga-list/"
 	protected open val isNsfwSource = false
 	protected open val datePattern = "dd-MM-yyyy"

@@ -40,12 +40,12 @@ internal abstract class Manga18Parser(


 	@JvmField
-	protected val ongoing: Set<String> = hashSetOf(
+	protected val ongoing: Set<String> = setOf(
 		"On Going",
 	)

 	@JvmField
-	protected val finished: Set<String> = hashSetOf(
+	protected val finished: Set<String> = setOf(
 		"Completed",
 	)

@@ -60,7 +60,7 @@ internal abstract class Manga18Parser(
 			append(domain)
 			when {
 				!query.isNullOrEmpty() -> {
-					append("/$listeurl/")
+					append("/$listeurl")
 					append(page.toString())
 					append("?search=")
 					append(query.urlEncoded())
@@ -68,7 +68,7 @@ internal abstract class Manga18Parser(
 				}

 				!tags.isNullOrEmpty() -> {
-					append("/$tagUrl/")
+					append("/$tagUrl")
 					for (tag in tags) {
 						append(tag.key)
 					}
@@ -78,7 +78,7 @@ internal abstract class Manga18Parser(
 				}

 				else -> {
-					append("/$listeurl/")
+					append("/$listeurl")
 					append(page.toString())
 					append("?")
 				}
@@ -94,7 +94,7 @@ internal abstract class Manga18Parser(
 		val doc = webClient.httpGet(url).parseHtml()

 		return doc.select("div.story_item").map { div ->
-			val href = div.selectFirst("a")?.attrAsRelativeUrlOrNull("href") ?: div.parseFailed("Link not found")
+			val href = div.selectFirstOrThrow("a").attrAsRelativeUrl("href")
 			Manga(
 				id = generateUid(href),
 				url = href,
@@ -116,7 +116,7 @@ internal abstract class Manga18Parser(
 		val doc = webClient.httpGet("https://$domain/$listeurl/").parseHtml()
 		return doc.select("div.grid_cate li").mapNotNullToSet { li ->
 			val a = li.selectFirst("a") ?: return@mapNotNullToSet null
-			val href = a.attr("href").substringAfterLast("/")
+			val href = a.attr("href").removeSuffix('/').substringAfterLast('/')
 			MangaTag(
 				key = href,
 				title = a.text(),
@@ -139,15 +139,7 @@ internal abstract class Manga18Parser(

 		val chaptersDeferred = async { getChapters(manga, doc) }

-		val desc = doc.select(selectdesc).let {
-			if (it.select("p").text().isNotEmpty()) {
-				it.select("p").joinToString(separator = "\n\n") { p ->
-					p.text().replace("<br>", "\n")
-				}
-			} else {
-				it.text()
-			}
-		}
+		val desc = doc.selectFirstOrThrow(selectdesc).html()

 		val stateDiv = body.selectFirst(selectState)

@@ -164,7 +156,7 @@ internal abstract class Manga18Parser(
 		manga.copy(
 			tags = doc.body().select(selectTag).mapNotNullToSet { a ->
 				MangaTag(
-					key = a.attr("href").removeSuffix("/").substringAfterLast('/'),
+					key = a.attr("href").removeSuffix('/').substringAfterLast('/'),
 					title = a.text().toTitleCase(),
 					source = source,
 				)
@@ -180,8 +172,8 @@ internal abstract class Manga18Parser(
 	protected open suspend fun getChapters(manga: Manga, doc: Document): List<MangaChapter> {
 		val dateFormat = SimpleDateFormat(datePattern, sourceLocale)
 		return doc.body().select(selectchapter).mapChapters(reversed = true) { i, li ->
-			val a = li.selectFirst("a")
-			val href = a?.attrAsRelativeUrlOrNull("href") ?: li.parseFailed("Link is missing")
+			val a = li.selectFirstOrThrow("a")
+			val href = a.attrAsRelativeUrl("href")
 			val dateText = li.selectFirst(selectdate)?.text()
 			MangaChapter(
 				id = generateUid(href),
--- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/manga18/en/Hentai3zCc.kt
+++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/manga18/en/Hentai3zCc.kt
@@ -9,11 +9,10 @@ import org.koitharu.kotatsu.parsers.model.MangaTag
 import org.koitharu.kotatsu.parsers.model.RATING_UNKNOWN
 import org.koitharu.kotatsu.parsers.model.SortOrder
 import org.koitharu.kotatsu.parsers.site.madara.Manga18Parser
-import org.koitharu.kotatsu.parsers.util.attrAsRelativeUrlOrNull
+import org.koitharu.kotatsu.parsers.util.attrAsRelativeUrl
 import org.koitharu.kotatsu.parsers.util.domain
 import org.koitharu.kotatsu.parsers.util.generateUid
 import org.koitharu.kotatsu.parsers.util.host
-import org.koitharu.kotatsu.parsers.util.parseFailed
 import org.koitharu.kotatsu.parsers.util.parseHtml
 import org.koitharu.kotatsu.parsers.util.selectFirstOrThrow
 import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl
@@ -73,7 +72,7 @@ internal class Hentai3zCc(context: MangaLoaderContext) :


 		return doc.select("div.story_item").map { div ->
-			val href = div.selectFirst("a")?.attrAsRelativeUrlOrNull("href") ?: div.parseFailed("Link not found")
+			val href = div.selectFirstOrThrow("a").attrAsRelativeUrl("href")
 			Manga(
 				id = generateUid(href),
 				url = href,
--- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mmrcms/MmrcmsParser.kt
+++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mmrcms/MmrcmsParser.kt
@@ -24,6 +24,7 @@ internal abstract class MmrcmsParser(
 	override val sortOrders: Set<SortOrder> = EnumSet.of(
 		SortOrder.POPULARITY,
 		SortOrder.ALPHABETICAL,
+		SortOrder.UPDATED,
 	)

 	protected open val listeurl = "filterList"
@@ -54,60 +55,98 @@ internal abstract class MmrcmsParser(
 		"Terminé",
 	)

+	protected open val imgUpdated = "/cover/cover_250x350.jpg"
+
 	override suspend fun getListPage(
 		page: Int,
 		query: String?,
 		tags: Set<MangaTag>?,
 		sortOrder: SortOrder,
 	): List<Manga> {
-		val url = buildString {
-			append("https://")
-			append(domain)

-			append("/$listeurl/")
-			append("?page=")
-			append(page.toString())
-			append("&asc=true&author=&tag=")
-
-			append("&alpha=")
-			if (!query.isNullOrEmpty()) {
-				append(query.urlEncoded())
-			}
-
-			append("&cat=")
-			if (!tags.isNullOrEmpty()) {
-
-				for (tag in tags) {
-					append(tag.key)
+		val url = if (sortOrder == SortOrder.UPDATED) {
+			//the Updated page doesn't really exist, we just use the home page to weight the latest chapters, so it doesn't include tag and page management.
+			buildString {
+				append("https://")
+				append(domain)
+				if (page == 2) {
+					append("/STOP")
 				}
 			}
+		} else {
+			buildString {
+				append("https://")
+				append(domain)

-			append("&sortBy=")
-			when (sortOrder) {
-				SortOrder.POPULARITY -> append("views")
-				SortOrder.ALPHABETICAL -> append("name")
-				else -> append("views")
+				append("/$listeurl/")
+				append("?page=")
+				append(page.toString())
+				append("&asc=true&author=&tag=")
+
+				append("&alpha=")
+				if (!query.isNullOrEmpty()) {
+					append(query.urlEncoded())
+				}
+
+				append("&cat=")
+				if (!tags.isNullOrEmpty()) {
+
+					for (tag in tags) {
+						append(tag.key)
+					}
+				}
+
+				append("&sortBy=")
+				when (sortOrder) {
+					SortOrder.POPULARITY -> append("views")
+					SortOrder.ALPHABETICAL -> append("name")
+					else -> append("views")
+				}
 			}
 		}
+
 		val doc = webClient.httpGet(url).parseHtml()

-		return doc.select("div.media").map { div ->
-			val href = div.selectFirst("a")?.attrAsRelativeUrlOrNull("href") ?: div.parseFailed("Link not found")
-			Manga(
-				id = generateUid(href),
-				url = href,
-				publicUrl = href.toAbsoluteUrl(div.host ?: domain),
-				coverUrl = div.selectFirst("img")?.src().orEmpty(),
-				title = div.selectFirstOrThrow("div.media-body h5").text().orEmpty(),
-				altTitle = null,
-				rating = div.selectFirstOrThrow("span").ownText().toFloatOrNull()?.div(5f) ?: -1f,
-				tags = emptySet(),
-				author = null,
-				state = null,
-				source = source,
-				isNsfw = isNsfwSource,
-			)
+		if (sortOrder == SortOrder.UPDATED) {
+
+			return doc.select("div.manga-item").map { div ->
+				val href = div.selectFirstOrThrow("a").attrAsRelativeUrl("href")
+				val deeplink = href.substringAfterLast("/")
+				Manga(
+					id = generateUid(href),
+					url = href,
+					publicUrl = href.toAbsoluteUrl(div.host ?: domain),
+					coverUrl = "https://$domain/uploads/manga/$deeplink$imgUpdated",
+					title = div.selectFirstOrThrow("a").text().orEmpty(),
+					altTitle = null,
+					rating = RATING_UNKNOWN,
+					tags = emptySet(),
+					author = null,
+					state = null,
+					source = source,
+					isNsfw = isNsfwSource,
+				)
+			}
+		} else {
+			return doc.select("div.media").map { div ->
+				val href = div.selectFirstOrThrow("a").attrAsRelativeUrl("href")
+				Manga(
+					id = generateUid(href),
+					url = href,
+					publicUrl = href.toAbsoluteUrl(div.host ?: domain),
+					coverUrl = div.selectFirst("img")?.src().orEmpty(),
+					title = div.selectFirstOrThrow("div.media-body h5").text().orEmpty(),
+					altTitle = null,
+					rating = div.selectFirstOrThrow("span").ownText().toFloatOrNull()?.div(5f) ?: RATING_UNKNOWN,
+					tags = emptySet(),
+					author = null,
+					state = null,
+					source = source,
+					isNsfw = isNsfwSource,
+				)
+			}
 		}
+
 	}

 	override suspend fun getTags(): Set<MangaTag> {
@@ -136,15 +175,7 @@ internal abstract class MmrcmsParser(

 		val chaptersDeferred = async { getChapters(manga, doc) }

-		val desc = doc.select(selectdesc).let {
-			if (it.select("p").text().isNotEmpty()) {
-				it.select("p").joinToString(separator = "\n\n") { p ->
-					p.text().replace("<br>", "\n")
-				}
-			} else {
-				it.text()
-			}
-		}
+		val desc = doc.selectFirstOrThrow(selectdesc).text()

 		val stateDiv = body.selectFirst(selectState)?.nextElementSibling()

@@ -164,7 +195,7 @@ internal abstract class MmrcmsParser(
 		manga.copy(
 			tags = tags.mapNotNullToSet { a ->
 				MangaTag(
-					key = a.attr("href").substringAfterLast("/"),
+					key = a.attr("href").removeSuffix('/').substringAfterLast('/'),
 					title = a.text().toTitleCase(),
 					source = source,
 				)
@@ -185,8 +216,8 @@ internal abstract class MmrcmsParser(
 		val dateFormat = SimpleDateFormat(datePattern, sourceLocale)

 		return doc.body().select(selectchapter).mapChapters(reversed = true) { i, li ->
-			val a = li.selectFirst("a")
-			val href = a?.attrAsRelativeUrlOrNull("href") ?: li.parseFailed("Link is missing")
+			val a = li.selectFirstOrThrow("a")
+			val href = a.attrAsRelativeUrl("href")
 			val dateText = li.selectFirst(selectdate)?.text()
 			MangaChapter(
 				id = generateUid(href),
--- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mmrcms/fr/MangaFr.kt
+++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mmrcms/fr/MangaFr.kt
@@ -13,5 +13,6 @@ internal class MangaFr(context: MangaLoaderContext) :
 	MmrcmsParser(context, MangaSource.MANGAFR, "manga-fr.me") {


+	override val imgUpdated = ".jpg"
 	override val sourceLocale: Locale = Locale.ENGLISH
 }
--- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mmrcms/fr/MangaScan.kt
+++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mmrcms/fr/MangaScan.kt
@@ -12,5 +12,6 @@ import java.util.Locale
 internal class MangaScan(context: MangaLoaderContext) :
 	MmrcmsParser(context, MangaSource.MANGA_SCAN, "manga-scan.co") {

+	override val imgUpdated = ".jpg"
 	override val sourceLocale: Locale = Locale.ENGLISH
 }
--- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mmrcms/pt/Animaregia.kt
+++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mmrcms/pt/Animaregia.kt
@@ -9,13 +9,10 @@ import org.koitharu.kotatsu.parsers.model.Manga
 import org.koitharu.kotatsu.parsers.model.MangaSource
 import org.koitharu.kotatsu.parsers.model.MangaState
 import org.koitharu.kotatsu.parsers.model.MangaTag
+import org.koitharu.kotatsu.parsers.model.SortOrder
 import org.koitharu.kotatsu.parsers.site.madara.MmrcmsParser
-import org.koitharu.kotatsu.parsers.util.domain
-import org.koitharu.kotatsu.parsers.util.mapNotNullToSet
-import org.koitharu.kotatsu.parsers.util.parseHtml
-import org.koitharu.kotatsu.parsers.util.selectFirstOrThrow
-import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl
-import org.koitharu.kotatsu.parsers.util.toTitleCase
+import org.koitharu.kotatsu.parsers.util.*
+import java.util.EnumSet
 import java.util.Locale


@@ -26,6 +23,12 @@ internal class Animaregia(context: MangaLoaderContext) :
 	override val selectdate = "div.col-md-4"
 	override val sourceLocale: Locale = Locale.ENGLISH

+	//temporary
+	override val sortOrders: Set<SortOrder> = EnumSet.of(
+		SortOrder.POPULARITY,
+		SortOrder.ALPHABETICAL,
+	)
+
 	override suspend fun getDetails(manga: Manga): Manga = coroutineScope {
 		val fullUrl = manga.url.toAbsoluteUrl(domain)
 		val doc = webClient.httpGet(fullUrl).parseHtml()
@@ -33,15 +36,7 @@ internal class Animaregia(context: MangaLoaderContext) :

 		val chaptersDeferred = async { getChapters(manga, doc) }

-		val desc = doc.select(selectdesc).let {
-			if (it.select("p").text().isNotEmpty()) {
-				it.select("p").joinToString(separator = "\n\n") { p ->
-					p.text().replace("<br>", "\n")
-				}
-			} else {
-				it.text()
-			}
-		}
+		val desc = doc.select(selectdesc).text()

 		val stateDiv = body.selectFirst("li.list-group-item:contains(Status)")?.lastElementChild()

@@ -59,7 +54,7 @@ internal class Animaregia(context: MangaLoaderContext) :
 		manga.copy(
 			tags = tags.mapNotNullToSet { a ->
 				MangaTag(
-					key = a.attr("href").substringAfterLast("/"),
+					key = a.attr("href").removeSuffix('/').substringAfterLast('/'),
 					title = a.text().toTitleCase(),
 					source = source,
 				)