add AnimeBootstrapParser and sources

3 years ago · 880ff923bc
parent cd0d4b103a
commit 880ff923bc
5 changed files with 409 additions and 0 deletions
--- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/animebootstrap/AnimeBootstrapParser.kt
+++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/animebootstrap/AnimeBootstrapParser.kt
@ -0,0 +1,210 @@
+package org.koitharu.kotatsu.parsers.site.animebootstrap
+
+import kotlinx.coroutines.async
+import kotlinx.coroutines.coroutineScope
+import org.json.JSONArray
+import org.jsoup.nodes.Document
+import org.jsoup.nodes.Element
+import org.koitharu.kotatsu.parsers.MangaLoaderContext
+import org.koitharu.kotatsu.parsers.PagedMangaParser
+import org.koitharu.kotatsu.parsers.config.ConfigKey
+import org.koitharu.kotatsu.parsers.model.*
+import org.koitharu.kotatsu.parsers.util.*
+import java.util.*
+
+// see https://themewagon.com/themes/free-bootstrap-4-html5-gaming-anime-website-template-anime/
+
+internal abstract class AnimeBootstrapParser(
+	context: MangaLoaderContext,
+	source: MangaSource,
+	domain: String,
+	pageSize: Int = 24,
+) : PagedMangaParser(context, source, pageSize) {
+
+	override val configKeyDomain = ConfigKey.Domain(domain)
+
+	override val sortOrders: Set<SortOrder> = EnumSet.of(
+		SortOrder.UPDATED,
+		SortOrder.POPULARITY,
+		SortOrder.ALPHABETICAL,
+		SortOrder.NEWEST,
+	)
+
+	protected open val listUrl = "/manga"
+	protected open val datePattern = "dd MMM. yyyy"
+
+
+	init {
+		paginator.firstPage = 1
+		searchPaginator.firstPage = 1
+	}
+
+	override suspend fun getListPage(
+		page: Int,
+		query: String?,
+		tags: Set<MangaTag>?,
+		sortOrder: SortOrder,
+	): List<Manga> {
+		val url = buildString {
+			append("https://")
+			append(domain)
+			append(listUrl)
+			append("?page=")
+			append(page.toString())
+			append("&type=all")
+
+			if (!query.isNullOrEmpty()) {
+				append("&search=")
+				append(query.urlEncoded())
+			}
+
+			if (!tags.isNullOrEmpty()) {
+				append("&categorie=")
+				for (tag in tags) {
+					append(tag.key)
+				}
+			}
+
+			append("&sort=")
+			when (sortOrder) {
+				SortOrder.POPULARITY -> append("view")
+				SortOrder.UPDATED -> append("updated")
+				SortOrder.ALPHABETICAL -> append("default")
+				SortOrder.NEWEST -> append("published")
+				else -> append("updated")
+			}
+		}
+		val doc = webClient.httpGet(url).parseHtml()
+
+		return doc.select("div.col-6 div.product__item").map { div ->
+			val href = div.selectFirstOrThrow("a").attrAsRelativeUrl("href")
+			Manga(
+				id = generateUid(href),
+				url = href,
+				publicUrl = href.toAbsoluteUrl(div.host ?: domain),
+				coverUrl = div.selectFirstOrThrow("div.product__item__pic").attr("data-setbg").orEmpty(),
+				title = div.selectFirstOrThrow("div.product__item__text").text().orEmpty(),
+				altTitle = null,
+				rating = RATING_UNKNOWN,
+				tags = emptySet(),
+				author = null,
+				state = null,
+				source = source,
+				isNsfw = isNsfwSource,
+			)
+		}
+	}
+
+	override suspend fun getTags(): Set<MangaTag> {
+		val doc = webClient.httpGet("https://$domain$listUrl").parseHtml()
+		return doc.select("div.product__page__filter div:contains(Genre:) option ").mapNotNullToSet { option ->
+			val key = option.attr("value") ?: return@mapNotNullToSet null
+			val name = option.text()
+			MangaTag(
+				key = key,
+				title = name,
+				source = source,
+			)
+		}
+	}
+
+	protected open val selectDesc = "div.anime__details__text p"
+	protected open val selectState = "div.anime__details__widget li:contains(Ongoing)"
+	protected open val selectTag = "div.anime__details__widget li:contains(Categorie) a"
+
+	override suspend fun getDetails(manga: Manga): Manga = coroutineScope {
+		val fullUrl = manga.url.toAbsoluteUrl(domain)
+		val doc = webClient.httpGet(fullUrl).parseHtml()
+
+		val chaptersDeferred = async { getChapters(manga, doc) }
+
+		val desc = doc.selectFirstOrThrow(selectDesc).html()
+
+		val state = if (doc.select(selectState).isNullOrEmpty()) {
+			MangaState.FINISHED
+		} else {
+			MangaState.ONGOING
+		}
+
+		manga.copy(
+			tags = doc.body().select(selectTag).mapNotNullToSet { a ->
+				MangaTag(
+					key = a.attr("href").substringAfterLast('='),
+					title = a.text().toTitleCase().replace(",", ""),
+					source = source,
+				)
+			},
+			description = desc,
+			state = state,
+			chapters = chaptersDeferred.await(),
+		)
+	}
+
+
+	protected open val selectChapter = "div.anime__details__episodes a"
+
+	protected open suspend fun getChapters(manga: Manga, doc: Document): List<MangaChapter> {
+		return doc.body().select(selectChapter).mapChapters(reversed = true) { i, a ->
+			val href = a.attr("href")
+			MangaChapter(
+				id = generateUid(href),
+				name = a.text(),
+				number = i + 1,
+				url = href,
+				uploadDate = 0,
+				source = source,
+				scanlator = null,
+				branch = null,
+			)
+		}
+	}
+
+
+	protected open val selectPage = "div.read-img img"
+
+	override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
+		val fullUrl = chapter.url.toAbsoluteUrl(domain)
+		val doc = webClient.httpGet(fullUrl).parseHtml()
+
+		if (doc.select("script:containsData(page_image)").isNullOrEmpty()) {
+			return doc.select(selectPage).map { img ->
+				val url = img.attr("onerror").replace("this.onerror=null;this.src=`", "").replace("`;", "")
+				MangaPage(
+					id = generateUid(url),
+					url = url,
+					preview = null,
+					source = source,
+				)
+			}
+		} else {
+			val script = doc.selectFirstOrThrow("script:containsData(page_image)")
+			val images = JSONArray(script.data().substringAfterLast("var pages = ").substringBefore(';'))
+
+			val pages = ArrayList<MangaPage>(images.length())
+			for (i in 0 until images.length()) {
+
+				val pageTake = images.getJSONObject(i)
+				pages.add(
+					MangaPage(
+						id = generateUid(pageTake.getString("page_image")),
+						url = pageTake.getString("page_image"),
+						preview = null,
+						source = source,
+					),
+				)
+			}
+
+			return pages
+		}
+
+	}
+
+
+	protected fun Element.src(): String? {
+		var result = absUrl("data-src")
+		if (result.isEmpty()) result = absUrl("data-cfsrc")
+		if (result.isEmpty()) result = absUrl("src")
+		return result.ifEmpty { null }
+	}
+
+}
--- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/animebootstrap/fr/PapScan.kt
+++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/animebootstrap/fr/PapScan.kt
@ -0,0 +1,164 @@
+package org.koitharu.kotatsu.parsers.site.animebootstrap.fr
+
+
+import kotlinx.coroutines.async
+import kotlinx.coroutines.coroutineScope
+import org.jsoup.nodes.Document
+import org.koitharu.kotatsu.parsers.MangaLoaderContext
+import org.koitharu.kotatsu.parsers.MangaSourceParser
+import org.koitharu.kotatsu.parsers.model.Manga
+import org.koitharu.kotatsu.parsers.model.MangaChapter
+import org.koitharu.kotatsu.parsers.model.MangaSource
+import org.koitharu.kotatsu.parsers.model.MangaState
+import org.koitharu.kotatsu.parsers.model.MangaTag
+import org.koitharu.kotatsu.parsers.model.RATING_UNKNOWN
+import org.koitharu.kotatsu.parsers.model.SortOrder
+import org.koitharu.kotatsu.parsers.site.animebootstrap.AnimeBootstrapParser
+import org.koitharu.kotatsu.parsers.util.attrAsRelativeUrl
+import org.koitharu.kotatsu.parsers.util.domain
+import org.koitharu.kotatsu.parsers.util.generateUid
+import org.koitharu.kotatsu.parsers.util.host
+import org.koitharu.kotatsu.parsers.util.mapChapters
+import org.koitharu.kotatsu.parsers.util.mapNotNullToSet
+import org.koitharu.kotatsu.parsers.util.parseHtml
+import org.koitharu.kotatsu.parsers.util.removeSuffix
+import org.koitharu.kotatsu.parsers.util.selectFirstOrThrow
+import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl
+import org.koitharu.kotatsu.parsers.util.toTitleCase
+import org.koitharu.kotatsu.parsers.util.tryParse
+import org.koitharu.kotatsu.parsers.util.urlEncoded
+import java.text.SimpleDateFormat
+import java.util.EnumSet
+import java.util.Locale
+
+
+@MangaSourceParser("PAPSCAN", "PapScan", "fr")
+internal class PapScan(context: MangaLoaderContext) :
+	AnimeBootstrapParser(context, MangaSource.PAPSCAN, "papscan.com") {
+
+	override val sourceLocale: Locale = Locale.ENGLISH
+
+	override val listUrl = "/liste-manga"
+
+	override val selectState = "div.anime__details__widget li:contains(En cours)"
+	override val selectTag = "div.anime__details__widget li:contains(Genre) a"
+
+	override val selectChapter = "ul.chapters li"
+
+	override val sortOrders: Set<SortOrder> = EnumSet.of(
+		SortOrder.POPULARITY,
+		SortOrder.ALPHABETICAL,
+	)
+
+	override suspend fun getListPage(
+		page: Int,
+		query: String?,
+		tags: Set<MangaTag>?,
+		sortOrder: SortOrder,
+	): List<Manga> {
+		val url = buildString {
+			append("https://")
+			append(domain)
+			append("/filterList")
+			append("?page=")
+			append(page.toString())
+
+			if (!query.isNullOrEmpty()) {
+				append("&alpha=")
+				append(query.urlEncoded())
+			}
+
+			if (!tags.isNullOrEmpty()) {
+				append("&cat=")
+				for (tag in tags) {
+					append(tag.key)
+				}
+			}
+			append("&sortBy=")
+			when (sortOrder) {
+				SortOrder.POPULARITY -> append("views")
+				SortOrder.ALPHABETICAL -> append("name")
+				else -> append("updated")
+			}
+		}
+		val doc = webClient.httpGet(url).parseHtml()
+
+		return doc.select("div.product__item").map { div ->
+			val href = div.selectFirstOrThrow("h5 a").attrAsRelativeUrl("href")
+			Manga(
+				id = generateUid(href),
+				url = href,
+				publicUrl = href.toAbsoluteUrl(div.host ?: domain),
+				coverUrl = div.selectFirstOrThrow("div.product__item__pic").attr("data-setbg").orEmpty(),
+				title = div.selectFirstOrThrow("div.product__item__text h5").text().orEmpty(),
+				altTitle = null,
+				rating = RATING_UNKNOWN,
+				tags = emptySet(),
+				author = null,
+				state = null,
+				source = source,
+				isNsfw = isNsfwSource,
+			)
+		}
+	}
+
+	override suspend fun getTags(): Set<MangaTag> {
+		val doc = webClient.httpGet("https://$domain$listUrl").parseHtml()
+		return doc.select("a.category ").mapNotNullToSet { a ->
+			val key = a.attr("href").substringAfterLast('=')
+			val name = a.text()
+			MangaTag(
+				key = key,
+				title = name,
+				source = source,
+			)
+		}
+	}
+
+	override suspend fun getDetails(manga: Manga): Manga = coroutineScope {
+		val fullUrl = manga.url.toAbsoluteUrl(domain)
+		val doc = webClient.httpGet(fullUrl).parseHtml()
+
+		val chaptersDeferred = async { getChapters(manga, doc) }
+
+		val desc = doc.selectFirstOrThrow(selectDesc).html()
+
+		val state = if (doc.select(selectState).isNullOrEmpty()) {
+			MangaState.FINISHED
+		} else {
+			MangaState.ONGOING
+		}
+
+		manga.copy(
+			tags = doc.body().select(selectTag).mapNotNullToSet { a ->
+				MangaTag(
+					key = a.attr("href").removeSuffix('/').substringAfterLast('/'),
+					title = a.text().toTitleCase(),
+					source = source,
+				)
+			},
+			description = desc,
+			state = state,
+			chapters = chaptersDeferred.await(),
+		)
+	}
+
+	override suspend fun getChapters(manga: Manga, doc: Document): List<MangaChapter> {
+		val dateFormat = SimpleDateFormat(datePattern, sourceLocale)
+		return doc.body().select(selectChapter).mapChapters(reversed = true) { i, li ->
+			val href = li.selectFirstOrThrow("a").attr("href")
+			val dateText = li.selectFirst("span.date-chapter-title-rtl")?.text()
+			MangaChapter(
+				id = generateUid(href),
+				name = li.selectFirstOrThrow("span em").text(),
+				number = i + 1,
+				url = href,
+				uploadDate = dateFormat.tryParse(dateText),
+				source = source,
+				scanlator = null,
+				branch = null,
+			)
+		}
+	}
+
+}
--- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/animebootstrap/id/KomikzoId.kt
+++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/animebootstrap/id/KomikzoId.kt
@ -0,0 +1,12 @@
+package org.koitharu.kotatsu.parsers.site.animebootstrap.id
+
+
+import org.koitharu.kotatsu.parsers.MangaLoaderContext
+import org.koitharu.kotatsu.parsers.MangaSourceParser
+import org.koitharu.kotatsu.parsers.model.MangaSource
+import org.koitharu.kotatsu.parsers.site.animebootstrap.AnimeBootstrapParser
+
+
+@MangaSourceParser("KOMIKZOID", "KomikzoId", "id")
+internal class KomikzoId(context: MangaLoaderContext) :
+	AnimeBootstrapParser(context, MangaSource.KOMIKZOID, "komikzoid.xyz")
--- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/animebootstrap/id/NeuManga.kt
+++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/animebootstrap/id/NeuManga.kt
@ -0,0 +1,12 @@
+package org.koitharu.kotatsu.parsers.site.animebootstrap.id
+
+
+import org.koitharu.kotatsu.parsers.MangaLoaderContext
+import org.koitharu.kotatsu.parsers.MangaSourceParser
+import org.koitharu.kotatsu.parsers.model.MangaSource
+import org.koitharu.kotatsu.parsers.site.animebootstrap.AnimeBootstrapParser
+
+
+@MangaSourceParser("NEUMANGA", "Neu Manga", "id")
+internal class NeuManga(context: MangaLoaderContext) :
+	AnimeBootstrapParser(context, MangaSource.NEUMANGA, "neumanga.xyz")
--- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/animebootstrap/id/SekteKomik.kt
+++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/animebootstrap/id/SekteKomik.kt
@ -0,0 +1,11 @@
+package org.koitharu.kotatsu.parsers.site.animebootstrap.id
+
+
+import org.koitharu.kotatsu.parsers.MangaLoaderContext
+import org.koitharu.kotatsu.parsers.MangaSourceParser
+import org.koitharu.kotatsu.parsers.model.MangaSource
+import org.koitharu.kotatsu.parsers.site.animebootstrap.AnimeBootstrapParser
+
+@MangaSourceParser("SEKTEKOMIK", "Sekte Komik", "id")
+internal class SekteKomik(context: MangaLoaderContext) :
+	AnimeBootstrapParser(context, MangaSource.SEKTEKOMIK, "sektekomik.xyz")