add MadthemeParser and sources
devi
3 years ago
parent
ef6ede6b12
commit
87d4326411
@ -0,0 +1,307 @@
|
||||
package org.koitharu.kotatsu.parsers.site.madtheme
|
||||
|
||||
import kotlinx.coroutines.async
|
||||
import kotlinx.coroutines.coroutineScope
|
||||
import org.jsoup.nodes.Document
|
||||
import org.jsoup.nodes.Element
|
||||
import org.koitharu.kotatsu.parsers.MangaLoaderContext
|
||||
import org.koitharu.kotatsu.parsers.PagedMangaParser
|
||||
import org.koitharu.kotatsu.parsers.config.ConfigKey
|
||||
import org.koitharu.kotatsu.parsers.model.*
|
||||
import org.koitharu.kotatsu.parsers.util.*
|
||||
import java.text.DateFormat
|
||||
import java.text.SimpleDateFormat
|
||||
import java.util.*
|
||||
|
||||
internal abstract class MadthemeParser(
|
||||
context: MangaLoaderContext,
|
||||
source: MangaSource,
|
||||
domain: String,
|
||||
pageSize: Int = 48,
|
||||
) : PagedMangaParser(context, source, pageSize) {
|
||||
|
||||
override val configKeyDomain = ConfigKey.Domain(domain)
|
||||
|
||||
override val sortOrders: Set<SortOrder> = EnumSet.of(
|
||||
SortOrder.UPDATED,
|
||||
SortOrder.POPULARITY,
|
||||
SortOrder.ALPHABETICAL,
|
||||
SortOrder.NEWEST,
|
||||
SortOrder.RATING,
|
||||
)
|
||||
|
||||
protected open val listUrl = "search/"
|
||||
protected open val datePattern = "MMM dd, yyyy"
|
||||
|
||||
|
||||
init {
|
||||
paginator.firstPage = 1
|
||||
searchPaginator.firstPage = 1
|
||||
}
|
||||
|
||||
|
||||
@JvmField
|
||||
protected val ongoing: Set<String> = setOf(
|
||||
"On Going",
|
||||
"Ongoing",
|
||||
"ONGOING",
|
||||
)
|
||||
|
||||
@JvmField
|
||||
protected val finished: Set<String> = setOf(
|
||||
"Completed",
|
||||
"COMPLETED",
|
||||
)
|
||||
|
||||
override suspend fun getListPage(
|
||||
page: Int,
|
||||
query: String?,
|
||||
tags: Set<MangaTag>?,
|
||||
sortOrder: SortOrder,
|
||||
): List<Manga> {
|
||||
val url = buildString {
|
||||
append("https://")
|
||||
append(domain)
|
||||
append("/$listUrl?sort=")
|
||||
when (sortOrder) {
|
||||
SortOrder.POPULARITY -> append("views")
|
||||
SortOrder.UPDATED -> append("updated_at")
|
||||
SortOrder.ALPHABETICAL -> append("name") // On some sites without tags or searches, the alphabetical option is empty.
|
||||
SortOrder.NEWEST -> append("created_at")
|
||||
SortOrder.RATING -> append("rating")
|
||||
}
|
||||
if (!query.isNullOrEmpty()) {
|
||||
append("&q=")
|
||||
append(query.urlEncoded())
|
||||
}
|
||||
|
||||
if (!tags.isNullOrEmpty()) {
|
||||
for (tag in tags) {
|
||||
append("&")
|
||||
append("genre[]".urlEncoded())
|
||||
append("=")
|
||||
append(tag.key)
|
||||
}
|
||||
}
|
||||
|
||||
append("&page=")
|
||||
append(page.toString())
|
||||
}
|
||||
|
||||
val doc = webClient.httpGet(url).parseHtml()
|
||||
|
||||
return doc.select("div.book-item").map { div ->
|
||||
val href = div.selectFirstOrThrow("a").attrAsRelativeUrl("href")
|
||||
Manga(
|
||||
id = generateUid(href),
|
||||
url = href,
|
||||
publicUrl = href.toAbsoluteUrl(div.host ?: domain),
|
||||
coverUrl = div.selectFirst("img")?.src().orEmpty(),
|
||||
title = div.selectFirstOrThrow("div.meta").selectFirst("div.title")?.text().orEmpty(),
|
||||
altTitle = null,
|
||||
rating = div.selectFirstOrThrow("div.meta span.score").ownText().toFloatOrNull()?.div(5f)
|
||||
?: RATING_UNKNOWN,
|
||||
tags = doc.body().select("div.meta div.genres span").mapNotNullToSet { span ->
|
||||
MangaTag(
|
||||
key = span.attr("class"),
|
||||
title = span.text().toTitleCase(),
|
||||
source = source,
|
||||
)
|
||||
},
|
||||
author = null,
|
||||
state = null,
|
||||
source = source,
|
||||
isNsfw = isNsfwSource,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
override suspend fun getTags(): Set<MangaTag> {
|
||||
val doc = webClient.httpGet("https://$domain/$listUrl").parseHtml()
|
||||
return doc.select("div.genres label.checkbox").mapNotNullToSet { checkbox ->
|
||||
val key = checkbox.selectFirstOrThrow("input").attr("value") ?: return@mapNotNullToSet null
|
||||
val name = checkbox.selectFirstOrThrow("span.radio__label").text()
|
||||
MangaTag(
|
||||
key = key,
|
||||
title = name,
|
||||
source = source,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
protected open val selectDesc = "div.section-body.summary p.content"
|
||||
protected open val selectState = "div.detail p:contains(Status) span"
|
||||
protected open val selectAlt = "div.detail div.name h2"
|
||||
protected open val selectTag = "div.detail p:contains(Genres) a"
|
||||
|
||||
override suspend fun getDetails(manga: Manga): Manga = coroutineScope {
|
||||
val fullUrl = manga.url.toAbsoluteUrl(domain)
|
||||
val doc = webClient.httpGet(fullUrl).parseHtml()
|
||||
|
||||
val chaptersDeferred = async { getChapters(manga, doc) }
|
||||
|
||||
val desc = doc.selectFirstOrThrow(selectDesc).html()
|
||||
|
||||
val stateDiv = doc.selectFirst(selectState)
|
||||
|
||||
val state = stateDiv?.let {
|
||||
when (it.text()) {
|
||||
in ongoing -> MangaState.ONGOING
|
||||
in finished -> MangaState.FINISHED
|
||||
else -> null
|
||||
}
|
||||
}
|
||||
|
||||
val alt = doc.body().select(selectAlt).text()
|
||||
|
||||
val nsfw = doc.getElementById("adt-warning") != null
|
||||
|
||||
manga.copy(
|
||||
tags = doc.body().select(selectTag).mapNotNullToSet { a ->
|
||||
MangaTag(
|
||||
key = a.attr("href").removeSuffix('/').substringAfterLast('/'),
|
||||
title = a.text().toTitleCase().replace(",", ""),
|
||||
source = source,
|
||||
)
|
||||
},
|
||||
description = desc,
|
||||
altTitle = alt,
|
||||
state = state,
|
||||
chapters = chaptersDeferred.await(),
|
||||
isNsfw = nsfw || manga.isNsfw,
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
protected open val selectDate = "div .chapter-update"
|
||||
protected open val selectChapter = "ul#chapter-list li"
|
||||
|
||||
protected open suspend fun getChapters(manga: Manga, doc: Document): List<MangaChapter> {
|
||||
val dateFormat = SimpleDateFormat(datePattern, sourceLocale)
|
||||
return doc.body().select(selectChapter).mapChapters(reversed = true) { i, li ->
|
||||
val a = li.selectFirstOrThrow("a")
|
||||
val href = a.attrAsRelativeUrl("href")
|
||||
val dateText = li.selectFirst(selectDate)?.text()
|
||||
MangaChapter(
|
||||
id = generateUid(href),
|
||||
name = li.selectFirstOrThrow(".chapter-title").text(),
|
||||
number = i + 1,
|
||||
url = href,
|
||||
uploadDate = parseChapterDate(
|
||||
dateFormat,
|
||||
dateText,
|
||||
),
|
||||
source = source,
|
||||
scanlator = null,
|
||||
branch = null,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
protected open val selectPage = "div#chapter-images img"
|
||||
|
||||
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
|
||||
val fullUrl = chapter.url.toAbsoluteUrl(domain)
|
||||
val doc = webClient.httpGet(fullUrl).parseHtml()
|
||||
|
||||
return doc.select(selectPage).map { img ->
|
||||
val url = img.src()?.toRelativeUrl(domain) ?: img.parseFailed("Image src not found")
|
||||
MangaPage(
|
||||
id = generateUid(url),
|
||||
url = url,
|
||||
preview = null,
|
||||
source = source,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
protected fun Element.src(): String? {
|
||||
var result = absUrl("data-src")
|
||||
if (result.isEmpty()) result = absUrl("data-cfsrc")
|
||||
if (result.isEmpty()) result = absUrl("src")
|
||||
return result.ifEmpty { null }
|
||||
}
|
||||
|
||||
protected fun parseChapterDate(dateFormat: DateFormat, date: String?): Long {
|
||||
// Clean date (e.g. 5th December 2019 to 5 December 2019) before parsing it
|
||||
val d = date?.lowercase() ?: return 0
|
||||
return when {
|
||||
d.endsWith(" ago") ||
|
||||
// short Hours
|
||||
d.endsWith(" h") ||
|
||||
// short Day
|
||||
d.endsWith(" d") -> parseRelativeDate(date)
|
||||
|
||||
// Handle 'yesterday' and 'today', using midnight
|
||||
d.startsWith("year") -> Calendar.getInstance().apply {
|
||||
add(Calendar.DAY_OF_MONTH, -1) // yesterday
|
||||
set(Calendar.HOUR_OF_DAY, 0)
|
||||
set(Calendar.MINUTE, 0)
|
||||
set(Calendar.SECOND, 0)
|
||||
set(Calendar.MILLISECOND, 0)
|
||||
}.timeInMillis
|
||||
|
||||
d.startsWith("today") -> Calendar.getInstance().apply {
|
||||
set(Calendar.HOUR_OF_DAY, 0)
|
||||
set(Calendar.MINUTE, 0)
|
||||
set(Calendar.SECOND, 0)
|
||||
set(Calendar.MILLISECOND, 0)
|
||||
}.timeInMillis
|
||||
|
||||
date.contains(Regex("""\d(st|nd|rd|th)""")) -> date.split(" ").map {
|
||||
if (it.contains(Regex("""\d\D\D"""))) {
|
||||
it.replace(Regex("""\D"""), "")
|
||||
} else {
|
||||
it
|
||||
}
|
||||
}.let { dateFormat.tryParse(it.joinToString(" ")) }
|
||||
|
||||
else -> dateFormat.tryParse(date)
|
||||
}
|
||||
}
|
||||
|
||||
// Parses dates in this form:
|
||||
// 21 hours ago
|
||||
private fun parseRelativeDate(date: String): Long {
|
||||
val number = Regex("""(\d+)""").find(date)?.value?.toIntOrNull() ?: return 0
|
||||
val cal = Calendar.getInstance()
|
||||
|
||||
return when {
|
||||
WordSet(
|
||||
"day",
|
||||
"days",
|
||||
).anyWordIn(date) -> cal.apply { add(Calendar.DAY_OF_MONTH, -number) }.timeInMillis
|
||||
|
||||
WordSet("hour", "hours", "h").anyWordIn(date) -> cal.apply {
|
||||
add(
|
||||
Calendar.HOUR,
|
||||
-number,
|
||||
)
|
||||
}.timeInMillis
|
||||
|
||||
WordSet(
|
||||
"min",
|
||||
"minute",
|
||||
"minutes",
|
||||
).anyWordIn(date) -> cal.apply {
|
||||
add(
|
||||
Calendar.MINUTE,
|
||||
-number,
|
||||
)
|
||||
}.timeInMillis
|
||||
|
||||
WordSet("second").anyWordIn(date) -> cal.apply {
|
||||
add(
|
||||
Calendar.SECOND,
|
||||
-number,
|
||||
)
|
||||
}.timeInMillis
|
||||
|
||||
WordSet("month", "months").anyWordIn(date) -> cal.apply { add(Calendar.MONTH, -number) }.timeInMillis
|
||||
WordSet("year").anyWordIn(date) -> cal.apply { add(Calendar.YEAR, -number) }.timeInMillis
|
||||
else -> 0
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,16 @@
|
||||
package org.koitharu.kotatsu.parsers.site.madtheme.en
|
||||
|
||||
|
||||
import org.koitharu.kotatsu.parsers.MangaLoaderContext
|
||||
import org.koitharu.kotatsu.parsers.MangaSourceParser
|
||||
import org.koitharu.kotatsu.parsers.model.ContentType
|
||||
import org.koitharu.kotatsu.parsers.model.MangaSource
|
||||
import org.koitharu.kotatsu.parsers.site.madtheme.MadthemeParser
|
||||
|
||||
|
||||
@MangaSourceParser("BEEHENTAI", "BeeHentai", "en", ContentType.HENTAI)
|
||||
internal class BeeHentai(context: MangaLoaderContext) :
|
||||
MadthemeParser(context, MangaSource.BEEHENTAI, "beehentai.com") {
|
||||
|
||||
override val selectDesc = "div.section-body"
|
||||
}
|
||||
@ -0,0 +1,12 @@
|
||||
package org.koitharu.kotatsu.parsers.site.madtheme.en
|
||||
|
||||
|
||||
import org.koitharu.kotatsu.parsers.MangaLoaderContext
|
||||
import org.koitharu.kotatsu.parsers.MangaSourceParser
|
||||
import org.koitharu.kotatsu.parsers.model.MangaSource
|
||||
import org.koitharu.kotatsu.parsers.site.madtheme.MadthemeParser
|
||||
|
||||
|
||||
@MangaSourceParser("MANGABUDDY", "MangaBuddy", "en")
|
||||
internal class MangaBuddy(context: MangaLoaderContext) :
|
||||
MadthemeParser(context, MangaSource.MANGABUDDY, "mangabuddy.com")
|
||||
@ -0,0 +1,12 @@
|
||||
package org.koitharu.kotatsu.parsers.site.madtheme.en
|
||||
|
||||
|
||||
import org.koitharu.kotatsu.parsers.MangaLoaderContext
|
||||
import org.koitharu.kotatsu.parsers.MangaSourceParser
|
||||
import org.koitharu.kotatsu.parsers.model.MangaSource
|
||||
import org.koitharu.kotatsu.parsers.site.madtheme.MadthemeParser
|
||||
|
||||
|
||||
@MangaSourceParser("MANGACUTE", "MangaCute", "en")
|
||||
internal class MangaCute(context: MangaLoaderContext) :
|
||||
MadthemeParser(context, MangaSource.MANGACUTE, "mangacute.com")
|
||||
@ -0,0 +1,12 @@
|
||||
package org.koitharu.kotatsu.parsers.site.madtheme.en
|
||||
|
||||
|
||||
import org.koitharu.kotatsu.parsers.MangaLoaderContext
|
||||
import org.koitharu.kotatsu.parsers.MangaSourceParser
|
||||
import org.koitharu.kotatsu.parsers.model.MangaSource
|
||||
import org.koitharu.kotatsu.parsers.site.madtheme.MadthemeParser
|
||||
|
||||
|
||||
@MangaSourceParser("MANGAFOREST", "Manga Forest", "en")
|
||||
internal class MangaForest(context: MangaLoaderContext) :
|
||||
MadthemeParser(context, MangaSource.MANGAFOREST, "mangaforest.me")
|
||||
@ -0,0 +1,12 @@
|
||||
package org.koitharu.kotatsu.parsers.site.madtheme.en
|
||||
|
||||
|
||||
import org.koitharu.kotatsu.parsers.MangaLoaderContext
|
||||
import org.koitharu.kotatsu.parsers.MangaSourceParser
|
||||
import org.koitharu.kotatsu.parsers.model.MangaSource
|
||||
import org.koitharu.kotatsu.parsers.site.madtheme.MadthemeParser
|
||||
|
||||
|
||||
@MangaSourceParser("MANGAPUMA", "Manga Puma", "en")
|
||||
internal class MangaPuma(context: MangaLoaderContext) :
|
||||
MadthemeParser(context, MangaSource.MANGAPUMA, "mangapuma.com")
|
||||
@ -0,0 +1,12 @@
|
||||
package org.koitharu.kotatsu.parsers.site.madtheme.en
|
||||
|
||||
|
||||
import org.koitharu.kotatsu.parsers.MangaLoaderContext
|
||||
import org.koitharu.kotatsu.parsers.MangaSourceParser
|
||||
import org.koitharu.kotatsu.parsers.model.MangaSource
|
||||
import org.koitharu.kotatsu.parsers.site.madtheme.MadthemeParser
|
||||
|
||||
|
||||
@MangaSourceParser("MANGAXYZ", "Mangaxyz", "en")
|
||||
internal class Mangaxyz(context: MangaLoaderContext) :
|
||||
MadthemeParser(context, MangaSource.MANGAXYZ, "mangaxyz.com")
|
||||
@ -0,0 +1,16 @@
|
||||
package org.koitharu.kotatsu.parsers.site.madtheme.en
|
||||
|
||||
|
||||
import org.koitharu.kotatsu.parsers.MangaLoaderContext
|
||||
import org.koitharu.kotatsu.parsers.MangaSourceParser
|
||||
import org.koitharu.kotatsu.parsers.model.ContentType
|
||||
import org.koitharu.kotatsu.parsers.model.MangaSource
|
||||
import org.koitharu.kotatsu.parsers.site.madtheme.MadthemeParser
|
||||
|
||||
|
||||
@MangaSourceParser("TOONITUBE", "TooniTube", "en", ContentType.HENTAI)
|
||||
internal class TooniTube(context: MangaLoaderContext) :
|
||||
MadthemeParser(context, MangaSource.TOONITUBE, "toonitube.com") {
|
||||
|
||||
override val selectDesc = "div.summary div.section-body p.content"
|
||||
}
|
||||
@ -0,0 +1,16 @@
|
||||
package org.koitharu.kotatsu.parsers.site.madtheme.en
|
||||
|
||||
|
||||
import org.koitharu.kotatsu.parsers.MangaLoaderContext
|
||||
import org.koitharu.kotatsu.parsers.MangaSourceParser
|
||||
import org.koitharu.kotatsu.parsers.model.ContentType
|
||||
import org.koitharu.kotatsu.parsers.model.MangaSource
|
||||
import org.koitharu.kotatsu.parsers.site.madtheme.MadthemeParser
|
||||
|
||||
|
||||
@MangaSourceParser("TOONILY_ME", "Toonily Me", "en", ContentType.HENTAI)
|
||||
internal class ToonilyMe(context: MangaLoaderContext) :
|
||||
MadthemeParser(context, MangaSource.TOONILY_ME, "toonily.me") {
|
||||
|
||||
override val selectDesc = "div.summary div.section-body p.content"
|
||||
}
|
||||
@ -0,0 +1,12 @@
|
||||
package org.koitharu.kotatsu.parsers.site.madtheme.en
|
||||
|
||||
|
||||
import org.koitharu.kotatsu.parsers.MangaLoaderContext
|
||||
import org.koitharu.kotatsu.parsers.MangaSourceParser
|
||||
import org.koitharu.kotatsu.parsers.model.MangaSource
|
||||
import org.koitharu.kotatsu.parsers.site.madtheme.MadthemeParser
|
||||
|
||||
|
||||
@MangaSourceParser("TRUEMANGA", "True Manga", "en")
|
||||
internal class TrueManga(context: MangaLoaderContext) :
|
||||
MadthemeParser(context, MangaSource.TRUEMANGA, "truemanga.com")
|
||||
Loading…
Reference in New Issue