diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madtheme/MadthemeParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madtheme/MadthemeParser.kt new file mode 100644 index 00000000..6c7de534 --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madtheme/MadthemeParser.kt @@ -0,0 +1,307 @@ +package org.koitharu.kotatsu.parsers.site.madtheme + +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope +import org.jsoup.nodes.Document +import org.jsoup.nodes.Element +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.PagedMangaParser +import org.koitharu.kotatsu.parsers.config.ConfigKey +import org.koitharu.kotatsu.parsers.model.* +import org.koitharu.kotatsu.parsers.util.* +import java.text.DateFormat +import java.text.SimpleDateFormat +import java.util.* + +internal abstract class MadthemeParser( + context: MangaLoaderContext, + source: MangaSource, + domain: String, + pageSize: Int = 48, +) : PagedMangaParser(context, source, pageSize) { + + override val configKeyDomain = ConfigKey.Domain(domain) + + override val sortOrders: Set = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.ALPHABETICAL, + SortOrder.NEWEST, + SortOrder.RATING, + ) + + protected open val listUrl = "search/" + protected open val datePattern = "MMM dd, yyyy" + + + init { + paginator.firstPage = 1 + searchPaginator.firstPage = 1 + } + + + @JvmField + protected val ongoing: Set = setOf( + "On Going", + "Ongoing", + "ONGOING", + ) + + @JvmField + protected val finished: Set = setOf( + "Completed", + "COMPLETED", + ) + + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + val url = buildString { + append("https://") + append(domain) + append("/$listUrl?sort=") + when (sortOrder) { + SortOrder.POPULARITY -> append("views") + SortOrder.UPDATED -> append("updated_at") + SortOrder.ALPHABETICAL -> append("name") // On some sites without tags or searches, the alphabetical option is empty. + SortOrder.NEWEST -> append("created_at") + SortOrder.RATING -> append("rating") + } + if (!query.isNullOrEmpty()) { + append("&q=") + append(query.urlEncoded()) + } + + if (!tags.isNullOrEmpty()) { + for (tag in tags) { + append("&") + append("genre[]".urlEncoded()) + append("=") + append(tag.key) + } + } + + append("&page=") + append(page.toString()) + } + + val doc = webClient.httpGet(url).parseHtml() + + return doc.select("div.book-item").map { div -> + val href = div.selectFirstOrThrow("a").attrAsRelativeUrl("href") + Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(div.host ?: domain), + coverUrl = div.selectFirst("img")?.src().orEmpty(), + title = div.selectFirstOrThrow("div.meta").selectFirst("div.title")?.text().orEmpty(), + altTitle = null, + rating = div.selectFirstOrThrow("div.meta span.score").ownText().toFloatOrNull()?.div(5f) + ?: RATING_UNKNOWN, + tags = doc.body().select("div.meta div.genres span").mapNotNullToSet { span -> + MangaTag( + key = span.attr("class"), + title = span.text().toTitleCase(), + source = source, + ) + }, + author = null, + state = null, + source = source, + isNsfw = isNsfwSource, + ) + } + } + + override suspend fun getTags(): Set { + val doc = webClient.httpGet("https://$domain/$listUrl").parseHtml() + return doc.select("div.genres label.checkbox").mapNotNullToSet { checkbox -> + val key = checkbox.selectFirstOrThrow("input").attr("value") ?: return@mapNotNullToSet null + val name = checkbox.selectFirstOrThrow("span.radio__label").text() + MangaTag( + key = key, + title = name, + source = source, + ) + } + } + + protected open val selectDesc = "div.section-body.summary p.content" + protected open val selectState = "div.detail p:contains(Status) span" + protected open val selectAlt = "div.detail div.name h2" + protected open val selectTag = "div.detail p:contains(Genres) a" + + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { + val fullUrl = manga.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + + val chaptersDeferred = async { getChapters(manga, doc) } + + val desc = doc.selectFirstOrThrow(selectDesc).html() + + val stateDiv = doc.selectFirst(selectState) + + val state = stateDiv?.let { + when (it.text()) { + in ongoing -> MangaState.ONGOING + in finished -> MangaState.FINISHED + else -> null + } + } + + val alt = doc.body().select(selectAlt).text() + + val nsfw = doc.getElementById("adt-warning") != null + + manga.copy( + tags = doc.body().select(selectTag).mapNotNullToSet { a -> + MangaTag( + key = a.attr("href").removeSuffix('/').substringAfterLast('/'), + title = a.text().toTitleCase().replace(",", ""), + source = source, + ) + }, + description = desc, + altTitle = alt, + state = state, + chapters = chaptersDeferred.await(), + isNsfw = nsfw || manga.isNsfw, + ) + } + + + protected open val selectDate = "div .chapter-update" + protected open val selectChapter = "ul#chapter-list li" + + protected open suspend fun getChapters(manga: Manga, doc: Document): List { + val dateFormat = SimpleDateFormat(datePattern, sourceLocale) + return doc.body().select(selectChapter).mapChapters(reversed = true) { i, li -> + val a = li.selectFirstOrThrow("a") + val href = a.attrAsRelativeUrl("href") + val dateText = li.selectFirst(selectDate)?.text() + MangaChapter( + id = generateUid(href), + name = li.selectFirstOrThrow(".chapter-title").text(), + number = i + 1, + url = href, + uploadDate = parseChapterDate( + dateFormat, + dateText, + ), + source = source, + scanlator = null, + branch = null, + ) + } + } + + protected open val selectPage = "div#chapter-images img" + + override suspend fun getPages(chapter: MangaChapter): List { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + + return doc.select(selectPage).map { img -> + val url = img.src()?.toRelativeUrl(domain) ?: img.parseFailed("Image src not found") + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } + + + protected fun Element.src(): String? { + var result = absUrl("data-src") + if (result.isEmpty()) result = absUrl("data-cfsrc") + if (result.isEmpty()) result = absUrl("src") + return result.ifEmpty { null } + } + + protected fun parseChapterDate(dateFormat: DateFormat, date: String?): Long { + // Clean date (e.g. 5th December 2019 to 5 December 2019) before parsing it + val d = date?.lowercase() ?: return 0 + return when { + d.endsWith(" ago") || + // short Hours + d.endsWith(" h") || + // short Day + d.endsWith(" d") -> parseRelativeDate(date) + + // Handle 'yesterday' and 'today', using midnight + d.startsWith("year") -> Calendar.getInstance().apply { + add(Calendar.DAY_OF_MONTH, -1) // yesterday + set(Calendar.HOUR_OF_DAY, 0) + set(Calendar.MINUTE, 0) + set(Calendar.SECOND, 0) + set(Calendar.MILLISECOND, 0) + }.timeInMillis + + d.startsWith("today") -> Calendar.getInstance().apply { + set(Calendar.HOUR_OF_DAY, 0) + set(Calendar.MINUTE, 0) + set(Calendar.SECOND, 0) + set(Calendar.MILLISECOND, 0) + }.timeInMillis + + date.contains(Regex("""\d(st|nd|rd|th)""")) -> date.split(" ").map { + if (it.contains(Regex("""\d\D\D"""))) { + it.replace(Regex("""\D"""), "") + } else { + it + } + }.let { dateFormat.tryParse(it.joinToString(" ")) } + + else -> dateFormat.tryParse(date) + } + } + + // Parses dates in this form: + // 21 hours ago + private fun parseRelativeDate(date: String): Long { + val number = Regex("""(\d+)""").find(date)?.value?.toIntOrNull() ?: return 0 + val cal = Calendar.getInstance() + + return when { + WordSet( + "day", + "days", + ).anyWordIn(date) -> cal.apply { add(Calendar.DAY_OF_MONTH, -number) }.timeInMillis + + WordSet("hour", "hours", "h").anyWordIn(date) -> cal.apply { + add( + Calendar.HOUR, + -number, + ) + }.timeInMillis + + WordSet( + "min", + "minute", + "minutes", + ).anyWordIn(date) -> cal.apply { + add( + Calendar.MINUTE, + -number, + ) + }.timeInMillis + + WordSet("second").anyWordIn(date) -> cal.apply { + add( + Calendar.SECOND, + -number, + ) + }.timeInMillis + + WordSet("month", "months").anyWordIn(date) -> cal.apply { add(Calendar.MONTH, -number) }.timeInMillis + WordSet("year").anyWordIn(date) -> cal.apply { add(Calendar.YEAR, -number) }.timeInMillis + else -> 0 + } + } + +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madtheme/en/BeeHentai.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madtheme/en/BeeHentai.kt new file mode 100644 index 00000000..0d078865 --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madtheme/en/BeeHentai.kt @@ -0,0 +1,16 @@ +package org.koitharu.kotatsu.parsers.site.madtheme.en + + +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.model.ContentType +import org.koitharu.kotatsu.parsers.model.MangaSource +import org.koitharu.kotatsu.parsers.site.madtheme.MadthemeParser + + +@MangaSourceParser("BEEHENTAI", "BeeHentai", "en", ContentType.HENTAI) +internal class BeeHentai(context: MangaLoaderContext) : + MadthemeParser(context, MangaSource.BEEHENTAI, "beehentai.com") { + + override val selectDesc = "div.section-body" +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madtheme/en/MangaBuddy.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madtheme/en/MangaBuddy.kt new file mode 100644 index 00000000..3a1d2022 --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madtheme/en/MangaBuddy.kt @@ -0,0 +1,12 @@ +package org.koitharu.kotatsu.parsers.site.madtheme.en + + +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.model.MangaSource +import org.koitharu.kotatsu.parsers.site.madtheme.MadthemeParser + + +@MangaSourceParser("MANGABUDDY", "MangaBuddy", "en") +internal class MangaBuddy(context: MangaLoaderContext) : + MadthemeParser(context, MangaSource.MANGABUDDY, "mangabuddy.com") diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madtheme/en/MangaCute.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madtheme/en/MangaCute.kt new file mode 100644 index 00000000..25dd8935 --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madtheme/en/MangaCute.kt @@ -0,0 +1,12 @@ +package org.koitharu.kotatsu.parsers.site.madtheme.en + + +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.model.MangaSource +import org.koitharu.kotatsu.parsers.site.madtheme.MadthemeParser + + +@MangaSourceParser("MANGACUTE", "MangaCute", "en") +internal class MangaCute(context: MangaLoaderContext) : + MadthemeParser(context, MangaSource.MANGACUTE, "mangacute.com") diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madtheme/en/MangaForest.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madtheme/en/MangaForest.kt new file mode 100644 index 00000000..3cb28d55 --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madtheme/en/MangaForest.kt @@ -0,0 +1,12 @@ +package org.koitharu.kotatsu.parsers.site.madtheme.en + + +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.model.MangaSource +import org.koitharu.kotatsu.parsers.site.madtheme.MadthemeParser + + +@MangaSourceParser("MANGAFOREST", "Manga Forest", "en") +internal class MangaForest(context: MangaLoaderContext) : + MadthemeParser(context, MangaSource.MANGAFOREST, "mangaforest.me") diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madtheme/en/MangaPuma.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madtheme/en/MangaPuma.kt new file mode 100644 index 00000000..2d3b4bad --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madtheme/en/MangaPuma.kt @@ -0,0 +1,12 @@ +package org.koitharu.kotatsu.parsers.site.madtheme.en + + +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.model.MangaSource +import org.koitharu.kotatsu.parsers.site.madtheme.MadthemeParser + + +@MangaSourceParser("MANGAPUMA", "Manga Puma", "en") +internal class MangaPuma(context: MangaLoaderContext) : + MadthemeParser(context, MangaSource.MANGAPUMA, "mangapuma.com") diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madtheme/en/Mangaxyz.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madtheme/en/Mangaxyz.kt new file mode 100644 index 00000000..cc53998f --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madtheme/en/Mangaxyz.kt @@ -0,0 +1,12 @@ +package org.koitharu.kotatsu.parsers.site.madtheme.en + + +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.model.MangaSource +import org.koitharu.kotatsu.parsers.site.madtheme.MadthemeParser + + +@MangaSourceParser("MANGAXYZ", "Mangaxyz", "en") +internal class Mangaxyz(context: MangaLoaderContext) : + MadthemeParser(context, MangaSource.MANGAXYZ, "mangaxyz.com") diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madtheme/en/TooniTube.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madtheme/en/TooniTube.kt new file mode 100644 index 00000000..3e4eb64c --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madtheme/en/TooniTube.kt @@ -0,0 +1,16 @@ +package org.koitharu.kotatsu.parsers.site.madtheme.en + + +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.model.ContentType +import org.koitharu.kotatsu.parsers.model.MangaSource +import org.koitharu.kotatsu.parsers.site.madtheme.MadthemeParser + + +@MangaSourceParser("TOONITUBE", "TooniTube", "en", ContentType.HENTAI) +internal class TooniTube(context: MangaLoaderContext) : + MadthemeParser(context, MangaSource.TOONITUBE, "toonitube.com") { + + override val selectDesc = "div.summary div.section-body p.content" +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madtheme/en/ToonilyMe.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madtheme/en/ToonilyMe.kt new file mode 100644 index 00000000..4c1f74cc --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madtheme/en/ToonilyMe.kt @@ -0,0 +1,16 @@ +package org.koitharu.kotatsu.parsers.site.madtheme.en + + +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.model.ContentType +import org.koitharu.kotatsu.parsers.model.MangaSource +import org.koitharu.kotatsu.parsers.site.madtheme.MadthemeParser + + +@MangaSourceParser("TOONILY_ME", "Toonily Me", "en", ContentType.HENTAI) +internal class ToonilyMe(context: MangaLoaderContext) : + MadthemeParser(context, MangaSource.TOONILY_ME, "toonily.me") { + + override val selectDesc = "div.summary div.section-body p.content" +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madtheme/en/TrueManga.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madtheme/en/TrueManga.kt new file mode 100644 index 00000000..150f7218 --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madtheme/en/TrueManga.kt @@ -0,0 +1,12 @@ +package org.koitharu.kotatsu.parsers.site.madtheme.en + + +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.model.MangaSource +import org.koitharu.kotatsu.parsers.site.madtheme.MadthemeParser + + +@MangaSourceParser("TRUEMANGA", "True Manga", "en") +internal class TrueManga(context: MangaLoaderContext) : + MadthemeParser(context, MangaSource.TRUEMANGA, "truemanga.com") diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/manga18/Manga18Parser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/manga18/Manga18Parser.kt index a804baa2..8d99af62 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/manga18/Manga18Parser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/manga18/Manga18Parser.kt @@ -27,7 +27,7 @@ internal abstract class Manga18Parser( SortOrder.ALPHABETICAL, ) - protected open val listeurl = "list-manga/" + protected open val listUrl = "list-manga/" protected open val tagUrl = "manga-list/" protected open val datePattern = "dd-MM-yyyy" @@ -59,7 +59,7 @@ internal abstract class Manga18Parser( append(domain) when { !query.isNullOrEmpty() -> { - append("/$listeurl") + append("/$listUrl") append(page.toString()) append("?search=") append(query.urlEncoded()) @@ -77,7 +77,7 @@ internal abstract class Manga18Parser( } else -> { - append("/$listeurl") + append("/$listUrl") append(page.toString()) append("?") } @@ -112,7 +112,7 @@ internal abstract class Manga18Parser( } override suspend fun getTags(): Set { - val doc = webClient.httpGet("https://$domain/$listeurl/").parseHtml() + val doc = webClient.httpGet("https://$domain/$listUrl/").parseHtml() return doc.select("div.grid_cate li").mapNotNullToSet { li -> val a = li.selectFirst("a") ?: return@mapNotNullToSet null val href = a.attr("href").removeSuffix('/').substringAfterLast('/') diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/manga18/en/Hentai3zCc.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/manga18/en/Hentai3zCc.kt index fe77fddc..e74f6098 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/manga18/en/Hentai3zCc.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/manga18/en/Hentai3zCc.kt @@ -36,7 +36,7 @@ internal class Hentai3zCc(context: MangaLoaderContext) : val pages = page + 1 when { !query.isNullOrEmpty() -> { - append("/$listeurl/") + append("/$listUrl") append(pages.toString()) append("?search=") append(query.urlEncoded()) @@ -44,7 +44,7 @@ internal class Hentai3zCc(context: MangaLoaderContext) : } !tags.isNullOrEmpty() -> { - append("/$tagUrl/") + append("/$tagUrl") for (tag in tags) { append(tag.key) } @@ -54,7 +54,7 @@ internal class Hentai3zCc(context: MangaLoaderContext) : } else -> { - append("/$listeurl/") + append("/$listUrl") append(pages.toString()) append("?") }