From c8d33d06a182ecd8892eb28340dc1617a39ec551 Mon Sep 17 00:00:00 2001 From: devi Date: Thu, 27 Jul 2023 20:27:56 +0200 Subject: [PATCH] add fmreader --- .../parsers/site/fmreader/FmreaderParser.kt | 297 ++++++++++++++++++ .../parsers/site/fmreader/en/Manhwa18Com.kt | 95 ++++++ .../kotatsu/parsers/site/fmreader/ja/Klz9.kt | 141 +++++++++ .../parsers/site/fmreader/ja/WeLoveManga.kt | 69 ++++ .../parsers/site/fmreader/ja/Weloma.kt | 12 + 5 files changed, 614 insertions(+) create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/fmreader/FmreaderParser.kt create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/fmreader/en/Manhwa18Com.kt create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/fmreader/ja/Klz9.kt create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/fmreader/ja/WeLoveManga.kt create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/fmreader/ja/Weloma.kt diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/fmreader/FmreaderParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/fmreader/FmreaderParser.kt new file mode 100644 index 00000000..aa646806 --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/fmreader/FmreaderParser.kt @@ -0,0 +1,297 @@ +package org.koitharu.kotatsu.parsers.site.fmreader + +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope +import org.jsoup.nodes.Document +import org.jsoup.nodes.Element +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.PagedMangaParser +import org.koitharu.kotatsu.parsers.config.ConfigKey +import org.koitharu.kotatsu.parsers.model.* +import org.koitharu.kotatsu.parsers.util.* +import java.text.DateFormat +import java.text.SimpleDateFormat +import java.util.* + +internal abstract class FmreaderParser( + context: MangaLoaderContext, + source: MangaSource, + domain: String, + pageSize: Int = 20, +) : PagedMangaParser(context, source, pageSize) { + + override val configKeyDomain = ConfigKey.Domain(domain) + + override val sortOrders: Set = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.ALPHABETICAL, + ) + + protected open val listeurl = "/manga-list.html" + protected open val datePattern = "MMMM d, yyyy" + + + init { + paginator.firstPage = 1 + searchPaginator.firstPage = 1 + } + + + @JvmField + protected val ongoing: Set = setOf( + "On going", + "Incomplete", + ) + + @JvmField + protected val finished: Set = setOf( + "Completed", + ) + + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + val url = buildString { + append("https://") + append(domain) + append(listeurl) + append("?page=") + append(page.toString()) + when { + !query.isNullOrEmpty() -> { + + append("&name=") + append(query.urlEncoded()) + } + + !tags.isNullOrEmpty() -> { + append("&genre=") + for (tag in tags) { + append(tag.key) + } + } + } + + append("&sort=") + when (sortOrder) { + SortOrder.POPULARITY -> append("views") + SortOrder.UPDATED -> append("last_update") + SortOrder.ALPHABETICAL -> append("name") + else -> append("last_update") + } + } + val doc = webClient.httpGet(url).parseHtml() + + return doc.select("div.thumb-item-flow").map { div -> + + val href = div.selectFirstOrThrow("div.series-title").selectFirstOrThrow("a").attrAsRelativeUrl("href") + Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(div.host ?: domain), + coverUrl = div.selectFirstOrThrow("div.img-in-ratio").attr("style").substringAfter("('") + .substringBeforeLast("')"), + title = div.selectFirstOrThrow("div.series-title").text().orEmpty(), + altTitle = null, + rating = RATING_UNKNOWN, + tags = emptySet(), + author = null, + state = null, + source = source, + isNsfw = isNsfwSource, + ) + } + } + + override suspend fun getTags(): Set { + val doc = webClient.httpGet("https://$domain/$listeurl/").parseHtml() + return doc.select("ul.filter-type li").mapNotNullToSet { li -> + val a = li.selectFirst("a") ?: return@mapNotNullToSet null + val href = a.attr("href").substringAfter("manga-list-genre-").substringBeforeLast(".html") + MangaTag( + key = href, + title = a.text(), + source = source, + ) + } + } + + protected open val selectdesc = "div.summary-content" + protected open val selectState = "ul.manga-info li:contains(Status) a" + protected open val selectAlt = "ul.manga-info li:contains(Other names)" + protected open val selectAut = "ul.manga-info li:contains(Author(s)) a" + protected open val selectTag = "ul.manga-info li:contains(Genre(s)) a" + + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { + val fullUrl = manga.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + + val chaptersDeferred = async { getChapters(manga, doc) } + + val desc = doc.selectFirstOrThrow(selectdesc).html() + + val stateDiv = doc.selectFirst(selectState) + + val state = stateDiv?.let { + when (it.text()) { + in ongoing -> MangaState.ONGOING + in finished -> MangaState.FINISHED + else -> null + } + } + + val alt = doc.body().selectFirst(selectAlt)?.text()?.replace("Other names", "") + val auth = doc.body().selectFirst(selectAut)?.text() + manga.copy( + tags = doc.body().select(selectTag).mapNotNullToSet { a -> + MangaTag( + key = a.attr("href").substringAfter("manga-list-genre-").substringBeforeLast(".html"), + title = a.text().toTitleCase(), + source = source, + ) + }, + description = desc, + altTitle = alt, + author = auth, + state = state, + chapters = chaptersDeferred.await(), + ) + } + + + protected open val selectdate = "div.chapter-time" + protected open val selectchapter = "ul.list-chapters a" + + protected open suspend fun getChapters(manga: Manga, doc: Document): List { + val dateFormat = SimpleDateFormat(datePattern, sourceLocale) + return doc.body().select(selectchapter).mapChapters(reversed = true) { i, a -> + val href = a.attrAsRelativeUrl("href") + val dateText = a.selectFirst(selectdate)?.text() + MangaChapter( + id = generateUid(href), + name = a.selectFirstOrThrow("div.chapter-name").text(), + number = i + 1, + url = href, + uploadDate = parseChapterDate( + dateFormat, + dateText, + ), + source = source, + scanlator = null, + branch = null, + ) + } + } + + protected open val selectPage = "div.chapter-content img" + override suspend fun getPages(chapter: MangaChapter): List { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + + return doc.select(selectPage).map { img -> + val url = img.src()?.toRelativeUrl(domain) ?: img.parseFailed("Image src not found") + + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } + + + protected fun Element.src(): String? { + var result = absUrl("data-src") + if (result.isEmpty()) result = absUrl("data-original") + if (result.isEmpty()) result = absUrl("data-sizes") + if (result.isEmpty()) result = absUrl("src") + return result.ifEmpty { null } + } + + protected fun parseChapterDate(dateFormat: DateFormat, date: String?): Long { + // Clean date (e.g. 5th December 2019 to 5 December 2019) before parsing it + val d = date?.lowercase() ?: return 0 + return when { + d.endsWith(" ago") || + // short Hours + d.endsWith(" h") || + // short Day + d.endsWith(" d") -> parseRelativeDate(date) + + // Handle 'yesterday' and 'today', using midnight + d.startsWith("year") -> Calendar.getInstance().apply { + add(Calendar.DAY_OF_MONTH, -1) // yesterday + set(Calendar.HOUR_OF_DAY, 0) + set(Calendar.MINUTE, 0) + set(Calendar.SECOND, 0) + set(Calendar.MILLISECOND, 0) + }.timeInMillis + + d.startsWith("today") -> Calendar.getInstance().apply { + set(Calendar.HOUR_OF_DAY, 0) + set(Calendar.MINUTE, 0) + set(Calendar.SECOND, 0) + set(Calendar.MILLISECOND, 0) + }.timeInMillis + + date.contains(Regex("""\d(st|nd|rd|th)""")) -> date.split(" ").map { + if (it.contains(Regex("""\d\D\D"""))) { + it.replace(Regex("""\D"""), "") + } else { + it + } + }.let { dateFormat.tryParse(it.joinToString(" ")) } + + else -> dateFormat.tryParse(date) + } + } + + // Parses dates in this form: + // 21 hours ago + private fun parseRelativeDate(date: String): Long { + val number = Regex("""(\d+)""").find(date)?.value?.toIntOrNull() ?: return 0 + val cal = Calendar.getInstance() + + return when { + WordSet( + "day", + "days", + ).anyWordIn(date) -> cal.apply { add(Calendar.DAY_OF_MONTH, -number) }.timeInMillis + + WordSet("hour", "hours", "h").anyWordIn(date) -> cal.apply { + add( + Calendar.HOUR, + -number, + ) + }.timeInMillis + + WordSet( + "min", + "minute", + "minutes", + ).anyWordIn(date) -> cal.apply { + add( + Calendar.MINUTE, + -number, + ) + }.timeInMillis + + WordSet("second").anyWordIn(date) -> cal.apply { + add( + Calendar.SECOND, + -number, + ) + }.timeInMillis + + WordSet("month", "months").anyWordIn(date) -> cal.apply { add(Calendar.MONTH, -number) }.timeInMillis + WordSet("year").anyWordIn(date) -> cal.apply { add(Calendar.YEAR, -number) }.timeInMillis + else -> 0 + } + } + +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/fmreader/en/Manhwa18Com.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/fmreader/en/Manhwa18Com.kt new file mode 100644 index 00000000..d2b8dde1 --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/fmreader/en/Manhwa18Com.kt @@ -0,0 +1,95 @@ +package org.koitharu.kotatsu.parsers.site.fmreader.en + + +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope +import org.jsoup.nodes.Document +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.model.ContentType +import org.koitharu.kotatsu.parsers.model.Manga +import org.koitharu.kotatsu.parsers.model.MangaChapter +import org.koitharu.kotatsu.parsers.model.MangaSource +import org.koitharu.kotatsu.parsers.model.MangaState +import org.koitharu.kotatsu.parsers.model.MangaTag +import org.koitharu.kotatsu.parsers.site.fmreader.FmreaderParser +import org.koitharu.kotatsu.parsers.util.attrAsRelativeUrl +import org.koitharu.kotatsu.parsers.util.domain +import org.koitharu.kotatsu.parsers.util.generateUid +import org.koitharu.kotatsu.parsers.util.mapChapters +import org.koitharu.kotatsu.parsers.util.mapNotNullToSet +import org.koitharu.kotatsu.parsers.util.parseHtml +import org.koitharu.kotatsu.parsers.util.selectFirstOrThrow +import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl +import org.koitharu.kotatsu.parsers.util.toTitleCase +import java.text.SimpleDateFormat + +@MangaSourceParser("MANHWA18COM", "Manhwa18 Com", "en", ContentType.HENTAI) +internal class Manhwa18Com(context: MangaLoaderContext) : + FmreaderParser(context, MangaSource.MANHWA18COM, "manhwa18.com") { + + override val listeurl = "/tim-kiem" + + override val selectState = "div.info-item:contains(Status) span.info-value " + override val selectAlt = "div.info-item:contains(Other name) span.info-value " + override val selectTag = "div.info-item:contains(Genre) span.info-value a" + override val datePattern = "dd/MM/yyyy" + override val selectPage = "div#chapter-content img" + + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { + val fullUrl = manga.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + + val chaptersDeferred = async { getChapters(manga, doc) } + + val desc = doc.selectFirstOrThrow(selectdesc).html() + + val stateDiv = doc.selectFirst(selectState) + + val state = stateDiv?.let { + when (it.text()) { + in ongoing -> MangaState.ONGOING + in finished -> MangaState.FINISHED + else -> null + } + } + + val alt = doc.body().selectFirst(selectAlt)?.text()?.replace("Other name", "") + val auth = doc.body().selectFirst(selectAut)?.text() + manga.copy( + tags = doc.body().select(selectTag).mapNotNullToSet { a -> + MangaTag( + key = a.attr("href").substringAfter("manga-list-genre-").substringBeforeLast(".html"), + title = a.text().toTitleCase(), + source = source, + ) + }, + description = desc, + altTitle = alt, + author = auth, + state = state, + chapters = chaptersDeferred.await(), + ) + } + + override suspend fun getChapters(manga: Manga, doc: Document): List { + val dateFormat = SimpleDateFormat(datePattern, sourceLocale) + return doc.body().select(selectchapter).mapChapters(reversed = true) { i, a -> + val href = a.attrAsRelativeUrl("href") + val dateText = a.selectFirst(selectdate)?.text()?.substringAfter("- ") + MangaChapter( + id = generateUid(href), + name = a.selectFirstOrThrow("div.chapter-name").text(), + number = i + 1, + url = href, + uploadDate = parseChapterDate( + dateFormat, + dateText, + ), + source = source, + scanlator = null, + branch = null, + ) + } + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/fmreader/ja/Klz9.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/fmreader/ja/Klz9.kt new file mode 100644 index 00000000..44f17041 --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/fmreader/ja/Klz9.kt @@ -0,0 +1,141 @@ +package org.koitharu.kotatsu.parsers.site.fmreader.ja + + +import org.jsoup.nodes.Document +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.model.Manga +import org.koitharu.kotatsu.parsers.model.MangaChapter +import org.koitharu.kotatsu.parsers.model.MangaPage +import org.koitharu.kotatsu.parsers.model.MangaSource +import org.koitharu.kotatsu.parsers.model.MangaTag +import org.koitharu.kotatsu.parsers.model.RATING_UNKNOWN +import org.koitharu.kotatsu.parsers.model.SortOrder +import org.koitharu.kotatsu.parsers.site.fmreader.FmreaderParser +import org.koitharu.kotatsu.parsers.util.attrAsRelativeUrl +import org.koitharu.kotatsu.parsers.util.domain +import org.koitharu.kotatsu.parsers.util.generateUid +import org.koitharu.kotatsu.parsers.util.host +import org.koitharu.kotatsu.parsers.util.mapChapters +import org.koitharu.kotatsu.parsers.util.parseFailed +import org.koitharu.kotatsu.parsers.util.parseHtml +import org.koitharu.kotatsu.parsers.util.selectFirstOrThrow +import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl +import org.koitharu.kotatsu.parsers.util.toRelativeUrl +import org.koitharu.kotatsu.parsers.util.urlEncoded +import java.text.SimpleDateFormat + + +@MangaSourceParser("KLZ9", "Klz9", "ja") +internal class Klz9(context: MangaLoaderContext) : + FmreaderParser(context, MangaSource.KLZ9, "klz9.com") { + + override val selectdesc = "div.row:contains(Description)" + override val selectState = "ul.manga-info li:contains(Status) a" + override val selectAlt = "ul.manga-info li:contains(Other name (s))" + override val selectTag = "ul.manga-info li:contains(Genre(s)) a" + override val selectchapter = "tr" + override val selectdate = "td i" + override val selectPage = "img" + + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + val url = buildString { + append("https://") + append(domain) + append("/$listeurl") + append("?page=") + append(page.toString()) + when { + !query.isNullOrEmpty() -> { + + append("&name=") + append(query.urlEncoded()) + } + + !tags.isNullOrEmpty() -> { + append("&genre=") + for (tag in tags) { + append(tag.key) + } + } + } + + append("&sort=") + when (sortOrder) { + SortOrder.POPULARITY -> append("views") + SortOrder.UPDATED -> append("last_update") + SortOrder.ALPHABETICAL -> append("name") + else -> append("last_update") + } + } + val doc = webClient.httpGet(url).parseHtml() + + return doc.select("div.thumb-item-flow").map { div -> + val href = "/" + div.selectFirstOrThrow("a").attrAsRelativeUrl("href") + Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(div.host ?: domain), + coverUrl = div.selectFirstOrThrow("div.img-in-ratio").attr("style").substringAfter("('") + .substringBeforeLast("')"), + title = div.selectFirstOrThrow("div.series-title").text().orEmpty(), + altTitle = null, + rating = RATING_UNKNOWN, + tags = emptySet(), + author = null, + state = null, + source = source, + isNsfw = isNsfwSource, + ) + } + } + + override suspend fun getChapters(manga: Manga, doc: Document): List { + val slug = doc.selectFirstOrThrow("div.h0rating").attr("slug") + val docload = + webClient.httpGet("https://$domain/app/manga/controllers/cont.listChapter.php?slug=$slug").parseHtml() + val dateFormat = SimpleDateFormat(datePattern, sourceLocale) + return docload.body().select(selectchapter).mapChapters(reversed = true) { i, a -> + val href = "/" + a.selectFirstOrThrow("a.chapter").attrAsRelativeUrl("href") + val dateText = a.selectFirst(selectdate)?.text() + MangaChapter( + id = generateUid(href), + name = a.selectFirstOrThrow("a").text(), + number = i + 1, + url = href, + uploadDate = parseChapterDate( + dateFormat, + dateText, + ), + source = source, + scanlator = null, + branch = null, + ) + } + } + + override suspend fun getPages(chapter: MangaChapter): List { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + + val cid = doc.selectFirstOrThrow("#chapter").attr("value") + val docload = webClient.httpGet("https://$domain/app/manga/controllers/cont.listImg.php?cid=$cid").parseHtml() + + return docload.select(selectPage).map { img -> + val url = img.src()?.toRelativeUrl(domain) ?: img.parseFailed("Image src not found") + + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } + +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/fmreader/ja/WeLoveManga.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/fmreader/ja/WeLoveManga.kt new file mode 100644 index 00000000..2ad5ab65 --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/fmreader/ja/WeLoveManga.kt @@ -0,0 +1,69 @@ +package org.koitharu.kotatsu.parsers.site.fmreader.ja + + +import org.jsoup.nodes.Document +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.model.Manga +import org.koitharu.kotatsu.parsers.model.MangaChapter +import org.koitharu.kotatsu.parsers.model.MangaPage +import org.koitharu.kotatsu.parsers.model.MangaSource +import org.koitharu.kotatsu.parsers.site.fmreader.FmreaderParser +import org.koitharu.kotatsu.parsers.util.attrAsRelativeUrl +import org.koitharu.kotatsu.parsers.util.domain +import org.koitharu.kotatsu.parsers.util.generateUid +import org.koitharu.kotatsu.parsers.util.mapChapters +import org.koitharu.kotatsu.parsers.util.parseFailed +import org.koitharu.kotatsu.parsers.util.parseHtml +import org.koitharu.kotatsu.parsers.util.selectFirstOrThrow +import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl +import org.koitharu.kotatsu.parsers.util.toRelativeUrl +import java.text.SimpleDateFormat + + +@MangaSourceParser("WELOVEMANGA", "WeLoveManga", "ja") +internal class WeLoveManga(context: MangaLoaderContext) : + FmreaderParser(context, MangaSource.WELOVEMANGA, "welovemanga.one") { + + override suspend fun getChapters(manga: Manga, doc: Document): List { + val mid = doc.selectFirstOrThrow("div.cmt input").attr("value") + val docload = + webClient.httpGet("https://$domain/app/manga/controllers/cont.Listchapter.php?mid=$mid").parseHtml() + val dateFormat = SimpleDateFormat(datePattern, sourceLocale) + return docload.body().select(selectchapter).mapChapters(reversed = true) { i, a -> + val href = a.selectFirstOrThrow("a").attrAsRelativeUrl("href") + val dateText = a.selectFirst(selectdate)?.text() + MangaChapter( + id = generateUid(href), + name = a.selectFirstOrThrow("a").text(), + number = i + 1, + url = href, + uploadDate = parseChapterDate( + dateFormat, + dateText, + ), + source = source, + scanlator = null, + branch = null, + ) + } + } + + override suspend fun getPages(chapter: MangaChapter): List { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + + val cid = doc.selectFirstOrThrow("#chapter").attr("value") + val docload = webClient.httpGet("https://$domain/app/manga/controllers/cont.listImg.php?cid=$cid").parseHtml() + return docload.select("img").map { img -> + val url = img.src()?.toRelativeUrl(domain) ?: img.parseFailed("Image src not found") + + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/fmreader/ja/Weloma.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/fmreader/ja/Weloma.kt new file mode 100644 index 00000000..439fe210 --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/fmreader/ja/Weloma.kt @@ -0,0 +1,12 @@ +package org.koitharu.kotatsu.parsers.site.fmreader.ja + + +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.model.MangaSource +import org.koitharu.kotatsu.parsers.site.fmreader.FmreaderParser + + +@MangaSourceParser("WELOMA", "Weloma", "ja") +internal class Weloma(context: MangaLoaderContext) : + FmreaderParser(context, MangaSource.WELOMA, "weloma.art")