diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaParser.kt index 4a05b3d0f..7ae0228c2 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaParser.kt @@ -20,6 +20,9 @@ abstract class MangaParser @InternalParsersApi constructor(val source: MangaSour val config by lazy { context.getConfig(source) } + val sourceLocale: Locale? + get() = source.locale?.let { Locale(it) } + /** * Provide default domain and available alternatives, if any. * @@ -160,4 +163,13 @@ abstract class MangaParser @InternalParsersApi constructor(val source: MangaSour protected fun parseFailed(message: String? = null): Nothing { throw ParseException(message, null) } + + @InternalParsersApi + protected fun Set?.oneOrThrowIfMany(): MangaTag? { + return when { + isNullOrEmpty() -> null + size == 1 -> first() + else -> throw IllegalArgumentException("Multiple genres are not supported by this source") + } + } } \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/Madara5Parser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/Madara5Parser.kt new file mode 100644 index 000000000..837cfccb4 --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/Madara5Parser.kt @@ -0,0 +1,178 @@ +package org.koitharu.kotatsu.parsers.site + +import androidx.collection.arraySetOf +import org.jsoup.nodes.Element +import org.koitharu.kotatsu.parsers.InternalParsersApi +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaParser +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.config.ConfigKey +import org.koitharu.kotatsu.parsers.model.* +import org.koitharu.kotatsu.parsers.util.* +import java.text.SimpleDateFormat +import java.util.* + +abstract class Madara5Parser @InternalParsersApi constructor( + override val context: MangaLoaderContext, + source: MangaSource, + domain: String, +) : MangaParser(source) { + + protected open val pageSize = 22 + protected open val tagPrefix = "/mangas/" + protected open val nsfwTags = arraySetOf("yaoi", "yuri", "mature") + + override val sortOrders: Set = EnumSet.of(SortOrder.UPDATED) + + override val configKeyDomain = ConfigKey.Domain(domain, null) + + @InternalParsersApi + override suspend fun getList(offset: Int, query: String?, tags: Set?, sortOrder: SortOrder): List { + val page = (offset / pageSize.toFloat()).toIntUp() + val domain = getDomain() + val url = buildString { + append("https://") + append(domain) + append("/search?s=") + if (!query.isNullOrEmpty()) { + append(query.urlEncoded()) + } + append("&post_type=wp-manga") + if (!tags.isNullOrEmpty()) { + for (tag in tags) { + append("&genre%5B%5D=") + append(tag.key) + } + } + append("&op=1&author=&artist=&page=") + append(page) + } + val root = context.httpGet(url).parseHtml().body().selectFirstOrThrow(".search-wrap") + return root.select(".c-tabs-item__content").map { div -> + val a = div.selectFirstOrThrow("a") + val img = div.selectLastOrThrow("img") + val href = a.attrAsRelativeUrl("href") + val postContent = root.selectFirstOrThrow(".post-content") + val tagSet = postContent.getElementsContainingOwnText("Genre") + .firstOrNull()?.tableValue() + ?.getElementsByAttributeValueContaining("href", tagPrefix) + ?.mapToSet { it.asMangaTag() }.orEmpty() + Manga( + id = generateUid(href), + title = a.attr("title"), + altTitle = postContent.getElementsContainingOwnText("Alternative") + .firstOrNull()?.tableValue()?.text()?.trim(), + url = href, + publicUrl = a.attrAsAbsoluteUrl("href"), + coverUrl = img.attrAsAbsoluteUrl("src"), + author = postContent.getElementsContainingOwnText("Author") + .firstOrNull()?.tableValue()?.text()?.trim(), + state = postContent.getElementsContainingOwnText("Status") + .firstOrNull()?.tableValue()?.text()?.asMangaState(), + isNsfw = isNsfw(tagSet), + rating = div.selectFirstOrThrow(".score").text() + .toFloatOrNull()?.div(5f) ?: RATING_UNKNOWN, + tags = tagSet, + source = source, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val root = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml().body() + .selectFirstOrThrow(".site-content") + val postContent = root.selectFirstOrThrow(".post-content") + val tags = postContent.getElementsContainingOwnText("Genre") + .firstOrNull()?.tableValue() + ?.getElementsByAttributeValueContaining("href", tagPrefix) + ?.mapToSet { a -> a.asMangaTag() } ?: manga.tags + val mangaId = root.getElementById("manga-chapters-holder")?.attr("data-id")?.toLongOrNull() + ?: parseFailed("Cannot find mangaId") + return manga.copy( + description = (root.selectFirst(".detail-content") + ?: root.selectFirstOrThrow(".description-summary")).html(), + author = postContent.getElementsContainingOwnText("Author") + .firstOrNull()?.tableValue()?.text()?.trim(), + state = postContent.getElementsContainingOwnText("Status") + .firstOrNull()?.tableValue()?.text()?.asMangaState(), + tags = tags, + isNsfw = isNsfw(tags), + chapters = loadChapters(mangaId), + ) + } + + override suspend fun getPages(chapter: MangaChapter): List { + val fullUrl = chapter.url.toAbsoluteUrl(getDomain()) + val doc = context.httpGet(fullUrl).parseHtml() + val arrayData = doc.getElementById("arraydata") ?: parseFailed("#arraydata not found") + return arrayData.html().split(',').map { url -> + MangaPage( + id = generateUid(url), + url = url, + referer = fullUrl, + preview = null, + source = source, + ) + } + } + + override suspend fun getTags(): Set { + val doc = context.httpGet("http://${getDomain()}/").parseHtml().body() + return doc.getElementsByAttributeValueContaining("href", tagPrefix) + .mapToSet { it.asMangaTag() } + } + + private suspend fun loadChapters(mangaId: Long): List { + val dateFormat = SimpleDateFormat("MMMM dd, HH:mm", sourceLocale ?: Locale.US) + val doc = context.httpGet("https://${getDomain()}/ajax-list-chapter?mangaID=$mangaId").parseHtml() + return doc.select("li.wp-manga-chapter").asReversed().mapIndexed { i, li -> + val a = li.selectFirstOrThrow("a") + val href = a.attrAsRelativeUrl("href") + MangaChapter( + id = generateUid(href), + url = href, + name = a.text(), + number = i + 1, + branch = null, + uploadDate = dateFormat.tryParse( + li.selectFirst(".chapter-release-date")?.text()?.trim(), + ), + scanlator = null, + source = source, + ) + } + } + + protected fun isNsfw(tags: Set): Boolean { + return tags.any { it.key in nsfwTags } + } + + private fun Element.tableValue(): Element { + for (p in parents()) { + val children = p.children() + if (children.size == 2) { + return children[1] + } + } + parseFailed("Cannot find tableValue for node ${text()}") + } + + private fun String.asMangaState() = when (trim().lowercase(sourceLocale ?: Locale.US)) { + "ongoing" -> MangaState.ONGOING + "completed" -> MangaState.FINISHED + else -> null + } + + private fun Element.asMangaTag() = MangaTag( + title = ownText(), + key = attr("href").removeSuffix('/').substringAfterLast('/') + .replace('-', '+'), + source = source, + ) + + @MangaSourceParser("MANGAOWLS", "MangaOwls", "en") + class MangaOwls(context: MangaLoaderContext) : Madara5Parser(context, MangaSource.MANGAOWLS, "mangaowls.xyz") { + + override fun getFaviconUrl() = "http://${getDomain()}/frontend/images/favico.png" + } +} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MadaraParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MadaraParser.kt index 8f382293a..5b79f2053 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MadaraParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MadaraParser.kt @@ -36,11 +36,7 @@ internal abstract class MadaraParser( tags: Set?, sortOrder: SortOrder, ): List { - val tag = when { - tags.isNullOrEmpty() -> null - tags.size == 1 -> tags.first() - else -> throw IllegalArgumentException("Multiple genres are not supported by this source") - } + val tag = tags.oneOrThrowIfMany() val payload = createRequestTemplate() payload["page"] = (offset / PAGE_SIZE.toFloat()).toIntUp().toString() payload["vars[meta_key]"] = when (sortOrder) { diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Jsoup.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Jsoup.kt index 8eb920558..a88de5197 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Jsoup.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Jsoup.kt @@ -100,4 +100,8 @@ fun Element.requireElementById(id: String): Element { fun Element.selectLast(cssQuery: String): Element? { return select(cssQuery).lastOrNull() +} + +fun Element.selectLastOrThrow(cssQuery: String): Element { + return selectLast(cssQuery) ?: throw ParseException("Cannot find \"$cssQuery\"") } \ No newline at end of file diff --git a/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaLoaderContextMock.kt b/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaLoaderContextMock.kt index 60357aa43..cd687e6a1 100644 --- a/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaLoaderContextMock.kt +++ b/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaLoaderContextMock.kt @@ -8,6 +8,8 @@ import org.koitharu.kotatsu.parsers.config.MangaSourceConfig import org.koitharu.kotatsu.parsers.model.MangaSource import org.koitharu.kotatsu.parsers.util.await import java.util.concurrent.TimeUnit +import javax.net.ssl.HostnameVerifier +import javax.net.ssl.SSLSession internal class MangaLoaderContextMock : MangaLoaderContext() { @@ -26,6 +28,7 @@ internal class MangaLoaderContextMock : MangaLoaderContext() { .cookieJar(cookieJar) .addInterceptor(UserAgentInterceptor(userAgent)) .addInterceptor(CloudFlareInterceptor()) + .hostnameVerifier(ConsumeAllVerifier()) .connectTimeout(20, TimeUnit.SECONDS) .readTimeout(60, TimeUnit.SECONDS) .writeTimeout(20, TimeUnit.SECONDS) @@ -61,4 +64,10 @@ internal class MangaLoaderContextMock : MangaLoaderContext() { cookieJar.loadFromStream(it) } ?: println("No cookies loaded!") } + + private class ConsumeAllVerifier : HostnameVerifier { + override fun verify(hostname: String?, session: SSLSession?): Boolean { + return true + } + } } \ No newline at end of file