From b7e6ca8a26092620ac7c7bf243824b7771d1aef1 Mon Sep 17 00:00:00 2001 From: ObserverOfTime Date: Thu, 18 Jan 2024 16:47:56 +0200 Subject: [PATCH] Add MangAdventure sources - Arc-Relight - Assorted Scans --- .../site/foolslide/en/AssortedScans.kt | 125 ----------- .../site/mangadventure/MangAdventureParser.kt | 207 ++++++++++++++++++ .../site/mangadventure/en/ArcRelight.kt | 24 ++ .../site/mangadventure/en/AssortedScans.kt | 20 ++ 4 files changed, 251 insertions(+), 125 deletions(-) delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/foolslide/en/AssortedScans.kt create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangadventure/MangAdventureParser.kt create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangadventure/en/ArcRelight.kt create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangadventure/en/AssortedScans.kt diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/foolslide/en/AssortedScans.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/foolslide/en/AssortedScans.kt deleted file mode 100644 index 4dbf8916..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/foolslide/en/AssortedScans.kt +++ /dev/null @@ -1,125 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.foolslide.en - -import kotlinx.coroutines.coroutineScope -import org.jsoup.nodes.Document -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.site.foolslide.FoolSlideParser -import org.koitharu.kotatsu.parsers.util.* - -@MangaSourceParser("ASSORTEDSCANS", "AssortedScans", "en") -internal class AssortedScans(context: MangaLoaderContext) : - FoolSlideParser(context, MangaSource.ASSORTEDSCANS, "assortedscans.com", 56) { - - override val listUrl = "reader/" - override val pagination = false - override val selectInfo = "div.#series-info" - - override suspend fun getListPage(page: Int, filter: MangaListFilter?): List { - if (page > 1) { - return emptyList() - } - - val url = buildString { - append("https://") - append(domain) - append('/') - when (filter) { - is MangaListFilter.Search -> { - append(searchUrl) - append("?q=") - append(filter.query.urlEncoded()) - } - - is MangaListFilter.Advanced -> { - append(listUrl) - } - - null -> append(listUrl) - } - } - val doc = webClient.httpGet(url).parseHtml() - return doc.select("section.series, tr.result").map { div -> - val href = div.selectFirstOrThrow("a").attrAsRelativeUrl("href") - Manga( - id = generateUid(href), - url = href, - publicUrl = href.toAbsoluteUrl(div.host ?: domain), - coverUrl = div.selectFirst("img")?.src().orEmpty(),// in search no img - title = div.selectFirstOrThrow("a").text().orEmpty(), - altTitle = null, - rating = RATING_UNKNOWN, - tags = emptySet(), - author = null, - state = null, - source = source, - isNsfw = isNsfwSource, - ) - } - } - - override suspend fun getDetails(manga: Manga): Manga = coroutineScope { - val fullUrl = manga.url.toAbsoluteUrl(domain) - val testAdultPage = webClient.httpGet(fullUrl).parseHtml() - val doc = if (testAdultPage.selectFirst("div.info form") != null) { - webClient.httpPost(fullUrl, "adult=true").parseHtml() - } else { - testAdultPage - } - val chapters = getChapters(doc) - val desc = doc.getElementById("series-desc")?.selectFirst("div")?.html() - val alt = doc.getElementById("series-aliases")?.selectFirst("div.alias")?.text() - val author = doc.getElementById("series-authors")?.selectFirst("div.author")?.text() - val state = doc.getElementById("series-status")?.selectFirst("span")?.text() - manga.copy( - coverUrl = doc.selectFirst(".cover")?.src() ?: manga.coverUrl, - description = desc, - altTitle = alt, - author = author, - state = when (state) { - "Ongoing" -> MangaState.ONGOING - "Completed" -> MangaState.FINISHED - "Canceled" -> MangaState.ABANDONED - else -> null - }, - chapters = chapters, - ) - } - - override suspend fun getChapters(doc: Document): List { - return doc.body().select("div.chapter").mapChapters(reversed = true) { i, div -> - val a = div.selectFirstOrThrow("a") - val href = a.attrAsRelativeUrl("href") - MangaChapter( - id = generateUid(href), - name = a.text(), - number = i + 1, - url = href, - uploadDate = 0, - source = source, - scanlator = null, - branch = null, - ) - } - } - - override suspend fun getPages(chapter: MangaChapter): List { - val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() - return doc.body().select(".page-list .dropdown-list li a").map { a -> - val url = a.attr("href").toRelativeUrl(domain) - MangaPage( - id = generateUid(url), - url = url, - preview = null, - source = source, - ) - } - } - - override suspend fun getPageUrl(page: MangaPage): String { - val doc = webClient.httpGet(page.url.toAbsoluteUrl(domain)).parseHtml() - val root = doc.body() - return root.requireElementById("page-image").attr("src") ?: doc.parseFailed("Page image not found") - } -} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangadventure/MangAdventureParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangadventure/MangAdventureParser.kt new file mode 100644 index 00000000..13f0b0ed --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangadventure/MangAdventureParser.kt @@ -0,0 +1,207 @@ +package org.koitharu.kotatsu.parsers.site.mangadventure + +import okhttp3.Headers +import okhttp3.HttpUrl +import org.json.JSONObject +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.PagedMangaParser +import org.koitharu.kotatsu.parsers.config.ConfigKey +import org.koitharu.kotatsu.parsers.exception.NotFoundException +import org.koitharu.kotatsu.parsers.model.* +import org.koitharu.kotatsu.parsers.network.UserAgents +import org.koitharu.kotatsu.parsers.util.* +import org.koitharu.kotatsu.parsers.util.json.* +import java.util.EnumSet + +internal abstract class MangAdventureParser( + context: MangaLoaderContext, + source: MangaSource, + domain: String, + pageSize: Int = 25 +) : PagedMangaParser(context, source, pageSize) { + override val configKeyDomain = ConfigKey.Domain(domain) + + override val headers: Headers = + Headers.Builder().add("User-Agent", UserAgents.KOTATSU).build() + + override val availableStates: Set = EnumSet.of( + MangaState.ONGOING, + MangaState.FINISHED, + MangaState.ABANDONED, + MangaState.PAUSED + ) + + override val availableContentRating: Set = + EnumSet.of(ContentRating.SAFE) + + override val availableSortOrders: Set = EnumSet.of( + SortOrder.ALPHABETICAL, + SortOrder.ALPHABETICAL_DESC, + SortOrder.UPDATED, + SortOrder.POPULARITY + ) + + override val defaultSortOrder = SortOrder.ALPHABETICAL + + override val isTagsExclusionSupported = true + + override suspend fun getListPage(page: Int, filter: MangaListFilter?): List { + val url = apiUrl.addEncodedPathSegment("series") + .addEncodedQueryParameter("limit", pageSize.toString()) + .addEncodedQueryParameter("page", page.toString()) + when (filter) { + is MangaListFilter.Search -> { + url.addQueryParameter("title", filter.query) + } + is MangaListFilter.Advanced -> { + url.addQueryParameter( + "categories", + buildString { + if (filter.tags.isNotEmpty() && filter.tagsExclude.isNotEmpty()) { + filter.tags.joinTo(this, ",", postfix = ",") { it.key } + filter.tagsExclude.joinTo(this, ",") { "-" + it.key } + } else if (filter.tags.isNotEmpty()) { + filter.tags.joinTo(this, ",") { it.key } + } else if (filter.tagsExclude.isNotEmpty()) { + filter.tagsExclude.joinTo(this, ",") { "-" + it.key } + } + } + ) + when (filter.states.oneOrThrowIfMany()) { + null -> url.addEncodedQueryParameter("status", "any") + MangaState.ONGOING -> url.addEncodedQueryParameter("status", "ongoing") + MangaState.FINISHED -> url.addEncodedQueryParameter("status", "completed") + MangaState.ABANDONED -> url.addEncodedQueryParameter("status", "canceled") + MangaState.PAUSED -> url.addEncodedQueryParameter("status", "hiatus") + else -> throw IllegalArgumentException(ERROR_UNSUPPORTED_STATE) + } + when (filter.sortOrder) { + SortOrder.ALPHABETICAL -> url.addEncodedQueryParameter("sort", "title") + SortOrder.ALPHABETICAL_DESC -> url.addEncodedQueryParameter("sort", "-title") + SortOrder.UPDATED -> url.addEncodedQueryParameter("sort", "-latest_upload") + SortOrder.POPULARITY -> url.addEncodedQueryParameter("sort", "-views") + else -> throw IllegalArgumentException(ERROR_UNSUPPORTED_SORT_ORDER) + } + } + else -> {} + } + return runCatchingCancellable { getManga(url.get()) }.getOrElse { + if (it is NotFoundException) emptyList() else throw it + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val url = apiUrl.addEncodedPathSegment("series").addPathSegment(manga.slug) + val details = requireNotNull(url.get()) + val chapters = url.addEncodedPathSegment("chapters") + .addEncodedQueryParameter("date_format", "timestamp").get() + return manga.copy( + description = details.getStringOrNull("description"), + altTitle = details.getJSONArray("aliases").joinToString(), + author = buildString { + val authors = details.getJSONArray("authors") + val artists = details.getJSONArray("artists") + if (authors.length() > 0 && artists.length() > 0) { + authors.joinTo(this, postfix = ", ") + artists.joinTo(this) + } else if (authors.length() > 0) { + authors.joinTo(this) + } else if (artists.length() > 0) { + artists.joinTo(this) + } + }, + tags = details.getJSONArray("categories").mapTo(HashSet()) { + val name = it as String + MangaTag(name, name, source) + }, + state = when (details.getString("status")) { + "ongoing" -> MangaState.ONGOING + "completed" -> MangaState.FINISHED + "canceled" -> MangaState.ABANDONED + "hiatus" -> MangaState.PAUSED + else -> null + }, + chapters = chapters?.optJSONArray("results")?.mapJSON { + val number = it.getFloat("number") + MangaChapter( + id = it.getLong("id"), + name = it.getString("full_title"), + number = number.toInt(), + volume = it.getIntOrDefault("volume", 0), + url = it.getString("url"), + scanlator = it.getJSONArray("groups").joinToString(), + uploadDate = it.getString("published").toLong(), + branch = null, + source = source + ) + } ?: emptyList() + ) + } + + override suspend fun getPages(chapter: MangaChapter): List { + val url = apiUrl.addEncodedPathSegment("chapters") + .addEncodedPathSegment(chapter.id.toString()) + .addEncodedPathSegment("pages") + .addEncodedQueryParameter("track", "true") + return url.get()?.optJSONArray("results")?.mapJSON { + MangaPage(it.getLong("id"), it.getString("image"), null, source) + } ?: emptyList() + } + + override suspend fun getPageUrl(page: MangaPage) = page.url + + override suspend fun getAvailableTags(): Set { + val url = apiUrl.addEncodedPathSegment("categories") + return url.get()?.optJSONArray("results")?.mapJSONToSet { + val name = it.getString("name") + MangaTag(name, name, source) + } ?: emptySet() + } + + // webp favicons are not supported + override suspend fun getFavicons() = + Favicons(listOf(Favicon("https://$domain/media/logo.png", 512, "")), domain) + + /* Get a list of manga from the given [JSONObject]. */ + protected fun getManga(json: JSONObject?): List { + return json?.optJSONArray("results")?.mapJSONNotNull { + // exclude licensed series + if (it.opt("chapters") == JSONObject.NULL) + return@mapJSONNotNull null + val path = it.getString("url") + val publicUrl = urlBuilder().addEncodedPathSegments(path).toString() + Manga( + id = generateUid(it.getString("slug")), + title = it.getString("title"), + altTitle = null, + url = path, + publicUrl = publicUrl, + rating = RATING_UNKNOWN, + isNsfw = false, + coverUrl = it.getString("cover"), + tags = emptySet(), + state = null, + author = null, + source = source + ) + } ?: emptyList() + } + + protected val apiUrl: HttpUrl.Builder + get() = urlBuilder().addEncodedPathSegments("api/v2") + + // /reader/{slug}/ + private inline val Manga.slug: String + get() = url.substring(8, url.length - 1) + + protected suspend inline fun HttpUrl.Builder.get() = + webClient.httpGet(build()).body?.string()?.let(::JSONObject) + + private companion object { + private const val ERROR_UNSUPPORTED_STATE = + "The selected state is not supported by this source" + + private const val ERROR_UNSUPPORTED_SORT_ORDER = + "The selected sort order is not supported by this source" + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangadventure/en/ArcRelight.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangadventure/en/ArcRelight.kt new file mode 100644 index 00000000..6af81ddd --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangadventure/en/ArcRelight.kt @@ -0,0 +1,24 @@ +package org.koitharu.kotatsu.parsers.site.mangadventure.en + +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.model.Manga +import org.koitharu.kotatsu.parsers.model.MangaSource +import org.koitharu.kotatsu.parsers.site.mangadventure.MangAdventureParser +import org.koitharu.kotatsu.parsers.util.urlEncoded + +@MangaSourceParser("ARCRELIGHT", "Arc-Relight", "en") +internal class ArcRelight(context: MangaLoaderContext) : + MangAdventureParser(context, MangaSource.ARCRELIGHT, "arc-relight.com", pageSize = 10) { + private val franchises = setOf("Jubilee", "Steins;Gate", "Robotics;Notes", "Chaos;Head") + + override suspend fun getRelatedManga(seed: Manga): List { + val tags = franchises.filter { franchise -> + seed.tags.find { it.key == franchise } != null + } + if (tags.isEmpty()) return emptyList() + val url = apiUrl.addEncodedPathSegment("series") + .addQueryParameter("categories", tags.joinToString(",")) + return getManga(url.get()) + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangadventure/en/AssortedScans.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangadventure/en/AssortedScans.kt new file mode 100644 index 00000000..df93bb83 --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangadventure/en/AssortedScans.kt @@ -0,0 +1,20 @@ +package org.koitharu.kotatsu.parsers.site.mangadventure.en + +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.model.MangaSource +import org.koitharu.kotatsu.parsers.model.MangaTag +import org.koitharu.kotatsu.parsers.site.mangadventure.MangAdventureParser + +@MangaSourceParser("ASSORTEDSCANS", "Assorted Scans", "en") +internal class AssortedScans(context: MangaLoaderContext) : + MangAdventureParser(context, MangaSource.ASSORTEDSCANS, "assortedscans.com") { + // tags that don't have any series and make the tests fail + private val emptyTags = setOf( + "Doujinshi", "Harem", "Hentai", "Mecha", + "Shoujo Ai", "Shounen Ai", "Smut", "Yaoi" + ) + + override suspend fun getAvailableTags(): Set = + super.getAvailableTags().filterTo(HashSet()) { it.key !in emptyTags } +}