From a10c1101ab12a1a6f8c6720e8644914419ad3c16 Mon Sep 17 00:00:00 2001 From: Koitharu Date: Thu, 14 Mar 2024 16:58:33 +0200 Subject: [PATCH] New source: MangaOVH --- .gitignore | 1 + .../kotatsu/parsers/PagedMangaParser.kt | 5 +- .../kotatsu/parsers/site/ru/MangaOvhParser.kt | 243 ++++++++++++++++++ .../kotatsu/parsers/site/ru/RemangaParser.kt | 2 +- .../kotatsu/parsers/util/MangaParserEnv.kt | 4 +- .../koitharu/kotatsu/parsers/util/String.kt | 2 + .../org/koitharu/kotatsu/test_util/Util.kt | 2 +- 7 files changed, 253 insertions(+), 6 deletions(-) create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/ru/MangaOvhParser.kt diff --git a/.gitignore b/.gitignore index 63a8a581..6bed2a8e 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,7 @@ .idea/**/dynamic.xml .idea/**/uiDesigner.xml .idea/**/dbnavigator.xml +.idea/**/Project_Default.xml # Gradle .idea/**/gradle.xml diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/PagedMangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/PagedMangaParser.kt index 6d638a2a..5868bb8f 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/PagedMangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/PagedMangaParser.kt @@ -1,6 +1,6 @@ package org.koitharu.kotatsu.parsers -import androidx.annotation.RestrictTo +import androidx.annotation.VisibleForTesting import org.koitharu.kotatsu.parsers.model.* import org.koitharu.kotatsu.parsers.util.Paginator @@ -8,7 +8,7 @@ import org.koitharu.kotatsu.parsers.util.Paginator abstract class PagedMangaParser( context: MangaLoaderContext, source: MangaSource, - @RestrictTo(RestrictTo.Scope.TESTS) @JvmField internal val pageSize: Int, + @VisibleForTesting(otherwise = VisibleForTesting.PROTECTED) @JvmField internal val pageSize: Int, searchPageSize: Int = pageSize, ) : MangaParser(context, source) { @@ -40,6 +40,7 @@ abstract class PagedMangaParser( sortOrder: SortOrder, ): List = throw UnsupportedOperationException("You should use getListPage for PagedMangaParser") + @Deprecated("") open suspend fun getListPage( page: Int, query: String?, diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ru/MangaOvhParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ru/MangaOvhParser.kt new file mode 100644 index 00000000..81eeeb35 --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ru/MangaOvhParser.kt @@ -0,0 +1,243 @@ +package org.koitharu.kotatsu.parsers.site.ru + +import androidx.collection.ArrayMap +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope +import org.json.JSONObject +import org.koitharu.kotatsu.parsers.InternalParsersApi +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.PagedMangaParser +import org.koitharu.kotatsu.parsers.config.ConfigKey +import org.koitharu.kotatsu.parsers.model.* +import org.koitharu.kotatsu.parsers.util.* +import org.koitharu.kotatsu.parsers.util.json.* +import java.text.SimpleDateFormat +import java.util.* + +@MangaSourceParser("MANGA_OVH", "МангаОВХ", "ru") +class MangaOvhParser(context: MangaLoaderContext) : PagedMangaParser(context, MangaSource.MANGA_OVH, pageSize = 20) { + + override val availableSortOrders: Set = EnumSet.of( + SortOrder.POPULARITY, + SortOrder.RATING, + SortOrder.UPDATED, + SortOrder.NEWEST, + ) + + @InternalParsersApi + override val configKeyDomain = ConfigKey.Domain("manga.ovh") + + override val isTagsExclusionSupported = true + + override val availableStates: Set = EnumSet.of( + MangaState.UPCOMING, + MangaState.PAUSED, + MangaState.ONGOING, + MangaState.FINISHED, + ) + + override val availableContentRating: Set = EnumSet.allOf(ContentRating::class.java) + + init { + paginator.firstPage = 0 + searchPaginator.firstPage = 0 + } + + override suspend fun getListPage(page: Int, filter: MangaListFilter?): List { + val url = urlBuilder("api") + .addPathSegment("v2") + .addPathSegment("books") + .addQueryParameter("page", page.toString()) + .addQueryParameter("size", pageSize.toString()) + .addQueryParameter("type", "COMIC") + when (filter) { + is MangaListFilter.Advanced -> { + url.addQueryParameter( + "sort", + when (filter.sortOrder) { + SortOrder.UPDATED -> "updatedAt,desc" + SortOrder.POPULARITY -> "viewsCount,desc" + SortOrder.RATING -> "likesCount,desc" + SortOrder.NEWEST -> "createdAt,desc" + SortOrder.ALPHABETICAL, + SortOrder.ALPHABETICAL_DESC, + -> throw IllegalArgumentException("Unsupported ${filter.sortOrder}") + }, + ) + if (filter.tags.isNotEmpty()) { + url.addQueryParameter("labelsInclude", filter.tags.joinToString(",") { it.key }) + } + if (filter.tagsExclude.isNotEmpty()) { + url.addQueryParameter("labelsExclude", filter.tags.joinToString(",") { it.key }) + } + if (filter.states.isNotEmpty()) { + url.addQueryParameter( + "status", + filter.states.joinToString(",") { + when (it) { + MangaState.ONGOING -> "ONGOING" + MangaState.FINISHED -> "DONE" + MangaState.ABANDONED -> "" + MangaState.PAUSED -> "FROZEN" + MangaState.UPCOMING -> "ANNOUNCE" + } + }, + ) + } + if (filter.contentRating.isNotEmpty()) { + url.addQueryParameter( + "contentStatus", + filter.contentRating.joinToString(",") { + when (it) { + ContentRating.SAFE -> "SAFE" + ContentRating.SUGGESTIVE -> "UNSAFE,EROTIC" + ContentRating.ADULT -> "PORNOGRAPHIC" + } + }, + ) + } + } + + is MangaListFilter.Search -> { + url.addQueryParameter("search", filter.query) + } + + null -> Unit + } + val ja = webClient.httpGet(url.build()).parseJsonArray() + return ja.mapJSON { jo -> + Manga( + id = generateUid(jo.getString("id")), + title = jo.getJSONObject("name").getString("ru"), + altTitle = jo.getJSONObject("name").getStringOrNull("en"), + url = jo.getString("id"), + publicUrl = "https://$domain/manga/${jo.getString("slug")}", + rating = jo.getFloatOrDefault("averageRating", -10f) / 10f, + isNsfw = jo.getStringOrNull("contentStatus").isNsfw(), + coverUrl = jo.getString("poster"), + tags = setOf(), + state = jo.getStringOrNull("status")?.toMangaState(), + author = null, + source = source, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { + val chaptersDeferred = async { getChapters(manga.url) } + val url = urlBuilder("api") + .addPathSegment("v2") + .addPathSegment("books") + .addPathSegment(manga.url) + val jo = webClient.httpGet(url.build()).parseJson() + Manga( + id = generateUid(jo.getString("id")), + title = jo.getJSONObject("name").getString("ru"), + altTitle = jo.getJSONObject("name").getStringOrNull("en"), + url = jo.getString("id"), + publicUrl = "https://$domain/manga/${jo.getString("slug")}", + rating = jo.getFloatOrDefault("averageRating", -10f) / 10f, + isNsfw = jo.getStringOrNull("contentStatus").isNsfw(), + coverUrl = jo.getString("poster"), + tags = jo.getJSONArray("labels").mapJSONToSet { it.toMangaTag() }, + state = jo.getStringOrNull("status")?.toMangaState(), + author = jo.getJSONArray("relations").toJSONList().firstNotNullOfOrNull { + if (it.getStringOrNull("type") == "AUTHOR") { + it.getJSONObject("publisher").getStringOrNull("name") + } else { + null + } + }, + source = source, + largeCoverUrl = null, + description = jo.getString("description").nl2br(), + chapters = chaptersDeferred.await(), + ) + } + + override suspend fun getPages(chapter: MangaChapter): List { + val url = urlBuilder("api") + .addPathSegment("v2") + .addPathSegment("chapters") + .addPathSegment(chapter.url) + val json = webClient.httpGet(url.build()).parseJson() + return json.getJSONArray("pages").mapJSON { jo -> + MangaPage( + id = generateUid(jo.getString("id")), + url = jo.getString("image"), + preview = null, + source = source, + ) + } + } + + override suspend fun getAvailableTags(): Set { + val url = urlBuilder("api") + .addPathSegment("label") + val json = webClient.httpGet(url.build()).parseJson() + return json.getJSONArray("content").mapJSONToSet { jo -> + MangaTag( + title = jo.getJSONObject("name").getString("ru").toTitleCase(sourceLocale), + key = jo.getString("slug"), + source = source, + ) + } + } + + override suspend fun getPageUrl(page: MangaPage): String = page.url + + private suspend fun getChapters(mangaId: String): List { + val url = urlBuilder("api") + .addPathSegment("v2") + .addPathSegment("chapters") + .addQueryParameter("bookId", mangaId) + val ja = webClient.httpGet(url.build()).parseJsonArray() + val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'", Locale.ROOT) + val branches = ArrayMap() + return ja.mapJSON { jo -> + val number = jo.getFloatOrDefault("number", 0f) + val volume = jo.getIntOrDefault("volume", 0) + val branchId = jo.getString("branchId") + MangaChapter( + id = generateUid(jo.getString("id")), + name = jo.getStringOrNull("name") ?: buildString { + if (volume > 0) append("Том ").append(volume).append(' ') + if (number > 0) append("Глава ").append(number) else append("Без имени") + }, + number = number, + volume = volume, + url = jo.getString("id"), + scanlator = null, + uploadDate = dateFormat.tryParse(jo.getString("createdAt")), + branch = branches.getOrPut(branchId) { getBranchName(branchId) }, + source = source, + ) + }.reversed() + } + + private suspend fun getBranchName(id: String): String? { + val url = urlBuilder("api") + .addPathSegment("branch") + .addPathSegment(id) + val json = webClient.httpGet(url.build()).parseJson() + return json.getJSONArray("publishers").mapJSONToSet { it.getStringOrNull("name") }.firstOrNull() + } + + private fun String.toMangaState() = when (this.uppercase(Locale.ROOT)) { + "DONE" -> MangaState.FINISHED + "ONGOING" -> MangaState.ONGOING + "FROZEN" -> MangaState.PAUSED + "ANNOUNCE" -> MangaState.UPCOMING + else -> null + } + + private fun String?.isNsfw() = this.equals("EROTIC", ignoreCase = true) || + this.equals("PORNOGRAPHIC", ignoreCase = true) + + private fun JSONObject.toMangaTag() = MangaTag( + title = getString("name").toTitleCase(sourceLocale), + key = getString("slug"), + source = source, + ) +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ru/RemangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ru/RemangaParser.kt index 4b6a7c39..1884cce0 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ru/RemangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ru/RemangaParser.kt @@ -27,7 +27,7 @@ private const val STATUS_ONGOING = 1 private const val STATUS_FINISHED = 0 private const val TOO_MANY_REQUESTS = 429 -@MangaSourceParser("REMANGA", "ReManga", "ru") +@MangaSourceParser("REMANGA", "Реманга", "ru") internal class RemangaParser( context: MangaLoaderContext, ) : PagedMangaParser(context, MangaSource.REMANGA, PAGE_SIZE), MangaParserAuthProvider, Interceptor { diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/MangaParserEnv.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/MangaParserEnv.kt index d3e5ee7f..79fb7bc0 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/MangaParserEnv.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/MangaParserEnv.kt @@ -87,8 +87,8 @@ fun MangaParser.getDomain(subdomain: String): String { return subdomain + "." + domain.removePrefix("www.") } -fun MangaParser.urlBuilder(): HttpUrl.Builder { +fun MangaParser.urlBuilder(subdomain: String? = null): HttpUrl.Builder { return HttpUrl.Builder() .scheme("https") - .host(domain) + .host(if (subdomain == null) domain else "$subdomain.$domain") } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/String.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/String.kt index bdc23418..58e61287 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/String.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/String.kt @@ -102,6 +102,8 @@ fun String.urlEncoded(): String = URLEncoder.encode(this, Charsets.UTF_8.name()) fun String.urlDecode(): String = URLDecoder.decode(this, Charsets.UTF_8.name()) +fun String.nl2br() = replace("\n", "
") + fun ByteArray.byte2HexFormatted(): String { val str = StringBuilder(size * 2) for (i in indices) { diff --git a/src/test/kotlin/org/koitharu/kotatsu/test_util/Util.kt b/src/test/kotlin/org/koitharu/kotatsu/test_util/Util.kt index 6240da72..d044fed0 100644 --- a/src/test/kotlin/org/koitharu/kotatsu/test_util/Util.kt +++ b/src/test/kotlin/org/koitharu/kotatsu/test_util/Util.kt @@ -37,7 +37,7 @@ internal fun Collection.isDistinctByNotNull(selector: (T) -> K?): Bool return false } } - return set.size == size + return true } internal fun String.isUrlRelative() = matches(PATTERN_URL_RELATIVE)