diff --git a/build.gradle b/build.gradle index 5a25323b..0b33f538 100644 --- a/build.gradle +++ b/build.gradle @@ -20,6 +20,7 @@ compileKotlin { freeCompilerArgs += [ '-opt-in=kotlin.RequiresOptIn', '-opt-in=kotlin.contracts.ExperimentalContracts', + '-opt-in=kotlinx.coroutines.ExperimentalCoroutinesApi', '-opt-in=org.koitharu.kotatsu.parsers.InternalParsersApi', ] } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaDexParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaDexParser.kt index 3e76e5e6..1d891a40 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaDexParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaDexParser.kt @@ -1,12 +1,15 @@ package org.koitharu.kotatsu.parsers.site +import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll import kotlinx.coroutines.coroutineScope import org.json.JSONObject import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaParser import org.koitharu.kotatsu.parsers.MangaSourceParser import org.koitharu.kotatsu.parsers.config.ConfigKey +import org.koitharu.kotatsu.parsers.exception.ParseException import org.koitharu.kotatsu.parsers.model.* import org.koitharu.kotatsu.parsers.util.* import org.koitharu.kotatsu.parsers.util.json.* @@ -14,7 +17,9 @@ import java.text.SimpleDateFormat import java.util.* private const val PAGE_SIZE = 20 -private const val CHAPTERS_PAGE_SIZE = 60 +private const val CHAPTERS_FIRST_PAGE_SIZE = 120 +private const val CHAPTERS_MAX_PAGE_SIZE = 500 +private const val CHAPTERS_PARALLELISM = 3 private const val CONTENT_RATING = "contentRating[]=safe&contentRating[]=suggestive&contentRating[]=erotica&contentRating[]=pornographic" private const val LOCALE_FALLBACK = "en" @@ -62,6 +67,7 @@ internal class MangaDexParser(override val context: MangaLoaderContext) : MangaP when (sortOrder) { SortOrder.UPDATED, -> "[latestUploadedChapter]=desc" + SortOrder.ALPHABETICAL -> "[title]=asc" SortOrder.NEWEST -> "[createdAt]=desc" SortOrder.POPULARITY -> "[followedCount]=desc" @@ -118,12 +124,13 @@ internal class MangaDexParser(override val context: MangaLoaderContext) : MangaP override suspend fun getDetails(manga: Manga): Manga = coroutineScope { val domain = getDomain() + val mangaId = manga.url.removePrefix("/") val attrsDeferred = async { context.httpGet( - "https://api.$domain/manga/${manga.url}?includes[]=artist&includes[]=author&includes[]=cover_art", + "https://api.$domain/manga/${mangaId}?includes[]=artist&includes[]=author&includes[]=cover_art", ).parseJson().getJSONObject("data").getJSONObject("attributes") } - val feedDeferred = async { loadChapters(manga) } + val feedDeferred = async { loadChapters(mangaId) } val mangaAttrs = attrsDeferred.await() val feed = feedDeferred.await() // 2022-01-02T00:27:11+00:00 @@ -134,11 +141,11 @@ internal class MangaDexParser(override val context: MangaLoaderContext) : MangaP manga.copy( description = mangaAttrs.getJSONObject("description").selectByLocale() ?: manga.description, - chapters = feed.mapNotNull { jo -> + chapters = feed.mapChapters { _, jo -> val id = jo.getString("id") val attrs = jo.getJSONObject("attributes") if (!attrs.isNull("externalUrl")) { - return@mapNotNull null + return@mapChapters null } val locale = attrs.getStringOrNull("translatedLanguage")?.let { Locale.forLanguageTag(it) } val relations = jo.getJSONArray("relationships").associateByKey("type") @@ -201,30 +208,62 @@ internal class MangaDexParser(override val context: MangaLoaderContext) : MangaP return getStringOrNull(LOCALE_FALLBACK) ?: values().nextOrNull() as? String } - private suspend fun loadChapters(manga: Manga): List { - val domain = getDomain() - val result = ArrayList() - while (true) { - val url = buildString { - append("https://api.") - append(domain) - append("/manga/") - append(manga.url) - append("/feed") - append("?limit=") - append(CHAPTERS_PAGE_SIZE) - append("&includes[]=scanlation_group&order[volume]=asc&order[chapter]=asc&offset=") - append(result.size) - append('&') - append(CONTENT_RATING) - } - val json = context.httpGet(url).parseJson() - val data = json.optJSONArray("data") - if (data.isNullOrEmpty()) { - break - } - data.mapJSONTo(result) { it } + private suspend fun loadChapters(mangaId: String): List { + val firstPage = loadChapters(mangaId, offset = 0, limit = CHAPTERS_FIRST_PAGE_SIZE) + if (firstPage.size >= firstPage.total) { + return firstPage.data } + val tail = coroutineScope { + val leftCount = firstPage.total - firstPage.size + val pages = (leftCount / CHAPTERS_MAX_PAGE_SIZE.toFloat()).toIntUp() + val dispatcher = Dispatchers.Default.limitedParallelism(CHAPTERS_PARALLELISM) + List(pages) { page -> + val offset = page * CHAPTERS_MAX_PAGE_SIZE + firstPage.size + async(dispatcher) { + loadChapters(mangaId, offset, CHAPTERS_MAX_PAGE_SIZE) + } + }.awaitAll() + } + val result = ArrayList(firstPage.total) + result += firstPage.data + tail.flatMapTo(result) { it.data } return result } + + private suspend fun loadChapters(mangaId: String, offset: Int, limit: Int): Chapters { + val url = buildString { + append("https://api.") + append(getDomain()) + append("/manga/") + append(mangaId) + append("/feed") + append("?limit=") + append(limit) + append("&includes[]=scanlation_group&order[volume]=asc&order[chapter]=asc&offset=") + append(offset) + append('&') + append(CONTENT_RATING) + } + val json = context.httpGet(url).parseJson() + if (json.getString("result") == "ok") { + return Chapters( + data = json.optJSONArray("data")?.filterIsInstance().orEmpty(), + total = json.getInt("total"), + ) + } else { + val error = json.optJSONArray("errors").mapJSON { jo -> + jo.getString("detail") + }.joinToString("\n") + throw ParseException(error, url) + } + } + + private class Chapters( + val data: List, + val total: Int, + ) { + + val size: Int + get() = data.size + } } \ No newline at end of file