[MangaDex] Speed up large chapters list loading

build.gradle

@@ -20,6 +20,7 @@ compileKotlin {
 		freeCompilerArgs += [
 				'-opt-in=kotlin.RequiresOptIn',
 				'-opt-in=kotlin.contracts.ExperimentalContracts',
+				'-opt-in=kotlinx.coroutines.ExperimentalCoroutinesApi',
 				'-opt-in=org.koitharu.kotatsu.parsers.InternalParsersApi',
 		]
 	}

src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaDexParser.kt

@@ -1,12 +1,15 @@
 package org.koitharu.kotatsu.parsers.site
 
+import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.async
+import kotlinx.coroutines.awaitAll
 import kotlinx.coroutines.coroutineScope
 import org.json.JSONObject
 import org.koitharu.kotatsu.parsers.MangaLoaderContext
 import org.koitharu.kotatsu.parsers.MangaParser
 import org.koitharu.kotatsu.parsers.MangaSourceParser
 import org.koitharu.kotatsu.parsers.config.ConfigKey
+import org.koitharu.kotatsu.parsers.exception.ParseException
 import org.koitharu.kotatsu.parsers.model.*
 import org.koitharu.kotatsu.parsers.util.*
 import org.koitharu.kotatsu.parsers.util.json.*
@@ -14,7 +17,9 @@ import java.text.SimpleDateFormat
 import java.util.*
 
 private const val PAGE_SIZE = 20
-private const val CHAPTERS_PAGE_SIZE = 60
+private const val CHAPTERS_FIRST_PAGE_SIZE = 120
+private const val CHAPTERS_MAX_PAGE_SIZE = 500
+private const val CHAPTERS_PARALLELISM = 3
 private const val CONTENT_RATING =
 	"contentRating[]=safe&contentRating[]=suggestive&contentRating[]=erotica&contentRating[]=pornographic"
 private const val LOCALE_FALLBACK = "en"
@@ -62,6 +67,7 @@ internal class MangaDexParser(override val context: MangaLoaderContext) : MangaP
 				when (sortOrder) {
 					SortOrder.UPDATED,
 					-> "[latestUploadedChapter]=desc"
+
 					SortOrder.ALPHABETICAL -> "[title]=asc"
 					SortOrder.NEWEST -> "[createdAt]=desc"
 					SortOrder.POPULARITY -> "[followedCount]=desc"
@@ -118,12 +124,13 @@ internal class MangaDexParser(override val context: MangaLoaderContext) : MangaP
 
 	override suspend fun getDetails(manga: Manga): Manga = coroutineScope {
 		val domain = getDomain()
+		val mangaId = manga.url.removePrefix("/")
 		val attrsDeferred = async {
 			context.httpGet(
-				"https://api.$domain/manga/${manga.url}?includes[]=artist&includes[]=author&includes[]=cover_art",
+				"https://api.$domain/manga/${mangaId}?includes[]=artist&includes[]=author&includes[]=cover_art",
 			).parseJson().getJSONObject("data").getJSONObject("attributes")
 		}
-		val feedDeferred = async { loadChapters(manga) }
+		val feedDeferred = async { loadChapters(mangaId) }
 		val mangaAttrs = attrsDeferred.await()
 		val feed = feedDeferred.await()
 		// 2022-01-02T00:27:11+00:00
@@ -134,11 +141,11 @@ internal class MangaDexParser(override val context: MangaLoaderContext) : MangaP
 		manga.copy(
 			description = mangaAttrs.getJSONObject("description").selectByLocale()
 				?: manga.description,
-			chapters = feed.mapNotNull { jo ->
+			chapters = feed.mapChapters { _, jo ->
 				val id = jo.getString("id")
 				val attrs = jo.getJSONObject("attributes")
 				if (!attrs.isNull("externalUrl")) {
-					return@mapNotNull null
+					return@mapChapters null
 				}
 				val locale = attrs.getStringOrNull("translatedLanguage")?.let { Locale.forLanguageTag(it) }
 				val relations = jo.getJSONArray("relationships").associateByKey("type")
@@ -201,30 +208,62 @@ internal class MangaDexParser(override val context: MangaLoaderContext) : MangaP
 		return getStringOrNull(LOCALE_FALLBACK) ?: values().nextOrNull() as? String
 	}
 
-	private suspend fun loadChapters(manga: Manga): List<JSONObject> {
+	private suspend fun loadChapters(mangaId: String): List<JSONObject> {
-		val domain = getDomain()
+		val firstPage = loadChapters(mangaId, offset = 0, limit = CHAPTERS_FIRST_PAGE_SIZE)
-		val result = ArrayList<JSONObject>()
+		if (firstPage.size >= firstPage.total) {
-		while (true) {
+			return firstPage.data
+		}
+		val tail = coroutineScope {
+			val leftCount = firstPage.total - firstPage.size
+			val pages = (leftCount / CHAPTERS_MAX_PAGE_SIZE.toFloat()).toIntUp()
+			val dispatcher = Dispatchers.Default.limitedParallelism(CHAPTERS_PARALLELISM)
+			List(pages) { page ->
+				val offset = page * CHAPTERS_MAX_PAGE_SIZE + firstPage.size
+				async(dispatcher) {
+					loadChapters(mangaId, offset, CHAPTERS_MAX_PAGE_SIZE)
+				}
+			}.awaitAll()
+		}
+		val result = ArrayList<JSONObject>(firstPage.total)
+		result += firstPage.data
+		tail.flatMapTo(result) { it.data }
+		return result
+	}
+
+	private suspend fun loadChapters(mangaId: String, offset: Int, limit: Int): Chapters {
 		val url = buildString {
 			append("https://api.")
-				append(domain)
+			append(getDomain())
 			append("/manga/")
-				append(manga.url)
+			append(mangaId)
 			append("/feed")
 			append("?limit=")
-				append(CHAPTERS_PAGE_SIZE)
+			append(limit)
 			append("&includes[]=scanlation_group&order[volume]=asc&order[chapter]=asc&offset=")
-				append(result.size)
+			append(offset)
 			append('&')
 			append(CONTENT_RATING)
 		}
 		val json = context.httpGet(url).parseJson()
-			val data = json.optJSONArray("data")
+		if (json.getString("result") == "ok") {
-			if (data.isNullOrEmpty()) {
+			return Chapters(
-				break
+				data = json.optJSONArray("data")?.filterIsInstance<JSONObject>().orEmpty(),
+				total = json.getInt("total"),
+			)
+		} else {
+			val error = json.optJSONArray("errors").mapJSON { jo ->
+				jo.getString("detail")
+			}.joinToString("\n")
+			throw ParseException(error, url)
 		}
-			data.mapJSONTo(result) { it }
 	}
-		return result
+
+	private class Chapters(
+		val data: List<JSONObject>,
+		val total: Int,
+	) {
+
+		val size: Int
+			get() = data.size
 	}
 }