Paginator class to compute page number by offset

.idea/runConfigurations/TestsAndReport.xml

@@ -1,7 +0,0 @@
-<component name="ProjectRunConfigurationManager">
-  <configuration default="false" name="TestsAndReport" type="CompoundRunConfigurationType">
-    <toRun name="MangaParserTest" type="GradleRunConfiguration" />
-    <toRun name="kotatsu-parsers [generateTestsReport]" type="GradleRunConfiguration" />
-    <method v="2" />
-  </configuration>
-</component>

src/main/kotlin/org/koitharu/kotatsu/parsers/MangaParser.kt

@@ -69,7 +69,7 @@ abstract class MangaParser @InternalParsersApi constructor(val source: MangaSour
 	 * @param offset starting from 0 and used for pagination.
 	 * @param query search query
 	 */
-	suspend fun getList(offset: Int, query: String): List<Manga> {
+	open suspend fun getList(offset: Int, query: String): List<Manga> {
 		return getList(offset, query, null, defaultSortOrder)
 	}
 
@@ -81,7 +81,7 @@ abstract class MangaParser @InternalParsersApi constructor(val source: MangaSour
 	 * @param tags genres for filtering, values from [getTags] and [Manga.tags]. May be null or empty
 	 * @param sortOrder one of [sortOrders] or null for default value
 	 */
-	suspend fun getList(offset: Int, tags: Set<MangaTag>?, sortOrder: SortOrder?): List<Manga> {
+	open suspend fun getList(offset: Int, tags: Set<MangaTag>?, sortOrder: SortOrder?): List<Manga> {
 		return getList(offset, null, tags, sortOrder ?: defaultSortOrder)
 	}
 

src/main/kotlin/org/koitharu/kotatsu/parsers/PagedMangaParser.kt

@@ -0,0 +1,50 @@
+package org.koitharu.kotatsu.parsers
+
+import org.koitharu.kotatsu.parsers.model.Manga
+import org.koitharu.kotatsu.parsers.model.MangaSource
+import org.koitharu.kotatsu.parsers.model.MangaTag
+import org.koitharu.kotatsu.parsers.model.SortOrder
+import org.koitharu.kotatsu.parsers.util.Paginator
+
+@InternalParsersApi
+abstract class PagedMangaParser(
+	source: MangaSource,
+	pageSize: Int,
+	searchPageSize: Int = pageSize,
+) : MangaParser(source) {
+
+	protected val paginator = Paginator(pageSize)
+	protected val searchPaginator = Paginator(searchPageSize)
+
+	override suspend fun getList(offset: Int, query: String): List<Manga> {
+		return getList(searchPaginator, offset, query, null, defaultSortOrder)
+	}
+
+	override suspend fun getList(offset: Int, tags: Set<MangaTag>?, sortOrder: SortOrder?): List<Manga> {
+		return getList(paginator, offset, null, tags, sortOrder ?: defaultSortOrder)
+	}
+
+	@InternalParsersApi
+	@Deprecated("You should use getListPage for PagedMangaParser", level = DeprecationLevel.HIDDEN)
+	final override suspend fun getList(
+		offset: Int,
+		query: String?,
+		tags: Set<MangaTag>?,
+		sortOrder: SortOrder,
+	): List<Manga> = throw UnsupportedOperationException("You should use getListPage for PagedMangaParser")
+
+	abstract suspend fun getListPage(page: Int, query: String?, tags: Set<MangaTag>?, sortOrder: SortOrder): List<Manga>
+
+	private suspend fun getList(
+		paginator: Paginator,
+		offset: Int,
+		query: String?,
+		tags: Set<MangaTag>?,
+		sortOrder: SortOrder,
+	): List<Manga> {
+		val page = paginator.getPage(offset)
+		val list = getListPage(page, query, tags, sortOrder)
+		paginator.onListReceived(offset, page, list.size)
+		return list
+	}
+}

src/main/kotlin/org/koitharu/kotatsu/parsers/site/BatoToParser.kt

@@ -5,8 +5,8 @@ import org.json.JSONArray
 import org.json.JSONObject
 import org.jsoup.nodes.Element
 import org.koitharu.kotatsu.parsers.MangaLoaderContext
-import org.koitharu.kotatsu.parsers.MangaParser
 import org.koitharu.kotatsu.parsers.MangaSourceParser
+import org.koitharu.kotatsu.parsers.PagedMangaParser
 import org.koitharu.kotatsu.parsers.config.ConfigKey
 import org.koitharu.kotatsu.parsers.model.*
 import org.koitharu.kotatsu.parsers.util.*
@@ -17,11 +17,12 @@ import javax.crypto.Cipher
 import javax.crypto.spec.IvParameterSpec
 import javax.crypto.spec.SecretKeySpec
 
-private const val PAGE_SIZE = 60
-private const val PAGE_SIZE_SEARCH = 20
-
 @MangaSourceParser("BATOTO", "Bato.To")
-internal class BatoToParser(override val context: MangaLoaderContext) : MangaParser(MangaSource.BATOTO) {
+internal class BatoToParser(override val context: MangaLoaderContext) : PagedMangaParser(
+	source = MangaSource.BATOTO,
+	pageSize = 60,
+	searchPageSize = 20,
+) {
 
 	override val sortOrders: Set<SortOrder> = EnumSet.of(
 		SortOrder.NEWEST,
@@ -35,17 +36,15 @@ internal class BatoToParser(override val context: MangaLoaderContext) : MangaPar
 		arrayOf("bato.to", "mto.to", "mangatoto.com", "battwo.com", "batotwo.com", "comiko.net", "batotoo.com"),
 	)
 
-	override suspend fun getList(
+	override suspend fun getListPage(
-		offset: Int,
+		page: Int,
 		query: String?,
 		tags: Set<MangaTag>?,
 		sortOrder: SortOrder,
 	): List<Manga> {
 		if (!query.isNullOrEmpty()) {
-			return search(offset, query)
+			return search(page, query)
 		}
-		val page = (offset / PAGE_SIZE) + 1
-
 		@Suppress("NON_EXHAUSTIVE_WHEN_STATEMENT")
 		val url = buildString {
 			append("https://")
@@ -159,8 +158,7 @@ internal class BatoToParser(override val context: MangaLoaderContext) : MangaPar
 
 	override fun getFaviconUrl(): String = "https://styles.amarkcdn.com/img/batoto/favicon.ico?v0"
 
-	private suspend fun search(offset: Int, query: String): List<Manga> {
+	private suspend fun search(page: Int, query: String): List<Manga> {
-		val page = (offset / PAGE_SIZE_SEARCH) + 1
 		val url = buildString {
 			append("https://")
 			append(getDomain())

src/main/kotlin/org/koitharu/kotatsu/parsers/site/DesuMeParser.kt

@@ -1,8 +1,8 @@
 package org.koitharu.kotatsu.parsers.site
 
 import org.koitharu.kotatsu.parsers.MangaLoaderContext
-import org.koitharu.kotatsu.parsers.MangaParser
 import org.koitharu.kotatsu.parsers.MangaSourceParser
+import org.koitharu.kotatsu.parsers.PagedMangaParser
 import org.koitharu.kotatsu.parsers.config.ConfigKey
 import org.koitharu.kotatsu.parsers.exception.ParseException
 import org.koitharu.kotatsu.parsers.model.*
@@ -13,7 +13,7 @@ import org.koitharu.kotatsu.parsers.util.json.mapJSONToSet
 import java.util.*
 
 @MangaSourceParser("DESUME", "Desu.me", "ru")
-internal class DesuMeParser(override val context: MangaLoaderContext) : MangaParser(MangaSource.DESUME) {
+internal class DesuMeParser(override val context: MangaLoaderContext) : PagedMangaParser(MangaSource.DESUME, 20) {
 
 	override val configKeyDomain = ConfigKey.Domain("desu.me", null)
 
@@ -24,13 +24,13 @@ internal class DesuMeParser(override val context: MangaLoaderContext) : MangaPar
 		SortOrder.ALPHABETICAL,
 	)
 
-	override suspend fun getList(
+	override suspend fun getListPage(
-		offset: Int,
+		page: Int,
 		query: String?,
 		tags: Set<MangaTag>?,
 		sortOrder: SortOrder,
 	): List<Manga> {
-		if (query != null && offset != 0) {
+		if (query != null && page != searchPaginator.firstPage) {
 			return emptyList()
 		}
 		val domain = getDomain()
@@ -40,7 +40,7 @@ internal class DesuMeParser(override val context: MangaLoaderContext) : MangaPar
 			append("/manga/api/?limit=20&order=")
 			append(getSortKey(sortOrder))
 			append("&page=")
-			append((offset / 20) + 1)
+			append(page)
 			if (!tags.isNullOrEmpty()) {
 				append("&genres=")
 				appendAll(tags, ",") { it.key }

src/main/kotlin/org/koitharu/kotatsu/parsers/site/ExHentaiParser.kt

@@ -2,9 +2,9 @@ package org.koitharu.kotatsu.parsers.site
 
 import org.jsoup.nodes.Element
 import org.koitharu.kotatsu.parsers.MangaLoaderContext
-import org.koitharu.kotatsu.parsers.MangaParser
 import org.koitharu.kotatsu.parsers.MangaParserAuthProvider
 import org.koitharu.kotatsu.parsers.MangaSourceParser
+import org.koitharu.kotatsu.parsers.PagedMangaParser
 import org.koitharu.kotatsu.parsers.config.ConfigKey
 import org.koitharu.kotatsu.parsers.exception.AuthRequiredException
 import org.koitharu.kotatsu.parsers.exception.ParseException
@@ -19,7 +19,7 @@ private const val DOMAIN_AUTHORIZED = "exhentai.org"
 @MangaSourceParser("EXHENTAI", "ExHentai")
 internal class ExHentaiParser(
 	override val context: MangaLoaderContext,
-) : MangaParser(MangaSource.EXHENTAI), MangaParserAuthProvider {
+) : PagedMangaParser(MangaSource.EXHENTAI, pageSize = 25), MangaParserAuthProvider {
 
 	override val sortOrders: Set<SortOrder> = Collections.singleton(
 		SortOrder.NEWEST,
@@ -57,13 +57,12 @@ internal class ExHentaiParser(
 		context.cookieJar.insertCookies(DOMAIN_UNAUTHORIZED, "nw=1", "sl=dm_2")
 	}
 
-	override suspend fun getList(
+	override suspend fun getListPage(
-		offset: Int,
+		page: Int,
 		query: String?,
 		tags: Set<MangaTag>?,
 		sortOrder: SortOrder,
 	): List<Manga> {
-		val page = (offset / 25f).toIntUp()
 		var search = query?.urlEncoded().orEmpty()
 		val url = buildString {
 			append("https://")
@@ -98,7 +97,7 @@ internal class ExHentaiParser(
 				parseFailed("Cannot find root")
 			} else {
 				updateDm = true
-				return getList(offset, query, tags, sortOrder)
+				return getListPage(page, query, tags, sortOrder)
 			}
 		updateDm = false
 		return root.children().mapNotNull { tr ->

src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaInUaParser.kt

@@ -1,8 +1,8 @@
 package org.koitharu.kotatsu.parsers.site
 
 import org.koitharu.kotatsu.parsers.MangaLoaderContext
-import org.koitharu.kotatsu.parsers.MangaParser
 import org.koitharu.kotatsu.parsers.MangaSourceParser
+import org.koitharu.kotatsu.parsers.PagedMangaParser
 import org.koitharu.kotatsu.parsers.config.ConfigKey
 import org.koitharu.kotatsu.parsers.model.*
 import org.koitharu.kotatsu.parsers.util.*
@@ -12,26 +12,28 @@ import java.util.*
 private const val DEF_BRANCH_NAME = "Основний переклад"
 
 @MangaSourceParser("MANGAINUA", "MANGA/in/UA", "uk")
-class MangaInUaParser(override val context: MangaLoaderContext) : MangaParser(MangaSource.MANGAINUA) {
+class MangaInUaParser(override val context: MangaLoaderContext) : PagedMangaParser(
+	source = MangaSource.MANGAINUA,
+	pageSize = 24,
+	searchPageSize = 10,
+) {
 
 	override val sortOrders: Set<SortOrder>
 		get() = Collections.singleton(SortOrder.UPDATED)
 
 	override val configKeyDomain: ConfigKey.Domain = ConfigKey.Domain("manga.in.ua", null)
 
-	override suspend fun getList(
+	override suspend fun getListPage(
-		offset: Int,
+		page: Int,
 		query: String?,
 		tags: Set<MangaTag>?,
 		sortOrder: SortOrder,
 	): List<Manga> {
-		val page = (offset / 24f).toIntUp().inc()
-		val searchPage = (offset / 10f).toIntUp().inc()
 		val url = when {
 			!query.isNullOrEmpty() -> (
 					"/index.php?do=search" +
 							"&subaction=search" +
-							"&search_start=$searchPage" +
+							"&search_start=$page" +
 							"&full_search=1" +
 							"&story=$query" +
 							"&titleonly=3"

src/main/kotlin/org/koitharu/kotatsu/parsers/site/NHentaiParser.kt

@@ -6,8 +6,8 @@ import kotlinx.coroutines.awaitAll
 import kotlinx.coroutines.coroutineScope
 import org.jsoup.nodes.Element
 import org.koitharu.kotatsu.parsers.MangaLoaderContext
-import org.koitharu.kotatsu.parsers.MangaParser
 import org.koitharu.kotatsu.parsers.MangaSourceParser
+import org.koitharu.kotatsu.parsers.PagedMangaParser
 import org.koitharu.kotatsu.parsers.config.ConfigKey
 import org.koitharu.kotatsu.parsers.model.*
 import org.koitharu.kotatsu.parsers.util.*
@@ -15,7 +15,7 @@ import java.text.SimpleDateFormat
 import java.util.*
 
 @MangaSourceParser("NHENTAI", "N-Hentai")
-class NHentaiParser(override val context: MangaLoaderContext) : MangaParser(MangaSource.NHENTAI) {
+class NHentaiParser(override val context: MangaLoaderContext) : PagedMangaParser(MangaSource.NHENTAI, pageSize = 25) {
 
 	override val configKeyDomain: ConfigKey.Domain
 		get() = ConfigKey.Domain("nhentai.net", null)
@@ -23,17 +23,16 @@ class NHentaiParser(override val context: MangaLoaderContext) : MangaParser(Mang
 	override val sortOrders: Set<SortOrder>
 		get() = EnumSet.of(SortOrder.NEWEST, SortOrder.POPULARITY)
 
-	override suspend fun getList(
+	override suspend fun getListPage(
-		offset: Int,
+		page: Int,
 		query: String?,
 		tags: Set<MangaTag>?,
 		sortOrder: SortOrder,
 	): List<Manga> {
 		if (query.isNullOrEmpty() && tags != null && tags.size > 1) {
-			return getList(offset, buildQuery(tags), emptySet(), sortOrder)
+			return getListPage(page, buildQuery(tags), emptySet(), sortOrder)
 		}
 		val domain = getDomain()
-		val page = (offset / 25) + 1
 		val url = buildString {
 			append("https://")
 			append(domain)
@@ -101,7 +100,7 @@ class NHentaiParser(override val context: MangaLoaderContext) : MangaParser(Mang
 
 	override suspend fun getDetails(manga: Manga): Manga {
 		val root = context.httpGet(
-			url = manga.url.toAbsoluteUrl(getDomain())
+			url = manga.url.toAbsoluteUrl(getDomain()),
 		).parseHtml().body().requireElementById("bigcontainer")
 		val img = root.requireElementById("cover").selectFirstOrThrow("img")
 		val tagContainers = root.requireElementById("tags").select(".tag-container")
@@ -126,12 +125,12 @@ class NHentaiParser(override val context: MangaLoaderContext) : MangaParser(Mang
 					uploadDate = dateFormat.tryParse(
 						tagContainers.find { x -> x.ownText() == "Uploaded:" }
 							?.selectFirst("time")
-							?.attr("datetime")
+							?.attr("datetime"),
 					),
 					branch = null,
 					source = source,
-				)
+				),
-			)
+			),
 		)
 	}
 

src/main/kotlin/org/koitharu/kotatsu/parsers/site/NineMangaParser.kt

@@ -3,8 +3,8 @@ package org.koitharu.kotatsu.parsers.site
 import okhttp3.Headers
 import okhttp3.HttpUrl.Companion.toHttpUrl
 import org.koitharu.kotatsu.parsers.MangaLoaderContext
-import org.koitharu.kotatsu.parsers.MangaParser
 import org.koitharu.kotatsu.parsers.MangaSourceParser
+import org.koitharu.kotatsu.parsers.PagedMangaParser
 import org.koitharu.kotatsu.parsers.config.ConfigKey
 import org.koitharu.kotatsu.parsers.exception.ParseException
 import org.koitharu.kotatsu.parsers.model.*
@@ -12,13 +12,11 @@ import org.koitharu.kotatsu.parsers.util.*
 import java.text.SimpleDateFormat
 import java.util.*
 
-private const val PAGE_SIZE = 26
-
 internal abstract class NineMangaParser(
 	final override val context: MangaLoaderContext,
 	source: MangaSource,
 	defaultDomain: String,
-) : MangaParser(source) {
+) : PagedMangaParser(source, pageSize = 26) {
 
 	override val configKeyDomain = ConfigKey.Domain(defaultDomain, null)
 
@@ -34,13 +32,12 @@ internal abstract class NineMangaParser(
 		SortOrder.POPULARITY,
 	)
 
-	override suspend fun getList(
+	override suspend fun getListPage(
-		offset: Int,
+		page: Int,
 		query: String?,
 		tags: Set<MangaTag>?,
 		sortOrder: SortOrder,
 	): List<Manga> {
-		val page = (offset / PAGE_SIZE.toFloat()).toIntUp() + 1
 		val url = buildString {
 			append("https://")
 			append(getDomain())

src/main/kotlin/org/koitharu/kotatsu/parsers/site/RemangaParser.kt

@@ -6,9 +6,9 @@ import org.json.JSONArray
 import org.json.JSONException
 import org.json.JSONObject
 import org.koitharu.kotatsu.parsers.MangaLoaderContext
-import org.koitharu.kotatsu.parsers.MangaParser
 import org.koitharu.kotatsu.parsers.MangaParserAuthProvider
 import org.koitharu.kotatsu.parsers.MangaSourceParser
+import org.koitharu.kotatsu.parsers.PagedMangaParser
 import org.koitharu.kotatsu.parsers.config.ConfigKey
 import org.koitharu.kotatsu.parsers.exception.AuthRequiredException
 import org.koitharu.kotatsu.parsers.exception.ParseException
@@ -31,7 +31,7 @@ private const val STATUS_FINISHED = 0
 @MangaSourceParser("REMANGA", "Remanga", "ru")
 internal class RemangaParser(
 	override val context: MangaLoaderContext,
-) : MangaParser(MangaSource.REMANGA), MangaParserAuthProvider {
+) : PagedMangaParser(MangaSource.REMANGA, PAGE_SIZE), MangaParserAuthProvider {
 
 	override val configKeyDomain = ConfigKey.Domain("remanga.org", null)
 	override val authUrl: String
@@ -53,8 +53,8 @@ internal class RemangaParser(
 
 	private val regexLastUrlPath = Regex("/[^/]+/?$")
 
-	override suspend fun getList(
+	override suspend fun getListPage(
-		offset: Int,
+		page: Int,
 		query: String?,
 		tags: Set<MangaTag>?,
 		sortOrder: SortOrder,
@@ -77,7 +77,7 @@ internal class RemangaParser(
 		}
 		urlBuilder
 			.append("&page=")
-			.append((offset / PAGE_SIZE) + 1)
+			.append(page)
 			.append("&count=")
 			.append(PAGE_SIZE)
 		val content = context.httpGet(urlBuilder.toString(), getApiHeaders()).parseJson()

src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Madara5Parser.kt

@@ -4,8 +4,8 @@ import androidx.collection.arraySetOf
 import org.jsoup.nodes.Element
 import org.koitharu.kotatsu.parsers.InternalParsersApi
 import org.koitharu.kotatsu.parsers.MangaLoaderContext
-import org.koitharu.kotatsu.parsers.MangaParser
 import org.koitharu.kotatsu.parsers.MangaSourceParser
+import org.koitharu.kotatsu.parsers.PagedMangaParser
 import org.koitharu.kotatsu.parsers.config.ConfigKey
 import org.koitharu.kotatsu.parsers.model.*
 import org.koitharu.kotatsu.parsers.util.*
@@ -16,9 +16,8 @@ abstract class Madara5Parser @InternalParsersApi constructor(
 	override val context: MangaLoaderContext,
 	source: MangaSource,
 	domain: String,
-) : MangaParser(source) {
+) : PagedMangaParser(source, pageSize = 22) {
 
-	protected open val pageSize = 22
 	protected open val tagPrefix = "/mangas/"
 	protected open val nsfwTags = arraySetOf("yaoi", "yuri", "mature")
 
@@ -26,9 +25,12 @@ abstract class Madara5Parser @InternalParsersApi constructor(
 
 	override val configKeyDomain = ConfigKey.Domain(domain, null)
 
-	@InternalParsersApi
+	override suspend fun getListPage(
-	override suspend fun getList(offset: Int, query: String?, tags: Set<MangaTag>?, sortOrder: SortOrder): List<Manga> {
+		page: Int,
-		val page = (offset / pageSize.toFloat()).toIntUp()
+		query: String?,
+		tags: Set<MangaTag>?,
+		sortOrder: SortOrder,
+	): List<Manga> {
 		val domain = getDomain()
 		val url = buildString {
 			append("https://")

src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt

@@ -2,8 +2,8 @@ package org.koitharu.kotatsu.parsers.site.madara
 
 import org.jsoup.nodes.Element
 import org.koitharu.kotatsu.parsers.MangaLoaderContext
-import org.koitharu.kotatsu.parsers.MangaParser
 import org.koitharu.kotatsu.parsers.MangaSourceParser
+import org.koitharu.kotatsu.parsers.PagedMangaParser
 import org.koitharu.kotatsu.parsers.config.ConfigKey
 import org.koitharu.kotatsu.parsers.exception.ParseException
 import org.koitharu.kotatsu.parsers.model.*
@@ -12,13 +12,11 @@ import java.text.DateFormat
 import java.text.SimpleDateFormat
 import java.util.*
 
-private const val PAGE_SIZE = 12
-
 internal abstract class MadaraParser(
 	override val context: MangaLoaderContext,
 	source: MangaSource,
 	domain: String,
-) : MangaParser(source) {
+) : PagedMangaParser(source, pageSize = 12) {
 
 	override val configKeyDomain = ConfigKey.Domain(domain, null)
 
@@ -30,22 +28,27 @@ internal abstract class MadaraParser(
 	protected open val tagPrefix = "manga-genre/"
 	protected open val isNsfwSource = false
 
-	override suspend fun getList(
+	init {
-		offset: Int,
+		paginator.firstPage = 0
+		searchPaginator.firstPage = 0
+	}
+
+	override suspend fun getListPage(
+		page: Int,
 		query: String?,
 		tags: Set<MangaTag>?,
 		sortOrder: SortOrder,
 	): List<Manga> {
 		val tag = tags.oneOrThrowIfMany()
 		val payload = createRequestTemplate()
-		payload["page"] = (offset / PAGE_SIZE.toFloat()).toIntUp().toString()
+		payload["page"] = page.toString()
 		payload["vars[meta_key]"] = when (sortOrder) {
 			SortOrder.POPULARITY -> "_wp_manga_views"
 			SortOrder.UPDATED -> "_latest_update"
 			else -> "_wp_manga_views"
 		}
 		payload["vars[wp-manga-genre]"] = tag?.key.orEmpty()
-		payload["vars[s]"] = query.orEmpty()
+		payload["vars[s]"] = query?.urlEncoded().orEmpty()
 		val doc = context.httpPost(
 			"https://${getDomain()}/wp-admin/admin-ajax.php",
 			payload,

src/main/kotlin/org/koitharu/kotatsu/parsers/site/rulib/MangaLibParser.kt

@@ -5,9 +5,9 @@ import org.json.JSONArray
 import org.json.JSONObject
 import org.jsoup.nodes.Document
 import org.koitharu.kotatsu.parsers.MangaLoaderContext
-import org.koitharu.kotatsu.parsers.MangaParser
 import org.koitharu.kotatsu.parsers.MangaParserAuthProvider
 import org.koitharu.kotatsu.parsers.MangaSourceParser
+import org.koitharu.kotatsu.parsers.PagedMangaParser
 import org.koitharu.kotatsu.parsers.config.ConfigKey
 import org.koitharu.kotatsu.parsers.exception.AuthRequiredException
 import org.koitharu.kotatsu.parsers.exception.ParseException
@@ -22,7 +22,7 @@ import java.util.*
 internal open class MangaLibParser(
 	override val context: MangaLoaderContext,
 	source: MangaSource,
-) : MangaParser(source), MangaParserAuthProvider {
+) : PagedMangaParser(source, pageSize = 60), MangaParserAuthProvider {
 
 	override val configKeyDomain = ConfigKey.Domain("mangalib.me", null)
 
@@ -37,16 +37,15 @@ internal open class MangaLibParser(
 		SortOrder.NEWEST,
 	)
 
-	override suspend fun getList(
+	override suspend fun getListPage(
-		offset: Int,
+		page: Int,
 		query: String?,
 		tags: Set<MangaTag>?,
 		sortOrder: SortOrder,
 	): List<Manga> {
 		if (!query.isNullOrEmpty()) {
-			return if (offset == 0) search(query) else emptyList()
+			return if (page == searchPaginator.firstPage) search(query) else emptyList()
 		}
-		val page = (offset / 60f).toIntUp()
 		val url = buildString {
 			append("https://")
 			append(getDomain())

src/main/kotlin/org/koitharu/kotatsu/parsers/util/Paginator.kt

@@ -0,0 +1,25 @@
+package org.koitharu.kotatsu.parsers.util
+
+import androidx.collection.SparseArrayCompat
+import androidx.collection.set
+
+class Paginator constructor(private val initialPageSize: Int) {
+
+	var firstPage = 1
+	private var pages = SparseArrayCompat<Int>()
+
+	fun getPage(offset: Int): Int {
+		if (offset == 0) { // just an optimization
+			return firstPage
+		}
+		pages[offset]?.let { return it }
+		val pageSize = initialPageSize
+		val intPage = offset / pageSize
+		val tail = offset % pageSize
+		return intPage + firstPage + if (tail == 0) 0 else 1
+	}
+
+	fun onListReceived(offset: Int, page: Int, count: Int) {
+		pages[offset + count] = if (count > 0) page + 1 else page
+	}
+}

src/test/kotlin/org/koitharu/kotatsu/parsers/MangaParserTest.kt

@@ -3,7 +3,6 @@ package org.koitharu.kotatsu.parsers
 import kotlinx.coroutines.test.runTest
 import okhttp3.HttpUrl
 import org.junit.jupiter.api.Disabled
-import org.junit.jupiter.api.Test
 import org.junit.jupiter.api.extension.ExtendWith
 import org.junit.jupiter.params.ParameterizedTest
 import org.koitharu.kotatsu.parsers.model.Manga
@@ -11,7 +10,10 @@ import org.koitharu.kotatsu.parsers.model.MangaSource
 import org.koitharu.kotatsu.parsers.model.SortOrder
 import org.koitharu.kotatsu.parsers.util.medianOrNull
 import org.koitharu.kotatsu.parsers.util.mimeType
-import org.koitharu.kotatsu.test_util.*
+import org.koitharu.kotatsu.test_util.isDistinct
+import org.koitharu.kotatsu.test_util.isDistinctBy
+import org.koitharu.kotatsu.test_util.isUrlAbsolute
+import org.koitharu.kotatsu.test_util.maxDuplicates
 
 
 @ExtendWith(AuthCheckExtension::class)
@@ -19,24 +21,6 @@ internal class MangaParserTest {
 
 	private val context = MangaLoaderContextMock()
 
-	@Test
-	fun singleTest() = runTest {
-		val manga = mangaOf(MangaSource.MANGALIB, "https://mangalib.me/dorohedoro")
-		val parser = manga.source.newParser(context)
-		val details = parser.getDetails(manga)
-		val chapter = details.chapters!!.first()
-		val pages = parser.getPages(chapter)
-
-		assert(pages.isNotEmpty())
-		assert(pages.isDistinctBy { it.id })
-
-		val page = pages.medianOrNull() ?: error("No page")
-		val pageUrl = parser.getPageUrl(page)
-		assert(pageUrl.isNotEmpty())
-		assert(pageUrl.isUrlAbsolute())
-		checkImageRequest(pageUrl, page.referer)
-	}
-
 	@ParameterizedTest(name = "{index}|list|{0}")
 	@MangaSources
 	fun list(source: MangaSource) = runTest {
@@ -46,6 +30,18 @@ internal class MangaParserTest {
 		assert(list.all { it.source == source })
 	}
 
+	@ParameterizedTest(name = "{index}|pagination|{0}")
+	@MangaSources
+	fun pagination(source: MangaSource) = runTest {
+		val parser = source.newParser(context)
+		val page1 = parser.getList(0, sortOrder = null, tags = null)
+		val page2 = parser.getList(page1.size, sortOrder = null, tags = null)
+		val intersection = page1.intersect(page2.toSet())
+		assert(intersection.isEmpty()) {
+			"Pages are intersected: " + intersection.joinToString { it.publicUrl }
+		}
+	}
+
 	@ParameterizedTest(name = "{index}|search|{0}")
 	@MangaSources
 	fun search(source: MangaSource) = runTest {

src/test/kotlin/org/koitharu/kotatsu/parsers/util/PaginatorTest.kt

@@ -0,0 +1,42 @@
+package org.koitharu.kotatsu.parsers.util
+
+import org.junit.jupiter.api.Assertions.assertEquals
+import org.junit.jupiter.api.Test
+
+internal class PaginatorTest {
+
+	@Test
+	fun singlePaginationTest() {
+		val paginator = Paginator(24)
+		assertEquals(1, paginator.getPage(0))
+		assertEquals(2, paginator.getPage(24))
+		assertEquals(3, paginator.getPage(48))
+	}
+
+	@Test
+	fun adaptivePaginationTest() {
+		val paginator = Paginator(12)
+		assertEquals(1, paginator.getPage(0))
+		paginator.onListReceived(0, 1, 24)
+		assertEquals(2, paginator.getPage(24))
+		paginator.onListReceived(24, 2, 18)
+		assertEquals(3, paginator.getPage(42))
+	}
+
+	@Test
+	fun endReachPaginationTest() {
+		val pageSize = 24
+		val paginator = Paginator(pageSize)
+		var size = 0
+		repeat(5) { i ->
+			val offset = i * pageSize
+			assertEquals(i + 1, paginator.getPage(offset))
+			paginator.onListReceived(offset, i + 1, pageSize)
+			size += pageSize
+		}
+		val nextPage = paginator.getPage(size)
+		assertEquals(nextPage, paginator.getPage(size))
+		paginator.onListReceived(size, nextPage, 0)
+		assertEquals(nextPage, paginator.getPage(size))
+	}
+}