diff --git a/.idea/runConfigurations/TestsAndReport.xml b/.idea/runConfigurations/TestsAndReport.xml deleted file mode 100644 index 614cfdf3..00000000 --- a/.idea/runConfigurations/TestsAndReport.xml +++ /dev/null @@ -1,7 +0,0 @@ - - - - - - - \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaParser.kt index c229f4f8..8bbca385 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaParser.kt @@ -69,7 +69,7 @@ abstract class MangaParser @InternalParsersApi constructor(val source: MangaSour * @param offset starting from 0 and used for pagination. * @param query search query */ - suspend fun getList(offset: Int, query: String): List { + open suspend fun getList(offset: Int, query: String): List { return getList(offset, query, null, defaultSortOrder) } @@ -81,7 +81,7 @@ abstract class MangaParser @InternalParsersApi constructor(val source: MangaSour * @param tags genres for filtering, values from [getTags] and [Manga.tags]. May be null or empty * @param sortOrder one of [sortOrders] or null for default value */ - suspend fun getList(offset: Int, tags: Set?, sortOrder: SortOrder?): List { + open suspend fun getList(offset: Int, tags: Set?, sortOrder: SortOrder?): List { return getList(offset, null, tags, sortOrder ?: defaultSortOrder) } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/PagedMangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/PagedMangaParser.kt new file mode 100644 index 00000000..7040bce4 --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/PagedMangaParser.kt @@ -0,0 +1,50 @@ +package org.koitharu.kotatsu.parsers + +import org.koitharu.kotatsu.parsers.model.Manga +import org.koitharu.kotatsu.parsers.model.MangaSource +import org.koitharu.kotatsu.parsers.model.MangaTag +import org.koitharu.kotatsu.parsers.model.SortOrder +import org.koitharu.kotatsu.parsers.util.Paginator + +@InternalParsersApi +abstract class PagedMangaParser( + source: MangaSource, + pageSize: Int, + searchPageSize: Int = pageSize, +) : MangaParser(source) { + + protected val paginator = Paginator(pageSize) + protected val searchPaginator = Paginator(searchPageSize) + + override suspend fun getList(offset: Int, query: String): List { + return getList(searchPaginator, offset, query, null, defaultSortOrder) + } + + override suspend fun getList(offset: Int, tags: Set?, sortOrder: SortOrder?): List { + return getList(paginator, offset, null, tags, sortOrder ?: defaultSortOrder) + } + + @InternalParsersApi + @Deprecated("You should use getListPage for PagedMangaParser", level = DeprecationLevel.HIDDEN) + final override suspend fun getList( + offset: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List = throw UnsupportedOperationException("You should use getListPage for PagedMangaParser") + + abstract suspend fun getListPage(page: Int, query: String?, tags: Set?, sortOrder: SortOrder): List + + private suspend fun getList( + paginator: Paginator, + offset: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + val page = paginator.getPage(offset) + val list = getListPage(page, query, tags, sortOrder) + paginator.onListReceived(offset, page, list.size) + return list + } +} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BatoToParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BatoToParser.kt index 097725ff..058eb1c7 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BatoToParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BatoToParser.kt @@ -5,8 +5,8 @@ import org.json.JSONArray import org.json.JSONObject import org.jsoup.nodes.Element import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaParser import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.PagedMangaParser import org.koitharu.kotatsu.parsers.config.ConfigKey import org.koitharu.kotatsu.parsers.model.* import org.koitharu.kotatsu.parsers.util.* @@ -17,11 +17,12 @@ import javax.crypto.Cipher import javax.crypto.spec.IvParameterSpec import javax.crypto.spec.SecretKeySpec -private const val PAGE_SIZE = 60 -private const val PAGE_SIZE_SEARCH = 20 - @MangaSourceParser("BATOTO", "Bato.To") -internal class BatoToParser(override val context: MangaLoaderContext) : MangaParser(MangaSource.BATOTO) { +internal class BatoToParser(override val context: MangaLoaderContext) : PagedMangaParser( + source = MangaSource.BATOTO, + pageSize = 60, + searchPageSize = 20, +) { override val sortOrders: Set = EnumSet.of( SortOrder.NEWEST, @@ -35,17 +36,15 @@ internal class BatoToParser(override val context: MangaLoaderContext) : MangaPar arrayOf("bato.to", "mto.to", "mangatoto.com", "battwo.com", "batotwo.com", "comiko.net", "batotoo.com"), ) - override suspend fun getList( - offset: Int, + override suspend fun getListPage( + page: Int, query: String?, tags: Set?, sortOrder: SortOrder, ): List { if (!query.isNullOrEmpty()) { - return search(offset, query) + return search(page, query) } - val page = (offset / PAGE_SIZE) + 1 - @Suppress("NON_EXHAUSTIVE_WHEN_STATEMENT") val url = buildString { append("https://") @@ -159,8 +158,7 @@ internal class BatoToParser(override val context: MangaLoaderContext) : MangaPar override fun getFaviconUrl(): String = "https://styles.amarkcdn.com/img/batoto/favicon.ico?v0" - private suspend fun search(offset: Int, query: String): List { - val page = (offset / PAGE_SIZE_SEARCH) + 1 + private suspend fun search(page: Int, query: String): List { val url = buildString { append("https://") append(getDomain()) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/DesuMeParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/DesuMeParser.kt index 5b759883..76ff88f5 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/DesuMeParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/DesuMeParser.kt @@ -1,8 +1,8 @@ package org.koitharu.kotatsu.parsers.site import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaParser import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.PagedMangaParser import org.koitharu.kotatsu.parsers.config.ConfigKey import org.koitharu.kotatsu.parsers.exception.ParseException import org.koitharu.kotatsu.parsers.model.* @@ -13,7 +13,7 @@ import org.koitharu.kotatsu.parsers.util.json.mapJSONToSet import java.util.* @MangaSourceParser("DESUME", "Desu.me", "ru") -internal class DesuMeParser(override val context: MangaLoaderContext) : MangaParser(MangaSource.DESUME) { +internal class DesuMeParser(override val context: MangaLoaderContext) : PagedMangaParser(MangaSource.DESUME, 20) { override val configKeyDomain = ConfigKey.Domain("desu.me", null) @@ -24,13 +24,13 @@ internal class DesuMeParser(override val context: MangaLoaderContext) : MangaPar SortOrder.ALPHABETICAL, ) - override suspend fun getList( - offset: Int, + override suspend fun getListPage( + page: Int, query: String?, tags: Set?, sortOrder: SortOrder, ): List { - if (query != null && offset != 0) { + if (query != null && page != searchPaginator.firstPage) { return emptyList() } val domain = getDomain() @@ -40,7 +40,7 @@ internal class DesuMeParser(override val context: MangaLoaderContext) : MangaPar append("/manga/api/?limit=20&order=") append(getSortKey(sortOrder)) append("&page=") - append((offset / 20) + 1) + append(page) if (!tags.isNullOrEmpty()) { append("&genres=") appendAll(tags, ",") { it.key } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ExHentaiParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ExHentaiParser.kt index 9c841ea9..f7628cc9 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ExHentaiParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ExHentaiParser.kt @@ -2,9 +2,9 @@ package org.koitharu.kotatsu.parsers.site import org.jsoup.nodes.Element import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaParser import org.koitharu.kotatsu.parsers.MangaParserAuthProvider import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.PagedMangaParser import org.koitharu.kotatsu.parsers.config.ConfigKey import org.koitharu.kotatsu.parsers.exception.AuthRequiredException import org.koitharu.kotatsu.parsers.exception.ParseException @@ -19,7 +19,7 @@ private const val DOMAIN_AUTHORIZED = "exhentai.org" @MangaSourceParser("EXHENTAI", "ExHentai") internal class ExHentaiParser( override val context: MangaLoaderContext, -) : MangaParser(MangaSource.EXHENTAI), MangaParserAuthProvider { +) : PagedMangaParser(MangaSource.EXHENTAI, pageSize = 25), MangaParserAuthProvider { override val sortOrders: Set = Collections.singleton( SortOrder.NEWEST, @@ -57,13 +57,12 @@ internal class ExHentaiParser( context.cookieJar.insertCookies(DOMAIN_UNAUTHORIZED, "nw=1", "sl=dm_2") } - override suspend fun getList( - offset: Int, + override suspend fun getListPage( + page: Int, query: String?, tags: Set?, sortOrder: SortOrder, ): List { - val page = (offset / 25f).toIntUp() var search = query?.urlEncoded().orEmpty() val url = buildString { append("https://") @@ -98,7 +97,7 @@ internal class ExHentaiParser( parseFailed("Cannot find root") } else { updateDm = true - return getList(offset, query, tags, sortOrder) + return getListPage(page, query, tags, sortOrder) } updateDm = false return root.children().mapNotNull { tr -> diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaInUaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaInUaParser.kt index ead9a911..51308e56 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaInUaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaInUaParser.kt @@ -1,8 +1,8 @@ package org.koitharu.kotatsu.parsers.site import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaParser import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.PagedMangaParser import org.koitharu.kotatsu.parsers.config.ConfigKey import org.koitharu.kotatsu.parsers.model.* import org.koitharu.kotatsu.parsers.util.* @@ -12,26 +12,28 @@ import java.util.* private const val DEF_BRANCH_NAME = "Основний переклад" @MangaSourceParser("MANGAINUA", "MANGA/in/UA", "uk") -class MangaInUaParser(override val context: MangaLoaderContext) : MangaParser(MangaSource.MANGAINUA) { +class MangaInUaParser(override val context: MangaLoaderContext) : PagedMangaParser( + source = MangaSource.MANGAINUA, + pageSize = 24, + searchPageSize = 10, +) { override val sortOrders: Set get() = Collections.singleton(SortOrder.UPDATED) override val configKeyDomain: ConfigKey.Domain = ConfigKey.Domain("manga.in.ua", null) - override suspend fun getList( - offset: Int, + override suspend fun getListPage( + page: Int, query: String?, tags: Set?, sortOrder: SortOrder, ): List { - val page = (offset / 24f).toIntUp().inc() - val searchPage = (offset / 10f).toIntUp().inc() val url = when { !query.isNullOrEmpty() -> ( "/index.php?do=search" + "&subaction=search" + - "&search_start=$searchPage" + + "&search_start=$page" + "&full_search=1" + "&story=$query" + "&titleonly=3" diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NHentaiParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NHentaiParser.kt index b664acfd..f1d12996 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NHentaiParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NHentaiParser.kt @@ -6,8 +6,8 @@ import kotlinx.coroutines.awaitAll import kotlinx.coroutines.coroutineScope import org.jsoup.nodes.Element import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaParser import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.PagedMangaParser import org.koitharu.kotatsu.parsers.config.ConfigKey import org.koitharu.kotatsu.parsers.model.* import org.koitharu.kotatsu.parsers.util.* @@ -15,7 +15,7 @@ import java.text.SimpleDateFormat import java.util.* @MangaSourceParser("NHENTAI", "N-Hentai") -class NHentaiParser(override val context: MangaLoaderContext) : MangaParser(MangaSource.NHENTAI) { +class NHentaiParser(override val context: MangaLoaderContext) : PagedMangaParser(MangaSource.NHENTAI, pageSize = 25) { override val configKeyDomain: ConfigKey.Domain get() = ConfigKey.Domain("nhentai.net", null) @@ -23,17 +23,16 @@ class NHentaiParser(override val context: MangaLoaderContext) : MangaParser(Mang override val sortOrders: Set get() = EnumSet.of(SortOrder.NEWEST, SortOrder.POPULARITY) - override suspend fun getList( - offset: Int, + override suspend fun getListPage( + page: Int, query: String?, tags: Set?, sortOrder: SortOrder, ): List { if (query.isNullOrEmpty() && tags != null && tags.size > 1) { - return getList(offset, buildQuery(tags), emptySet(), sortOrder) + return getListPage(page, buildQuery(tags), emptySet(), sortOrder) } val domain = getDomain() - val page = (offset / 25) + 1 val url = buildString { append("https://") append(domain) @@ -101,7 +100,7 @@ class NHentaiParser(override val context: MangaLoaderContext) : MangaParser(Mang override suspend fun getDetails(manga: Manga): Manga { val root = context.httpGet( - url = manga.url.toAbsoluteUrl(getDomain()) + url = manga.url.toAbsoluteUrl(getDomain()), ).parseHtml().body().requireElementById("bigcontainer") val img = root.requireElementById("cover").selectFirstOrThrow("img") val tagContainers = root.requireElementById("tags").select(".tag-container") @@ -126,12 +125,12 @@ class NHentaiParser(override val context: MangaLoaderContext) : MangaParser(Mang uploadDate = dateFormat.tryParse( tagContainers.find { x -> x.ownText() == "Uploaded:" } ?.selectFirst("time") - ?.attr("datetime") + ?.attr("datetime"), ), branch = null, source = source, - ) - ) + ), + ), ) } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NineMangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NineMangaParser.kt index 26eee91b..5cab866b 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NineMangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NineMangaParser.kt @@ -3,8 +3,8 @@ package org.koitharu.kotatsu.parsers.site import okhttp3.Headers import okhttp3.HttpUrl.Companion.toHttpUrl import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaParser import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.PagedMangaParser import org.koitharu.kotatsu.parsers.config.ConfigKey import org.koitharu.kotatsu.parsers.exception.ParseException import org.koitharu.kotatsu.parsers.model.* @@ -12,13 +12,11 @@ import org.koitharu.kotatsu.parsers.util.* import java.text.SimpleDateFormat import java.util.* -private const val PAGE_SIZE = 26 - internal abstract class NineMangaParser( final override val context: MangaLoaderContext, source: MangaSource, defaultDomain: String, -) : MangaParser(source) { +) : PagedMangaParser(source, pageSize = 26) { override val configKeyDomain = ConfigKey.Domain(defaultDomain, null) @@ -34,13 +32,12 @@ internal abstract class NineMangaParser( SortOrder.POPULARITY, ) - override suspend fun getList( - offset: Int, + override suspend fun getListPage( + page: Int, query: String?, tags: Set?, sortOrder: SortOrder, ): List { - val page = (offset / PAGE_SIZE.toFloat()).toIntUp() + 1 val url = buildString { append("https://") append(getDomain()) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/RemangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/RemangaParser.kt index 82695c3e..4c585c11 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/RemangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/RemangaParser.kt @@ -6,9 +6,9 @@ import org.json.JSONArray import org.json.JSONException import org.json.JSONObject import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaParser import org.koitharu.kotatsu.parsers.MangaParserAuthProvider import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.PagedMangaParser import org.koitharu.kotatsu.parsers.config.ConfigKey import org.koitharu.kotatsu.parsers.exception.AuthRequiredException import org.koitharu.kotatsu.parsers.exception.ParseException @@ -31,7 +31,7 @@ private const val STATUS_FINISHED = 0 @MangaSourceParser("REMANGA", "Remanga", "ru") internal class RemangaParser( override val context: MangaLoaderContext, -) : MangaParser(MangaSource.REMANGA), MangaParserAuthProvider { +) : PagedMangaParser(MangaSource.REMANGA, PAGE_SIZE), MangaParserAuthProvider { override val configKeyDomain = ConfigKey.Domain("remanga.org", null) override val authUrl: String @@ -53,8 +53,8 @@ internal class RemangaParser( private val regexLastUrlPath = Regex("/[^/]+/?$") - override suspend fun getList( - offset: Int, + override suspend fun getListPage( + page: Int, query: String?, tags: Set?, sortOrder: SortOrder, @@ -77,7 +77,7 @@ internal class RemangaParser( } urlBuilder .append("&page=") - .append((offset / PAGE_SIZE) + 1) + .append(page) .append("&count=") .append(PAGE_SIZE) val content = context.httpGet(urlBuilder.toString(), getApiHeaders()).parseJson() diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Madara5Parser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Madara5Parser.kt index a408ee0e..2d641f94 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Madara5Parser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Madara5Parser.kt @@ -4,8 +4,8 @@ import androidx.collection.arraySetOf import org.jsoup.nodes.Element import org.koitharu.kotatsu.parsers.InternalParsersApi import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaParser import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.PagedMangaParser import org.koitharu.kotatsu.parsers.config.ConfigKey import org.koitharu.kotatsu.parsers.model.* import org.koitharu.kotatsu.parsers.util.* @@ -16,9 +16,8 @@ abstract class Madara5Parser @InternalParsersApi constructor( override val context: MangaLoaderContext, source: MangaSource, domain: String, -) : MangaParser(source) { +) : PagedMangaParser(source, pageSize = 22) { - protected open val pageSize = 22 protected open val tagPrefix = "/mangas/" protected open val nsfwTags = arraySetOf("yaoi", "yuri", "mature") @@ -26,9 +25,12 @@ abstract class Madara5Parser @InternalParsersApi constructor( override val configKeyDomain = ConfigKey.Domain(domain, null) - @InternalParsersApi - override suspend fun getList(offset: Int, query: String?, tags: Set?, sortOrder: SortOrder): List { - val page = (offset / pageSize.toFloat()).toIntUp() + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { val domain = getDomain() val url = buildString { append("https://") diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt index f2b885c2..b407b59a 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt @@ -2,8 +2,8 @@ package org.koitharu.kotatsu.parsers.site.madara import org.jsoup.nodes.Element import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaParser import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.PagedMangaParser import org.koitharu.kotatsu.parsers.config.ConfigKey import org.koitharu.kotatsu.parsers.exception.ParseException import org.koitharu.kotatsu.parsers.model.* @@ -12,13 +12,11 @@ import java.text.DateFormat import java.text.SimpleDateFormat import java.util.* -private const val PAGE_SIZE = 12 - internal abstract class MadaraParser( override val context: MangaLoaderContext, source: MangaSource, domain: String, -) : MangaParser(source) { +) : PagedMangaParser(source, pageSize = 12) { override val configKeyDomain = ConfigKey.Domain(domain, null) @@ -30,22 +28,27 @@ internal abstract class MadaraParser( protected open val tagPrefix = "manga-genre/" protected open val isNsfwSource = false - override suspend fun getList( - offset: Int, + init { + paginator.firstPage = 0 + searchPaginator.firstPage = 0 + } + + override suspend fun getListPage( + page: Int, query: String?, tags: Set?, sortOrder: SortOrder, ): List { val tag = tags.oneOrThrowIfMany() val payload = createRequestTemplate() - payload["page"] = (offset / PAGE_SIZE.toFloat()).toIntUp().toString() + payload["page"] = page.toString() payload["vars[meta_key]"] = when (sortOrder) { SortOrder.POPULARITY -> "_wp_manga_views" SortOrder.UPDATED -> "_latest_update" else -> "_wp_manga_views" } payload["vars[wp-manga-genre]"] = tag?.key.orEmpty() - payload["vars[s]"] = query.orEmpty() + payload["vars[s]"] = query?.urlEncoded().orEmpty() val doc = context.httpPost( "https://${getDomain()}/wp-admin/admin-ajax.php", payload, diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/rulib/MangaLibParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/rulib/MangaLibParser.kt index 1235748d..59858ae3 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/rulib/MangaLibParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/rulib/MangaLibParser.kt @@ -5,9 +5,9 @@ import org.json.JSONArray import org.json.JSONObject import org.jsoup.nodes.Document import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaParser import org.koitharu.kotatsu.parsers.MangaParserAuthProvider import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.PagedMangaParser import org.koitharu.kotatsu.parsers.config.ConfigKey import org.koitharu.kotatsu.parsers.exception.AuthRequiredException import org.koitharu.kotatsu.parsers.exception.ParseException @@ -22,7 +22,7 @@ import java.util.* internal open class MangaLibParser( override val context: MangaLoaderContext, source: MangaSource, -) : MangaParser(source), MangaParserAuthProvider { +) : PagedMangaParser(source, pageSize = 60), MangaParserAuthProvider { override val configKeyDomain = ConfigKey.Domain("mangalib.me", null) @@ -37,16 +37,15 @@ internal open class MangaLibParser( SortOrder.NEWEST, ) - override suspend fun getList( - offset: Int, + override suspend fun getListPage( + page: Int, query: String?, tags: Set?, sortOrder: SortOrder, ): List { if (!query.isNullOrEmpty()) { - return if (offset == 0) search(query) else emptyList() + return if (page == searchPaginator.firstPage) search(query) else emptyList() } - val page = (offset / 60f).toIntUp() val url = buildString { append("https://") append(getDomain()) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Paginator.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Paginator.kt new file mode 100644 index 00000000..5e41bd80 --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Paginator.kt @@ -0,0 +1,25 @@ +package org.koitharu.kotatsu.parsers.util + +import androidx.collection.SparseArrayCompat +import androidx.collection.set + +class Paginator constructor(private val initialPageSize: Int) { + + var firstPage = 1 + private var pages = SparseArrayCompat() + + fun getPage(offset: Int): Int { + if (offset == 0) { // just an optimization + return firstPage + } + pages[offset]?.let { return it } + val pageSize = initialPageSize + val intPage = offset / pageSize + val tail = offset % pageSize + return intPage + firstPage + if (tail == 0) 0 else 1 + } + + fun onListReceived(offset: Int, page: Int, count: Int) { + pages[offset + count] = if (count > 0) page + 1 else page + } +} \ No newline at end of file diff --git a/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaParserTest.kt b/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaParserTest.kt index bb7a1725..ee7dc809 100644 --- a/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaParserTest.kt +++ b/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaParserTest.kt @@ -3,7 +3,6 @@ package org.koitharu.kotatsu.parsers import kotlinx.coroutines.test.runTest import okhttp3.HttpUrl import org.junit.jupiter.api.Disabled -import org.junit.jupiter.api.Test import org.junit.jupiter.api.extension.ExtendWith import org.junit.jupiter.params.ParameterizedTest import org.koitharu.kotatsu.parsers.model.Manga @@ -11,7 +10,10 @@ import org.koitharu.kotatsu.parsers.model.MangaSource import org.koitharu.kotatsu.parsers.model.SortOrder import org.koitharu.kotatsu.parsers.util.medianOrNull import org.koitharu.kotatsu.parsers.util.mimeType -import org.koitharu.kotatsu.test_util.* +import org.koitharu.kotatsu.test_util.isDistinct +import org.koitharu.kotatsu.test_util.isDistinctBy +import org.koitharu.kotatsu.test_util.isUrlAbsolute +import org.koitharu.kotatsu.test_util.maxDuplicates @ExtendWith(AuthCheckExtension::class) @@ -19,24 +21,6 @@ internal class MangaParserTest { private val context = MangaLoaderContextMock() - @Test - fun singleTest() = runTest { - val manga = mangaOf(MangaSource.MANGALIB, "https://mangalib.me/dorohedoro") - val parser = manga.source.newParser(context) - val details = parser.getDetails(manga) - val chapter = details.chapters!!.first() - val pages = parser.getPages(chapter) - - assert(pages.isNotEmpty()) - assert(pages.isDistinctBy { it.id }) - - val page = pages.medianOrNull() ?: error("No page") - val pageUrl = parser.getPageUrl(page) - assert(pageUrl.isNotEmpty()) - assert(pageUrl.isUrlAbsolute()) - checkImageRequest(pageUrl, page.referer) - } - @ParameterizedTest(name = "{index}|list|{0}") @MangaSources fun list(source: MangaSource) = runTest { @@ -46,6 +30,18 @@ internal class MangaParserTest { assert(list.all { it.source == source }) } + @ParameterizedTest(name = "{index}|pagination|{0}") + @MangaSources + fun pagination(source: MangaSource) = runTest { + val parser = source.newParser(context) + val page1 = parser.getList(0, sortOrder = null, tags = null) + val page2 = parser.getList(page1.size, sortOrder = null, tags = null) + val intersection = page1.intersect(page2.toSet()) + assert(intersection.isEmpty()) { + "Pages are intersected: " + intersection.joinToString { it.publicUrl } + } + } + @ParameterizedTest(name = "{index}|search|{0}") @MangaSources fun search(source: MangaSource) = runTest { diff --git a/src/test/kotlin/org/koitharu/kotatsu/parsers/util/PaginatorTest.kt b/src/test/kotlin/org/koitharu/kotatsu/parsers/util/PaginatorTest.kt new file mode 100644 index 00000000..7b7fe70e --- /dev/null +++ b/src/test/kotlin/org/koitharu/kotatsu/parsers/util/PaginatorTest.kt @@ -0,0 +1,42 @@ +package org.koitharu.kotatsu.parsers.util + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test + +internal class PaginatorTest { + + @Test + fun singlePaginationTest() { + val paginator = Paginator(24) + assertEquals(1, paginator.getPage(0)) + assertEquals(2, paginator.getPage(24)) + assertEquals(3, paginator.getPage(48)) + } + + @Test + fun adaptivePaginationTest() { + val paginator = Paginator(12) + assertEquals(1, paginator.getPage(0)) + paginator.onListReceived(0, 1, 24) + assertEquals(2, paginator.getPage(24)) + paginator.onListReceived(24, 2, 18) + assertEquals(3, paginator.getPage(42)) + } + + @Test + fun endReachPaginationTest() { + val pageSize = 24 + val paginator = Paginator(pageSize) + var size = 0 + repeat(5) { i -> + val offset = i * pageSize + assertEquals(i + 1, paginator.getPage(offset)) + paginator.onListReceived(offset, i + 1, pageSize) + size += pageSize + } + val nextPage = paginator.getPage(size) + assertEquals(nextPage, paginator.getPage(size)) + paginator.onListReceived(size, nextPage, 0) + assertEquals(nextPage, paginator.getPage(size)) + } +} \ No newline at end of file