KotatsuApp/kotatsu-parsers
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Refactor getList function

Browse Source
This commit is contained in:
Koitharu
2022-05-21 15:11:51 +03:00
parent 22baf09cc3
commit d724460332
21 changed files with 108 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
kotatsu-parsers-ksp/src/main/kotlin/org/koitharu/kotatsu/parsers/ksp/ParserProcessor.kt
View File

@@ -73,8 +73,6 @@ class ParserProcessor(
"""
package org.koitharu.kotatsu.parsers.model
import org.koitharu.kotatsu.parsers.model.MangaSource
enum class MangaSource(
val title: String,
val locale: String?,

69
src/main/kotlin/org/koitharu/kotatsu/parsers/MangaParser.kt
View File

@@ -7,7 +7,7 @@ import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl
import java.util.*
abstract class MangaParser(val source: MangaSource) {
abstract class MangaParser @InternalParsersApi constructor(val source: MangaSource) {
protected abstract val context: MangaLoaderContext
@@ -27,6 +27,15 @@ abstract class MangaParser(val source: MangaSource) {
*/
protected abstract val configKeyDomain: ConfigKey.Domain
/**
* Used as fallback if value of `sortOrder` passed to [getList] is null
*/
protected open val defaultSortOrder: SortOrder
get() {
val supported = sortOrders
return SortOrder.values().first { it in supported }
}
/**
* Parse list of manga by specified criteria
*
@@ -36,13 +45,36 @@ abstract class MangaParser(val source: MangaSource) {
* @param tags genres for filtering, values from [getTags] and [Manga.tags]. May be null or empty
* @param sortOrder one of [sortOrders] or null for default value
*/
@InternalParsersApi
abstract suspend fun getList(
offset: Int,
query: String? = null,
tags: Set<MangaTag>? = null,
sortOrder: SortOrder? = null,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga>
/**
* Parse list of manga with search by text query
*
* @param offset starting from 0 and used for pagination.
* @param query search query
*/
suspend fun getList(offset: Int, query: String): List<Manga> {
return getList(offset, query, null, defaultSortOrder)
}
/**
* Parse list of manga by specified criteria
*
* @param offset starting from 0 and used for pagination.
* Note than passed value may not be divisible by internal page size, so you should adjust it manually.
* @param tags genres for filtering, values from [getTags] and [Manga.tags]. May be null or empty
* @param sortOrder one of [sortOrders] or null for default value
*/
suspend fun getList(offset: Int, tags: Set<MangaTag>?, sortOrder: SortOrder?): List<Manga> {
return getList(offset, null, tags, sortOrder ?: defaultSortOrder)
}
/**
* Parse details for [Manga]: chapters list, description, large cover, etc.
* Must return the same manga, may change any fields excepts id, url and source
@@ -121,12 +153,29 @@ abstract class MangaParser(val source: MangaSource) {
/**
* Convert relative url to an absolute using [getDomain]
*/
protected fun String.withDomain(subdomain: String? = null): String {
var domain = getDomain()
if (subdomain != null) {
domain = subdomain + "." + domain.removePrefix("www.")
}
return toAbsoluteUrl(domain)
@Deprecated(
message = "Use toAbsoluteUrl() instead",
replaceWith = ReplaceWith(
"toAbsoluteUrl(getDomain(), subdomain)",
"org.koitharu.kotatsu.parsers.util.toAbsoluteUrl",
),
)
protected fun String.withDomain(subdomain: String): String {
return toAbsoluteUrl(getDomain(), subdomain)
}
/**
* Convert relative url to an absolute using [getDomain]
*/
@Deprecated(
message = "Use toAbsoluteUrl() instead",
replaceWith = ReplaceWith(
"toAbsoluteUrl(getDomain())",
"org.koitharu.kotatsu.parsers.util.toAbsoluteUrl",
),
)
protected fun String.withDomain(): String {
return toAbsoluteUrl(getDomain())
}
@InternalParsersApi

2
src/main/kotlin/org/koitharu/kotatsu/parsers/site/AnibelParser.kt
View File

@@ -30,7 +30,7 @@ internal class AnibelParser(override val context: MangaLoaderContext) : MangaPar
offset: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder?,
sortOrder: SortOrder,
): List<Manga> {
if (!query.isNullOrEmpty()) {
return if (offset == 0) {

3
src/main/kotlin/org/koitharu/kotatsu/parsers/site/BatoToParser.kt
View File

@@ -39,7 +39,7 @@ internal class BatoToParser(override val context: MangaLoaderContext) : MangaPar
offset: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder?,
sortOrder: SortOrder,
): List<Manga> {
if (!query.isNullOrEmpty()) {
return search(offset, query)
@@ -52,7 +52,6 @@ internal class BatoToParser(override val context: MangaLoaderContext) : MangaPar
append(getDomain())
append("/browse?sort=")
when (sortOrder) {
null,
SortOrder.UPDATED,
-> append("update.za")
SortOrder.POPULARITY -> append("views_a.za")

10
src/main/kotlin/org/koitharu/kotatsu/parsers/site/ChanParser.kt
View File

@@ -19,7 +19,7 @@ internal abstract class ChanParser(source: MangaSource) : MangaParser(source) {
offset: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder?,
sortOrder: SortOrder,
): List<Manga> {
val domain = getDomain()
val url = when {
@@ -143,16 +143,16 @@ internal abstract class ChanParser(source: MangaSource) : MangaParser(source) {
}
}
private fun getSortKey(sortOrder: SortOrder?) =
when (sortOrder ?: sortOrders.minByOrNull { it.ordinal }) {
private fun getSortKey(sortOrder: SortOrder) =
when (sortOrder) {
SortOrder.ALPHABETICAL -> "catalog"
SortOrder.POPULARITY -> "mostfavorites"
SortOrder.NEWEST -> "manga/new"
else -> "mostfavorites"
}
private fun getSortKey2(sortOrder: SortOrder?) =
when (sortOrder ?: sortOrders.minByOrNull { it.ordinal }) {
private fun getSortKey2(sortOrder: SortOrder) =
when (sortOrder) {
SortOrder.ALPHABETICAL -> "abcasc"
SortOrder.POPULARITY -> "favdesc"
SortOrder.NEWEST -> "datedesc"

2
src/main/kotlin/org/koitharu/kotatsu/parsers/site/ComickFunParser.kt
View File

@@ -39,7 +39,7 @@ internal class ComickFunParser(override val context: MangaLoaderContext) : Manga
offset: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder?,
sortOrder: SortOrder,
): List<Manga> {
val domain = getDomain()
val url = buildString {

4
src/main/kotlin/org/koitharu/kotatsu/parsers/site/DesuMeParser.kt
View File

@@ -28,7 +28,7 @@ internal class DesuMeParser(override val context: MangaLoaderContext) : MangaPar
offset: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder?,
sortOrder: SortOrder,
): List<Manga> {
if (query != null && offset != 0) {
return emptyList()
@@ -149,7 +149,7 @@ internal class DesuMeParser(override val context: MangaLoaderContext) : MangaPar
}
}
private fun getSortKey(sortOrder: SortOrder?) =
private fun getSortKey(sortOrder: SortOrder) =
when (sortOrder) {
SortOrder.ALPHABETICAL -> "name"
SortOrder.POPULARITY -> "popular"

2
src/main/kotlin/org/koitharu/kotatsu/parsers/site/ExHentaiParser.kt
View File

@@ -61,7 +61,7 @@ internal class ExHentaiParser(
offset: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder?,
sortOrder: SortOrder,
): List<Manga> {
val page = (offset / 25f).toIntUp()
var search = query?.urlEncoded().orEmpty()

16
src/main/kotlin/org/koitharu/kotatsu/parsers/site/GroupleParser.kt
View File

@@ -5,7 +5,6 @@ import okhttp3.HttpUrl.Companion.toHttpUrl
import okhttp3.Response
import org.json.JSONArray
import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.exception.ParseException
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import org.koitharu.kotatsu.parsers.util.json.mapJSON
@@ -34,7 +33,7 @@ internal abstract class GroupleParser(source: MangaSource, userAgent: String) :
offset: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder?,
sortOrder: SortOrder,
): List<Manga> {
val domain = getDomain()
val doc = when {
@@ -48,17 +47,13 @@ internal abstract class GroupleParser(source: MangaSource, userAgent: String) :
)
tags.isNullOrEmpty() -> context.httpGet(
"https://$domain/list?sortType=${
getSortKey(
sortOrder,
)
getSortKey(sortOrder)
}&offset=${offset upBy PAGE_SIZE}",
headers,
)
tags.size == 1 -> context.httpGet(
"https://$domain/list/genre/${tags.first().key}?sortType=${
getSortKey(
sortOrder,
)
getSortKey(sortOrder)
}&offset=${offset upBy PAGE_SIZE}",
headers,
)
@@ -234,14 +229,13 @@ internal abstract class GroupleParser(source: MangaSource, userAgent: String) :
}
}
private fun getSortKey(sortOrder: SortOrder?) =
when (sortOrder ?: sortOrders.minByOrNull { it.ordinal }) {
private fun getSortKey(sortOrder: SortOrder) =
when (sortOrder) {
SortOrder.ALPHABETICAL -> "name"
SortOrder.POPULARITY -> "rate"
SortOrder.UPDATED -> "updated"
SortOrder.NEWEST -> "created"
SortOrder.RATING -> "votes"
null -> "updated"
}
private suspend fun advancedSearch(domain: String, tags: Set<MangaTag>): Response {

2
src/main/kotlin/org/koitharu/kotatsu/parsers/site/HenChanParser.kt
View File

@@ -21,7 +21,7 @@ internal class HenChanParser(override val context: MangaLoaderContext) : ChanPar
offset: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder?,
sortOrder: SortOrder,
): List<Manga> {
return super.getList(offset, query, tags, sortOrder).map {
it.copy(

2
src/main/kotlin/org/koitharu/kotatsu/parsers/site/MadaraParser.kt
View File

@@ -34,7 +34,7 @@ internal abstract class MadaraParser(
offset: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder?,
sortOrder: SortOrder,
): List<Manga> {
val tag = when {
tags.isNullOrEmpty() -> null

3
src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaDexParser.kt
View File

@@ -35,7 +35,7 @@ internal class MangaDexParser(override val context: MangaLoaderContext) : MangaP
offset: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder?,
sortOrder: SortOrder,
): List<Manga> {
val domain = getDomain()
val url = buildString {
@@ -60,7 +60,6 @@ internal class MangaDexParser(override val context: MangaLoaderContext) : MangaP
append("&order")
append(
when (sortOrder) {
null,
SortOrder.UPDATED,
-> "[latestUploadedChapter]=desc"
SortOrder.ALPHABETICAL -> "[title]=asc"

2
src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaInUaParser.kt
View File

@@ -23,7 +23,7 @@ class MangaInUaParser(override val context: MangaLoaderContext) : MangaParser(Ma
offset: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder?,
sortOrder: SortOrder,
): List<Manga> {
val page = (offset / 24f).toIntUp().inc()
val searchPage = (offset / 10f).toIntUp().inc()

2
src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaLibParser.kt
View File

@@ -41,7 +41,7 @@ internal open class MangaLibParser(
offset: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder?,
sortOrder: SortOrder,
): List<Manga> {
if (!query.isNullOrEmpty()) {
return if (offset == 0) search(query) else emptyList()

10
src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaOwlParser.kt
View File

@@ -27,7 +27,7 @@ internal class MangaOwlParser(override val context: MangaLoaderContext) : MangaP
offset: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder?,
sortOrder: SortOrder,
): List<Manga> {
val page = (offset / 36f).toIntUp().inc()
val link = buildString {
@@ -165,16 +165,16 @@ internal class MangaOwlParser(override val context: MangaLoaderContext) : MangaP
}
}
private fun getSortKey(sortOrder: SortOrder?) =
when (sortOrder ?: sortOrders.minByOrNull { it.ordinal }) {
private fun getSortKey(sortOrder: SortOrder) =
when (sortOrder) {
SortOrder.POPULARITY -> "popular"
SortOrder.NEWEST -> "new_release"
SortOrder.UPDATED -> "lastest"
else -> "lastest"
}
private fun getAlternativeSortKey(sortOrder: SortOrder?) =
when (sortOrder ?: sortOrders.minByOrNull { it.ordinal }) {
private fun getAlternativeSortKey(sortOrder: SortOrder) =
when (sortOrder) {
SortOrder.POPULARITY -> "0"
SortOrder.NEWEST -> "2"
SortOrder.UPDATED -> "3"

2
src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaTownParser.kt
View File

@@ -29,7 +29,7 @@ internal class MangaTownParser(override val context: MangaLoaderContext) : Manga
offset: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder?,
sortOrder: SortOrder,
): List<Manga> {
val sortKey = when (sortOrder) {
SortOrder.ALPHABETICAL -> "?name.az"

2
src/main/kotlin/org/koitharu/kotatsu/parsers/site/NineMangaParser.kt
View File

@@ -38,7 +38,7 @@ internal abstract class NineMangaParser(
offset: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder?,
sortOrder: SortOrder,
): List<Manga> {
val page = (offset / PAGE_SIZE.toFloat()).toIntUp() + 1
val url = buildString {

6
src/main/kotlin/org/koitharu/kotatsu/parsers/site/NudeMoonParser.kt
View File

@@ -48,7 +48,7 @@ internal class NudeMoonParser(
offset: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder?,
sortOrder: SortOrder,
): List<Manga> {
val domain = getDomain()
val url = when {
@@ -206,8 +206,8 @@ internal class NudeMoonParser(
return "https://${getDomain()}/favicon.jpg"
}
private fun getSortKey(sortOrder: SortOrder?) =
when (sortOrder ?: sortOrders.minByOrNull { it.ordinal }) {
private fun getSortKey(sortOrder: SortOrder) =
when (sortOrder) {
SortOrder.POPULARITY -> "views"
SortOrder.NEWEST -> "date"
SortOrder.RATING -> "like"

2
src/main/kotlin/org/koitharu/kotatsu/parsers/site/RemangaParser.kt
View File

@@ -57,7 +57,7 @@ internal class RemangaParser(
offset: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder?,
sortOrder: SortOrder,
): List<Manga> {
copyCookies()
val domain = getDomain()

13
src/main/kotlin/org/koitharu/kotatsu/parsers/util/Parse.kt
View File

@@ -105,15 +105,24 @@ fun String.toRelativeUrl(domain: String): String {
}
/**
* Convert url to absolute with specified [domain]
* Convert url to absolute with specified domain
* @return an absolute url with [domain] if this is relative
*/
fun String.toAbsoluteUrl(domain: String): String = when {
this.startsWith("//") -> "https:$this"
this.startsWith("/") -> "https://$domain$this"
this.startsWith('/') -> "https://$domain$this"
else -> this
}
/**
* Convert url to absolute with specified domain and subdomain
* @return an absolute url with [subdomain].[domain] if this is relative
*/
fun String.toAbsoluteUrl(domain: String, subdomain: String): String {
if (!this.startsWith('/')) return this
return toAbsoluteUrl(subdomain + "." + domain.removePrefix("www."))
}
@Deprecated(
message = "",
level = DeprecationLevel.ERROR,

12
src/test/kotlin/org/koitharu/kotatsu/parsers/MangaParserTest.kt
View File

@@ -25,7 +25,7 @@ internal class MangaParserTest {
@MangaSources
fun list(source: MangaSource) = runTest {
val parser = source.newParser(context)
val list = parser.getList(20, query = null, sortOrder = SortOrder.POPULARITY, tags = null)
val list = parser.getList(20, sortOrder = SortOrder.POPULARITY, tags = null)
checkMangaList(list, "list")
assert(list.all { it.source == source })
}
@@ -34,12 +34,12 @@ internal class MangaParserTest {
@MangaSources
fun search(source: MangaSource) = runTest {
val parser = source.newParser(context)
val subject = parser.getList(20, query = null, sortOrder = SortOrder.POPULARITY, tags = null).minByOrNull {
val subject = parser.getList(20, sortOrder = SortOrder.POPULARITY, tags = null).minByOrNull {
it.title.length
} ?: error("No manga found")
val query = subject.title
check(query.isNotBlank()) { "Manga title '$query' is blank" }
val list = parser.getList(offset = 0, query, sortOrder = null, tags = null)
val list = parser.getList(0, query)
assert(list.singleOrNull { it.url == subject.url && it.id == subject.id } != null) {
"Single subject '${subject.title} (${subject.publicUrl})' not found in search results"
}
@@ -62,7 +62,7 @@ internal class MangaParserTest {
assert(tags.all { it.source == source })
val tag = tags.last()
val list = parser.getList(offset = 0, tags = setOf(tag), query = null, sortOrder = null)
val list = parser.getList(offset = 0, tags = setOf(tag), sortOrder = null)
checkMangaList(list, "${tag.title} (${tag.key})")
assert(list.all { it.source == source })
}
@@ -71,7 +71,7 @@ internal class MangaParserTest {
@MangaSources
fun details(source: MangaSource) = runTest {
val parser = source.newParser(context)
val list = parser.getList(20, query = null, sortOrder = SortOrder.POPULARITY, tags = null)
val list = parser.getList(20, sortOrder = SortOrder.POPULARITY, tags = null)
val manga = list[3]
parser.getDetails(manga).apply {
assert(!chapters.isNullOrEmpty()) { "Chapters are null or empty" }
@@ -100,7 +100,7 @@ internal class MangaParserTest {
@MangaSources
fun pages(source: MangaSource) = runTest {
val parser = source.newParser(context)
val list = parser.getList(20, query = null, sortOrder = SortOrder.POPULARITY, tags = null)
val list = parser.getList(20, sortOrder = SortOrder.POPULARITY, tags = null)
val manga = list.first()
val chapter = parser.getDetails(manga).chapters?.firstOrNull() ?: error("Chapter is null")
val pages = parser.getPages(chapter)