KotatsuApp/kotatsu-parsers
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

MangaParser interface

Browse Source
This commit is contained in:
Koitharu
2025-02-24 17:59:40 +02:00
parent a4827d1b7d
commit 29cf04c804
25 changed files with 208 additions and 167 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.github/summary.yaml vendored
View File

@@ -1 +1 @@
total: 1189
total: 1189

128
src/main/kotlin/org/koitharu/kotatsu/parsers/AbstractMangaParser.kt Normal file
View File

@@ -0,0 +1,128 @@
package org.koitharu.kotatsu.parsers
import androidx.annotation.CallSuper
import okhttp3.Headers
import okhttp3.HttpUrl
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.config.MangaSourceConfig
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.model.search.MangaSearchQuery
import org.koitharu.kotatsu.parsers.model.search.MangaSearchQueryCapabilities
import org.koitharu.kotatsu.parsers.network.OkHttpWebClient
import org.koitharu.kotatsu.parsers.network.WebClient
import org.koitharu.kotatsu.parsers.util.*
import java.util.*
@InternalParsersApi
public abstract class AbstractMangaParser @InternalParsersApi constructor(
@property:InternalParsersApi public val context: MangaLoaderContext,
public override val source: MangaParserSource,
) : MangaParser {
@Deprecated("Please check searchQueryCapabilities")
public abstract val filterCapabilities: MangaListFilterCapabilities
public override val searchQueryCapabilities: MangaSearchQueryCapabilities
get() = filterCapabilities.toMangaSearchQueryCapabilities()
public override val config: MangaSourceConfig by lazy { context.getConfig(source) }
public open val sourceLocale: Locale
get() = if (source.locale.isEmpty()) Locale.ROOT else Locale(source.locale)
protected val isNsfwSource: Boolean = source.contentType == ContentType.HENTAI
/**
* Provide default domain and available alternatives, if any.
*
* Never hardcode domain in requests, use [domain] instead.
*/
@InternalParsersApi
public abstract val configKeyDomain: ConfigKey.Domain
protected open val userAgentKey: ConfigKey.UserAgent = ConfigKey.UserAgent(context.getDefaultUserAgent())
override fun getRequestHeaders(): Headers = Headers.Builder()
.add("User-Agent", config[userAgentKey])
.build()
/**
* Used as fallback if value of `order` passed to [getList] is null
*/
public open val defaultSortOrder: SortOrder
get() {
val supported = availableSortOrders
return SortOrder.entries.first { it in supported }
}
override val domain: String
get() = config[configKeyDomain]
@JvmField
protected val webClient: WebClient = OkHttpWebClient(context.httpClient, source)
/**
* Search list of manga by specified searchQuery
*
* @param searchQuery searchQuery
*/
public override suspend fun queryManga(searchQuery: MangaSearchQuery): List<Manga> {
if (!searchQuery.skipValidation) {
searchQueryCapabilities.validate(searchQuery)
}
return getList(searchQuery)
}
/**
* Search list of manga by specified searchQuery
*
* @param query searchQuery
*/
protected open suspend fun getList(query: MangaSearchQuery): List<Manga> = getList(
offset = query.offset,
order = query.order ?: defaultSortOrder,
filter = convertToMangaListFilter(query),
)
/**
* Parse list of manga by specified criteria
*
* @param offset starting from 0 and used for pagination.
* Note than passed value may not be divisible by internal page size, so you should adjust it manually.
* @param order one of [availableSortOrders] or [defaultSortOrder] for default value
* @param filter is a set of filter rules
*
* @deprecated New [getList] should be preferred.
*/
@Deprecated("New getList(query: MangaSearchQuery) method should be preferred")
public abstract suspend fun getList(offset: Int, order: SortOrder, filter: MangaListFilter): List<Manga>
/**
* Fetch direct link to the page image.
*/
public override suspend fun getPageUrl(page: MangaPage): String = page.url.toAbsoluteUrl(domain)
/**
* Parse favicons from the main page of the source`s website
*/
public override suspend fun getFavicons(): Favicons {
return FaviconParser(webClient, domain).parseFavicons()
}
@CallSuper
public override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) {
keys.add(configKeyDomain)
}
public override suspend fun getRelatedManga(seed: Manga): List<Manga> {
return RelatedMangaFinder(listOf(this)).invoke(seed)
}
/**
* Return [Manga] object by web link to it
* @see [Manga.publicUrl]
*/
internal open suspend fun resolveLink(resolver: LinkResolver, link: HttpUrl): Manga? = null
}

121
src/main/kotlin/org/koitharu/kotatsu/parsers/MangaParser.kt
View File

@@ -1,145 +1,60 @@
package org.koitharu.kotatsu.parsers
import androidx.annotation.CallSuper
import okhttp3.Headers
import okhttp3.HttpUrl
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.config.MangaSourceConfig
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.model.search.*
import org.koitharu.kotatsu.parsers.network.OkHttpWebClient
import org.koitharu.kotatsu.parsers.network.WebClient
import org.koitharu.kotatsu.parsers.util.*
import org.koitharu.kotatsu.parsers.model.search.MangaSearchQuery
import org.koitharu.kotatsu.parsers.model.search.MangaSearchQueryCapabilities
import java.util.*
public abstract class MangaParser @InternalParsersApi constructor(
@property:InternalParsersApi public val context: MangaLoaderContext,
public val source: MangaParserSource,
) {
public interface MangaParser {
public val source: MangaParserSource
/**
* Supported [SortOrder] variants. Must not be empty.
*
* For better performance use [EnumSet] for more than one item.
*/
public abstract val availableSortOrders: Set<SortOrder>
public val availableSortOrders: Set<SortOrder>
@Deprecated("Please check searchQueryCapabilities")
public abstract val filterCapabilities: MangaListFilterCapabilities
public val searchQueryCapabilities: MangaSearchQueryCapabilities
public open val searchQueryCapabilities: MangaSearchQueryCapabilities
get() = filterCapabilities.toMangaSearchQueryCapabilities()
public val config: MangaSourceConfig
public val config: MangaSourceConfig by lazy { context.getConfig(source) }
public val domain: String
public open val sourceLocale: Locale
get() = if (source.locale.isEmpty()) Locale.ROOT else Locale(source.locale)
protected val isNsfwSource: Boolean = source.contentType == ContentType.HENTAI
/**
* Provide default domain and available alternatives, if any.
*
* Never hardcode domain in requests, use [domain] instead.
*/
@InternalParsersApi
public abstract val configKeyDomain: ConfigKey.Domain
protected open val userAgentKey: ConfigKey.UserAgent = ConfigKey.UserAgent(context.getDefaultUserAgent())
public open fun getRequestHeaders(): Headers = Headers.Builder()
.add("User-Agent", config[userAgentKey])
.build()
/**
* Used as fallback if value of `order` passed to [getList] is null
*/
public open val defaultSortOrder: SortOrder
get() {
val supported = availableSortOrders
return SortOrder.entries.first { it in supported }
}
@JvmField
protected val webClient: WebClient = OkHttpWebClient(context.httpClient, source)
/**
* Search list of manga by specified searchQuery
*
* @param searchQuery searchQuery
*/
public suspend fun queryManga(searchQuery: MangaSearchQuery): List<Manga> {
if (!searchQuery.skipValidation) {
searchQueryCapabilities.validate(searchQuery)
}
return getList(searchQuery)
}
/**
* Search list of manga by specified searchQuery
*
* @param query searchQuery
*/
protected open suspend fun getList(query: MangaSearchQuery): List<Manga> = getList(
offset = query.offset,
order = query.order ?: defaultSortOrder,
filter = convertToMangaListFilter(query),
)
/**
* Parse list of manga by specified criteria
*
* @param offset starting from 0 and used for pagination.
* Note than passed value may not be divisible by internal page size, so you should adjust it manually.
* @param order one of [availableSortOrders] or [defaultSortOrder] for default value
* @param filter is a set of filter rules
*
* @deprecated New [getList] should be preferred.
*/
@Deprecated("New getList(query: MangaSearchQuery) method should be preferred")
public abstract suspend fun getList(offset: Int, order: SortOrder, filter: MangaListFilter): List<Manga>
public suspend fun queryManga(searchQuery: MangaSearchQuery): List<Manga>
/**
* Parse details for [Manga]: chapters list, description, large cover, etc.
* Must return the same manga, may change any fields excepts id, url and source
* @see Manga.copy
*/
public abstract suspend fun getDetails(manga: Manga): Manga
public suspend fun getDetails(manga: Manga): Manga
/**
* Parse pages list for specified chapter.
* @see MangaPage for details
*/
public abstract suspend fun getPages(chapter: MangaChapter): List<MangaPage>
public suspend fun getPages(chapter: MangaChapter): List<MangaPage>
/**
* Fetch direct link to the page image.
*/
public open suspend fun getPageUrl(page: MangaPage): String = page.url.toAbsoluteUrl(domain)
public suspend fun getPageUrl(page: MangaPage): String
public abstract suspend fun getFilterOptions(): MangaListFilterOptions
public suspend fun getFilterOptions(): MangaListFilterOptions
/**
* Parse favicons from the main page of the source`s website
*/
public open suspend fun getFavicons(): Favicons {
return FaviconParser(webClient, domain).parseFavicons()
}
public suspend fun getFavicons(): Favicons
@CallSuper
public open fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) {
keys.add(configKeyDomain)
}
public fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>)
public open suspend fun getRelatedManga(seed: Manga): List<Manga> {
return RelatedMangaFinder(listOf(this)).invoke(seed)
}
/**
* Return [Manga] object by web link to it
* @see [Manga.publicUrl]
*/
internal open suspend fun resolveLink(resolver: LinkResolver, link: HttpUrl): Manga? = null
public suspend fun getRelatedManga(seed: Manga): List<Manga>
public fun getRequestHeaders(): Headers
}

2
src/main/kotlin/org/koitharu/kotatsu/parsers/PagedMangaParser.kt
View File

@@ -16,7 +16,7 @@ public abstract class PagedMangaParser(
source: MangaParserSource,
@VisibleForTesting(otherwise = VisibleForTesting.PROTECTED) @JvmField public val pageSize: Int,
searchPageSize: Int = pageSize,
) : MangaParser(context, source) {
) : AbstractMangaParser(context, source) {
@JvmField
protected val paginator: Paginator = Paginator(pageSize)

2
src/main/kotlin/org/koitharu/kotatsu/parsers/SinglePageMangaParser.kt
View File

@@ -11,7 +11,7 @@ import org.koitharu.kotatsu.parsers.util.convertToMangaListFilter
public abstract class SinglePageMangaParser(
context: MangaLoaderContext,
source: MangaParserSource,
) : MangaParser(context, source) {
) : AbstractMangaParser(context, source) {
final override suspend fun getList(query: MangaSearchQuery): List<Manga> {

5
src/main/kotlin/org/koitharu/kotatsu/parsers/model/search/MangaSearchQuery.kt
View File

@@ -81,4 +81,9 @@ public data class MangaSearchQuery private constructor(
return uniqueCriteria.values.toSet()
}
}
public companion object {
public val EMPTY: MangaSearchQuery = MangaSearchQuery(emptySet(), null, 0, false)
}
}

4
src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/HitomiLaParser.kt
View File

@@ -11,7 +11,7 @@ import org.json.JSONArray
import org.json.JSONObject
import org.jsoup.Jsoup
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.AbstractMangaParser
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
@@ -28,7 +28,7 @@ import kotlin.math.min
@OptIn(ExperimentalUnsignedTypes::class)
@MangaSourceParser("HITOMILA", "Hitomi.La", type = ContentType.HENTAI)
internal class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaParserSource.HITOMILA) {
internal class HitomiLaParser(context: MangaLoaderContext) : AbstractMangaParser(context, MangaParserSource.HITOMILA) {
override val configKeyDomain = ConfigKey.Domain("hitomi.la")
override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) {

4
src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/LineWebtoonsParser.kt
View File

@@ -6,7 +6,7 @@ import okhttp3.HttpUrl
import okhttp3.HttpUrl.Companion.toHttpUrl
import org.json.JSONObject
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.AbstractMangaParser
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.exception.NotFoundException
@@ -21,7 +21,7 @@ import javax.crypto.spec.SecretKeySpec
internal abstract class LineWebtoonsParser(
context: MangaLoaderContext,
source: MangaParserSource,
) : MangaParser(context, source) {
) : AbstractMangaParser(context, source) {
override val filterCapabilities: MangaListFilterCapabilities
get() = MangaListFilterCapabilities(

4
src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/MangaDexParser.kt
View File

@@ -8,7 +8,7 @@ import okhttp3.HttpUrl
import org.json.JSONArray
import org.json.JSONObject
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.AbstractMangaParser
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.exception.ParseException
@@ -34,7 +34,7 @@ private const val SERVER_DATA = "data"
private const val SERVER_DATA_SAVER = "data-saver"
@MangaSourceParser("MANGADEX", "MangaDex")
internal class MangaDexParser(context: MangaLoaderContext) : MangaParser(context, MangaParserSource.MANGADEX) {
internal class MangaDexParser(context: MangaLoaderContext) : AbstractMangaParser(context, MangaParserSource.MANGADEX) {
override val configKeyDomain = ConfigKey.Domain("mangadex.org")

4
src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt
View File

@@ -8,7 +8,7 @@ import okhttp3.HttpUrl.Companion.toHttpUrl
import org.json.JSONArray
import org.json.JSONObject
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.AbstractMangaParser
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.exception.NotFoundException
@@ -24,7 +24,7 @@ import javax.crypto.spec.SecretKeySpec
internal abstract class WebtoonsParser(
context: MangaLoaderContext,
source: MangaParserSource,
) : MangaParser(context, source) {
) : AbstractMangaParser(context, source) {
private val signer by lazy {
WebtoonsUrlSigner("gUtPzJFZch4ZyAGviiyH94P99lQ3pFdRTwpJWDlSGFfwgpr6ses5ALOxWHOIT7R1")

4
src/main/kotlin/org/koitharu/kotatsu/parsers/site/be/AnibelParser.kt
View File

@@ -5,7 +5,7 @@ import org.json.JSONArray
import org.json.JSONObject
import org.koitharu.kotatsu.parsers.Broken
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.AbstractMangaParser
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
@@ -18,7 +18,7 @@ import java.util.*
@Broken
@MangaSourceParser("ANIBEL", "Anibel", "be")
internal class AnibelParser(context: MangaLoaderContext) : MangaParser(context, MangaParserSource.ANIBEL) {
internal class AnibelParser(context: MangaLoaderContext) : AbstractMangaParser(context, MangaParserSource.ANIBEL) {
override val configKeyDomain = ConfigKey.Domain("anibel.net")

2
src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/WeebCentral.kt
View File

@@ -16,7 +16,7 @@ import java.util.EnumSet
import java.util.Locale
@MangaSourceParser("WEEBCENTRAL", "Weeb Central", "en")
internal class WeebCentral(context: MangaLoaderContext) : MangaParser(context, MangaParserSource.WEEBCENTRAL),
internal class WeebCentral(context: MangaLoaderContext) : AbstractMangaParser(context, MangaParserSource.WEEBCENTRAL),
MangaParserAuthProvider {
override val configKeyDomain = ConfigKey.Domain("weebcentral.com")

4
src/main/kotlin/org/koitharu/kotatsu/parsers/site/ja/NicovideoSeigaParser.kt
View File

@@ -1,7 +1,7 @@
package org.koitharu.kotatsu.parsers.site.ja
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.AbstractMangaParser
import org.koitharu.kotatsu.parsers.MangaParserAuthProvider
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
@@ -16,7 +16,7 @@ private const val STATUS_FINISHED = "完結"
@MangaSourceParser("NICOVIDEO_SEIGA", "NicoVideo Seiga", "ja")
internal class NicovideoSeigaParser(context: MangaLoaderContext) :
MangaParser(context, MangaParserSource.NICOVIDEO_SEIGA),
AbstractMangaParser(context, MangaParserSource.NICOVIDEO_SEIGA),
MangaParserAuthProvider {
override val configKeyDomain: ConfigKey.Domain = ConfigKey.Domain("nicovideo.jp")

4
src/main/kotlin/org/koitharu/kotatsu/parsers/site/nepnep/NepnepParser.kt
View File

@@ -3,7 +3,7 @@ package org.koitharu.kotatsu.parsers.site.nepnep
import org.json.JSONArray
import org.json.JSONObject
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.AbstractMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.network.UserAgents
@@ -20,7 +20,7 @@ internal abstract class NepnepParser(
context: MangaLoaderContext,
source: MangaParserSource,
domain: String,
) : MangaParser(context, source) {
) : AbstractMangaParser(context, source) {
override val configKeyDomain = ConfigKey.Domain(domain)

4
src/main/kotlin/org/koitharu/kotatsu/parsers/site/ru/NudeMoonParser.kt
View File

@@ -1,7 +1,7 @@
package org.koitharu.kotatsu.parsers.site.ru
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.AbstractMangaParser
import org.koitharu.kotatsu.parsers.MangaParserAuthProvider
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
@@ -14,7 +14,7 @@ import java.util.*
@MangaSourceParser("NUDEMOON", "Nude-Moon", "ru", type = ContentType.HENTAI)
internal class NudeMoonParser(
context: MangaLoaderContext,
) : MangaParser(context, MangaParserSource.NUDEMOON), MangaParserAuthProvider {
) : AbstractMangaParser(context, MangaParserSource.NUDEMOON), MangaParserAuthProvider {
override val configKeyDomain = ConfigKey.Domain(
"b.nude-moon.fun",

4
src/main/kotlin/org/koitharu/kotatsu/parsers/site/ru/grouple/GroupleParser.kt
View File

@@ -17,7 +17,7 @@ import okio.IOException
import org.json.JSONArray
import org.jsoup.nodes.Element
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.AbstractMangaParser
import org.koitharu.kotatsu.parsers.MangaParserAuthProvider
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.exception.AuthRequiredException
@@ -43,7 +43,7 @@ internal abstract class GroupleParser(
context: MangaLoaderContext,
source: MangaParserSource,
private val siteId: Int,
) : MangaParser(context, source), MangaParserAuthProvider, Interceptor {
) : AbstractMangaParser(context, source), MangaParserAuthProvider, Interceptor {
@Volatile
private var cachedPagesServer: String? = null

4
src/main/kotlin/org/koitharu/kotatsu/parsers/site/ru/multichan/ChanParser.kt
View File

@@ -3,7 +3,7 @@ package org.koitharu.kotatsu.parsers.site.ru.multichan
import okhttp3.HttpUrl
import org.jsoup.internal.StringUtil
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.AbstractMangaParser
import org.koitharu.kotatsu.parsers.MangaParserAuthProvider
import org.koitharu.kotatsu.parsers.exception.AuthRequiredException
import org.koitharu.kotatsu.parsers.model.*
@@ -14,7 +14,7 @@ import java.util.*
internal abstract class ChanParser(
context: MangaLoaderContext,
source: MangaParserSource,
) : MangaParser(context, source), MangaParserAuthProvider {
) : AbstractMangaParser(context, source), MangaParserAuthProvider {
override val availableSortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.NEWEST,

4
src/main/kotlin/org/koitharu/kotatsu/parsers/site/uk/HentaiUkrParser.kt
View File

@@ -8,7 +8,7 @@ import okhttp3.Response
import org.json.JSONArray
import org.json.JSONObject
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.AbstractMangaParser
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
@@ -25,7 +25,7 @@ private const val PAGE_SIZE = 60
// NOTE High profile focus
@MangaSourceParser("HENTAIUKR", "HentaiUkr", "uk", ContentType.HENTAI)
internal class HentaiUkrParser(context: MangaLoaderContext) : MangaParser(context, MangaParserSource.HENTAIUKR),
internal class HentaiUkrParser(context: MangaLoaderContext) : AbstractMangaParser(context, MangaParserSource.HENTAIUKR),
Interceptor {
private val date = SimpleDateFormat("yyyy-MM-dd", Locale.US)

4
src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/HentaiVNParser.kt
View File

@@ -8,7 +8,7 @@ import kotlinx.coroutines.sync.withLock
import org.jsoup.nodes.Document
import org.koitharu.kotatsu.parsers.Broken
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.AbstractMangaParser
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
@@ -21,7 +21,7 @@ private const val SEARCH_PAGE_SIZE = 10
@Broken
@MangaSourceParser("HENTAIVN", "HentaiVN", "vi", type = ContentType.HENTAI)
internal class HentaiVNParser(context: MangaLoaderContext) : MangaParser(context, MangaParserSource.HENTAIVN) {
internal class HentaiVNParser(context: MangaLoaderContext) : AbstractMangaParser(context, MangaParserSource.HENTAIVN) {
override val configKeyDomain: ConfigKey.Domain = ConfigKey.Domain("hentaihvn.tv")

11
src/main/kotlin/org/koitharu/kotatsu/parsers/util/LinkResolver.kt
View File

@@ -4,8 +4,8 @@ import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.runInterruptible
import okhttp3.HttpUrl
import okhttp3.HttpUrl.Companion.toHttpUrlOrNull
import org.koitharu.kotatsu.parsers.AbstractMangaParser
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.suspendlazy.suspendLazy
@@ -19,14 +19,15 @@ public class LinkResolver internal constructor(
public suspend fun getSource(): MangaParserSource? = source.get()
public suspend fun getManga(): Manga? {
val parser = context.newParserInstance(source.get() ?: return null)
val parser = context.newParserInstance(source.get() ?: return null) as? AbstractMangaParser
?: return null
return parser.resolveLink(this, link) ?: resolveManga(parser)
}
private suspend fun resolveSource(): MangaParserSource? = runInterruptible(Dispatchers.Default) {
val domains = setOfNotNull(link.host, link.topPrivateDomain())
for (s in MangaParserSource.entries) {
val parser = context.newParserInstance(s)
val parser = context.newParserInstance(s) as AbstractMangaParser
for (d in parser.configKeyDomain.presetValues) {
if (d in domains) {
return@runInterruptible s
@@ -37,7 +38,7 @@ public class LinkResolver internal constructor(
}
internal suspend fun resolveManga(
parser: MangaParser,
parser: AbstractMangaParser,
url: String = link.toString().toRelativeUrl(link.host),
id: Long = parser.generateUid(url),
title: String = STUB_TITLE,
@@ -62,7 +63,7 @@ public class LinkResolver internal constructor(
),
)
private suspend fun resolveBySeed(parser: MangaParser, s: Manga): Manga? {
private suspend fun resolveBySeed(parser: AbstractMangaParser, s: Manga): Manga? {
val seed = parser.getDetails(s)
if (!parser.filterCapabilities.isSearchSupported) {
return seed.takeUnless { it.chapters.isNullOrEmpty() }

7
src/main/kotlin/org/koitharu/kotatsu/parsers/util/MangaParserEnv.kt
View File

@@ -2,6 +2,7 @@ package org.koitharu.kotatsu.parsers.util
import okhttp3.HttpUrl
import org.jsoup.nodes.Element
import org.koitharu.kotatsu.parsers.AbstractMangaParser
import org.koitharu.kotatsu.parsers.ErrorMessages
import org.koitharu.kotatsu.parsers.InternalParsersApi
import org.koitharu.kotatsu.parsers.MangaParser
@@ -81,17 +82,17 @@ private fun <T> Set<T>?.oneOrThrowIfMany(msg: String): T? = when {
else -> throw IllegalArgumentException(msg)
}
public val MangaParser.domain: String
public val AbstractMangaParser.domain: String
get() = config[configKeyDomain]
@InternalParsersApi
public fun MangaParser.getDomain(subdomain: String): String {
public fun AbstractMangaParser.getDomain(subdomain: String): String {
val domain = domain
return subdomain + "." + domain.removePrefix("www.")
}
@InternalParsersApi
public fun MangaParser.urlBuilder(subdomain: String? = null): HttpUrl.Builder {
public fun AbstractMangaParser.urlBuilder(subdomain: String? = null): HttpUrl.Builder {
return HttpUrl.Builder()
.scheme(SCHEME_HTTPS)
.host(if (subdomain == null) domain else "$subdomain.$domain")

11
src/main/kotlin/org/koitharu/kotatsu/parsers/util/RelatedMangaFinder.kt
View File

@@ -6,8 +6,10 @@ import kotlinx.coroutines.awaitAll
import kotlinx.coroutines.coroutineScope
import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.model.Manga
import org.koitharu.kotatsu.parsers.model.MangaListFilter
import org.koitharu.kotatsu.parsers.model.SortOrder
import org.koitharu.kotatsu.parsers.model.search.MangaSearchQuery
import org.koitharu.kotatsu.parsers.model.search.QueryCriteria
import org.koitharu.kotatsu.parsers.model.search.SearchableField
public class RelatedMangaFinder(
private val parsers: Collection<MangaParser>,
@@ -34,7 +36,12 @@ public class RelatedMangaFinder(
}
val results = words.map { keyword ->
scope.async {
val result = parser.getList(0, SortOrder.RELEVANCE, MangaListFilter(query = keyword))
val result = parser.queryManga(
MangaSearchQuery.Builder()
.order(SortOrder.RELEVANCE)
.criterion(QueryCriteria.Match(SearchableField.TITLE_NAME, keyword))
.build(),
)
result.filter { it.id != seed.id && it.containKeyword(keyword) }
}
}.awaitAll()

6
src/test/kotlin/org/koitharu/kotatsu/parsers/CommonHeadersInterceptor.kt
View File

@@ -5,8 +5,6 @@ import okhttp3.Request
import okhttp3.Response
import org.koitharu.kotatsu.parsers.model.MangaParserSource
import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.util.domain
import org.koitharu.kotatsu.parsers.util.mergeWith
private const val HEADER_REFERER = "Referer"
@@ -20,11 +18,7 @@ internal class CommonHeadersInterceptor : Interceptor {
} else {
null
}
val sourceHeaders = parser?.getRequestHeaders()
val headersBuilder = request.headers.newBuilder()
if (sourceHeaders != null) {
headersBuilder.mergeWith(sourceHeaders, replaceExisting = false)
}
if (headersBuilder[HEADER_REFERER] == null && parser != null) {
headersBuilder[HEADER_REFERER] = "https://${parser.domain}/"
}

27
src/test/kotlin/org/koitharu/kotatsu/parsers/MangaParserTest.kt
View File

@@ -10,7 +10,6 @@ import org.koitharu.kotatsu.parsers.model.search.MangaSearchQuery
import org.koitharu.kotatsu.parsers.model.search.QueryCriteria
import org.koitharu.kotatsu.parsers.model.search.QueryCriteria.Include
import org.koitharu.kotatsu.parsers.model.search.SearchableField.*
import org.koitharu.kotatsu.parsers.util.domain
import org.koitharu.kotatsu.parsers.util.medianOrNull
import org.koitharu.kotatsu.parsers.util.mimeType
import org.koitharu.kotatsu.test_util.*
@@ -26,7 +25,7 @@ internal class MangaParserTest {
@MangaSources
fun list(source: MangaParserSource) = runTest(timeout = timeout) {
val parser = context.newParserInstance(source)
val list = parser.getList(0, parser.defaultSortOrder, MangaListFilter.EMPTY)
val list = parser.queryManga(MangaSearchQuery.Builder().build())
checkMangaList(list, "list")
assert(list.all { it.source == source })
}
@@ -38,9 +37,9 @@ internal class MangaParserTest {
if (parser is SinglePageMangaParser) {
return@runTest
}
val page1 = parser.queryManga(MangaSearchQuery.Builder().offset(0).order(parser.defaultSortOrder).build())
val page1 = parser.queryManga(MangaSearchQuery.EMPTY)
val page2 =
parser.queryManga(MangaSearchQuery.Builder().offset(page1.size).order(parser.defaultSortOrder).build())
parser.queryManga(MangaSearchQuery.Builder().offset(page1.size).build())
if (parser is PagedMangaParser) {
assert(parser.pageSize >= page1.size) {
"Page size is ${page1.size} but ${parser.pageSize} expected"
@@ -59,12 +58,7 @@ internal class MangaParserTest {
@MangaSources
fun searchByTitleName(source: MangaParserSource) = runTest(timeout = timeout) {
val parser = context.newParserInstance(source)
val subject = parser.queryManga(
MangaSearchQuery.Builder()
.offset(0)
.order(parser.defaultSortOrder)
.build(),
).minByOrNull {
val subject = parser.queryManga(MangaSearchQuery.EMPTY).minByOrNull {
it.title.length
} ?: error("No manga found")
@@ -106,7 +100,6 @@ internal class MangaParserTest {
val list = parser.queryManga(
MangaSearchQuery.Builder()
.offset(0)
.order(parser.defaultSortOrder)
.criterion(Include(TAG, setOf(tag)))
.build(),
)
@@ -118,13 +111,12 @@ internal class MangaParserTest {
@MangaSources
fun tagsMultiple(source: MangaParserSource) = runTest(timeout = timeout) {
val parser = context.newParserInstance(source)
if (!parser.filterCapabilities.isMultipleTagsSupported) return@runTest
// if (!parser.filterCapabilities.isMultipleTagsSupported) return@runTest
val tags = parser.getFilterOptions().availableTags.shuffled().take(2).toSet()
val list = parser.queryManga(
MangaSearchQuery.Builder()
.offset(0)
.order(parser.defaultSortOrder)
.criterion(Include(TAG, tags))
.build(),
)
@@ -144,8 +136,6 @@ internal class MangaParserTest {
val locale = locales.random()
val list = parser.queryManga(
MangaSearchQuery.Builder()
.offset(0)
.order(parser.defaultSortOrder)
.criterion(Include(LANGUAGE, setOf(locale)))
.criterion(Include(LANGUAGE, setOf(locale)))
.criterion(Include(ORIGINAL_LANGUAGE, setOf(locales.random())))
@@ -160,7 +150,7 @@ internal class MangaParserTest {
@MangaSources
fun details(source: MangaParserSource) = runTest(timeout = timeout) {
val parser = context.newParserInstance(source)
val list = parser.queryManga(MangaSearchQuery.Builder().offset(0).order(parser.defaultSortOrder).build())
val list = parser.queryManga(MangaSearchQuery.EMPTY)
val manga = list[0]
parser.getDetails(manga).apply {
@@ -191,7 +181,7 @@ internal class MangaParserTest {
@MangaSources
fun pages(source: MangaParserSource) = runTest(timeout = timeout) {
val parser = context.newParserInstance(source)
val list = parser.queryManga(MangaSearchQuery.Builder().offset(0).order(parser.defaultSortOrder).build())
val list = parser.queryManga(MangaSearchQuery.EMPTY)
val manga = list.first()
val chapter = parser.getDetails(manga).chapters?.firstOrNull() ?: error("Chapter is null at ${manga.publicUrl}")
val pages = parser.getPages(chapter)
@@ -246,8 +236,7 @@ internal class MangaParserTest {
@MangaSources
fun link(source: MangaParserSource) = runTest(timeout = timeout) {
val parser = context.newParserInstance(source)
val manga =
parser.queryManga(MangaSearchQuery.Builder().offset(0).order(parser.defaultSortOrder).build()).first()
val manga = parser.queryManga(MangaSearchQuery.Builder().build()).first()
val resolved = context.newLinkResolver(manga.publicUrl).getManga()
Assertions.assertNotNull(resolved)
resolved ?: return@runTest

3
src/test/kotlin/org/koitharu/kotatsu/parsers/util/IntentFilterGenerator.kt
View File

@@ -1,6 +1,7 @@
package org.koitharu.kotatsu.parsers.util
import org.junit.jupiter.api.Test
import org.koitharu.kotatsu.parsers.AbstractMangaParser
import org.koitharu.kotatsu.parsers.MangaLoaderContextMock
import org.koitharu.kotatsu.parsers.model.MangaParserSource
import org.koitharu.kotatsu.parsers.newParser
@@ -25,7 +26,7 @@ class IntentFilterGenerator {
if (source == MangaParserSource.DUMMY) {
continue
}
val parser = source.newParser(MangaLoaderContextMock)
val parser = source.newParser(MangaLoaderContextMock) as AbstractMangaParser
parser.configKeyDomain.presetValues.forEach { domain ->
writer.appendTab().append("<data android:host=\"").append(domain).appendLine("\" />")
}