MangaParser interface

master
Koitharu 1 year ago
parent a4827d1b7d
commit 29cf04c804
Signed by: Koitharu
GPG Key ID: 676DEE768C17A9D7

@ -0,0 +1,128 @@
package org.koitharu.kotatsu.parsers
import androidx.annotation.CallSuper
import okhttp3.Headers
import okhttp3.HttpUrl
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.config.MangaSourceConfig
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.model.search.MangaSearchQuery
import org.koitharu.kotatsu.parsers.model.search.MangaSearchQueryCapabilities
import org.koitharu.kotatsu.parsers.network.OkHttpWebClient
import org.koitharu.kotatsu.parsers.network.WebClient
import org.koitharu.kotatsu.parsers.util.*
import java.util.*
@InternalParsersApi
public abstract class AbstractMangaParser @InternalParsersApi constructor(
@property:InternalParsersApi public val context: MangaLoaderContext,
public override val source: MangaParserSource,
) : MangaParser {
@Deprecated("Please check searchQueryCapabilities")
public abstract val filterCapabilities: MangaListFilterCapabilities
public override val searchQueryCapabilities: MangaSearchQueryCapabilities
get() = filterCapabilities.toMangaSearchQueryCapabilities()
public override val config: MangaSourceConfig by lazy { context.getConfig(source) }
public open val sourceLocale: Locale
get() = if (source.locale.isEmpty()) Locale.ROOT else Locale(source.locale)
protected val isNsfwSource: Boolean = source.contentType == ContentType.HENTAI
/**
* Provide default domain and available alternatives, if any.
*
* Never hardcode domain in requests, use [domain] instead.
*/
@InternalParsersApi
public abstract val configKeyDomain: ConfigKey.Domain
protected open val userAgentKey: ConfigKey.UserAgent = ConfigKey.UserAgent(context.getDefaultUserAgent())
override fun getRequestHeaders(): Headers = Headers.Builder()
.add("User-Agent", config[userAgentKey])
.build()
/**
* Used as fallback if value of `order` passed to [getList] is null
*/
public open val defaultSortOrder: SortOrder
get() {
val supported = availableSortOrders
return SortOrder.entries.first { it in supported }
}
override val domain: String
get() = config[configKeyDomain]
@JvmField
protected val webClient: WebClient = OkHttpWebClient(context.httpClient, source)
/**
* Search list of manga by specified searchQuery
*
* @param searchQuery searchQuery
*/
public override suspend fun queryManga(searchQuery: MangaSearchQuery): List<Manga> {
if (!searchQuery.skipValidation) {
searchQueryCapabilities.validate(searchQuery)
}
return getList(searchQuery)
}
/**
* Search list of manga by specified searchQuery
*
* @param query searchQuery
*/
protected open suspend fun getList(query: MangaSearchQuery): List<Manga> = getList(
offset = query.offset,
order = query.order ?: defaultSortOrder,
filter = convertToMangaListFilter(query),
)
/**
* Parse list of manga by specified criteria
*
* @param offset starting from 0 and used for pagination.
* Note than passed value may not be divisible by internal page size, so you should adjust it manually.
* @param order one of [availableSortOrders] or [defaultSortOrder] for default value
* @param filter is a set of filter rules
*
* @deprecated New [getList] should be preferred.
*/
@Deprecated("New getList(query: MangaSearchQuery) method should be preferred")
public abstract suspend fun getList(offset: Int, order: SortOrder, filter: MangaListFilter): List<Manga>
/**
* Fetch direct link to the page image.
*/
public override suspend fun getPageUrl(page: MangaPage): String = page.url.toAbsoluteUrl(domain)
/**
* Parse favicons from the main page of the source`s website
*/
public override suspend fun getFavicons(): Favicons {
return FaviconParser(webClient, domain).parseFavicons()
}
@CallSuper
public override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) {
keys.add(configKeyDomain)
}
public override suspend fun getRelatedManga(seed: Manga): List<Manga> {
return RelatedMangaFinder(listOf(this)).invoke(seed)
}
/**
* Return [Manga] object by web link to it
* @see [Manga.publicUrl]
*/
internal open suspend fun resolveLink(resolver: LinkResolver, link: HttpUrl): Manga? = null
}

@ -1,145 +1,60 @@
package org.koitharu.kotatsu.parsers package org.koitharu.kotatsu.parsers
import androidx.annotation.CallSuper
import okhttp3.Headers import okhttp3.Headers
import okhttp3.HttpUrl
import org.koitharu.kotatsu.parsers.config.ConfigKey import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.config.MangaSourceConfig import org.koitharu.kotatsu.parsers.config.MangaSourceConfig
import org.koitharu.kotatsu.parsers.model.* import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.model.search.* import org.koitharu.kotatsu.parsers.model.search.MangaSearchQuery
import org.koitharu.kotatsu.parsers.network.OkHttpWebClient import org.koitharu.kotatsu.parsers.model.search.MangaSearchQueryCapabilities
import org.koitharu.kotatsu.parsers.network.WebClient
import org.koitharu.kotatsu.parsers.util.*
import java.util.* import java.util.*
public abstract class MangaParser @InternalParsersApi constructor( public interface MangaParser {
@property:InternalParsersApi public val context: MangaLoaderContext,
public val source: MangaParserSource, public val source: MangaParserSource
) {
/** /**
* Supported [SortOrder] variants. Must not be empty. * Supported [SortOrder] variants. Must not be empty.
* *
* For better performance use [EnumSet] for more than one item. * For better performance use [EnumSet] for more than one item.
*/ */
public abstract val availableSortOrders: Set<SortOrder> public val availableSortOrders: Set<SortOrder>
@Deprecated("Please check searchQueryCapabilities")
public abstract val filterCapabilities: MangaListFilterCapabilities
public open val searchQueryCapabilities: MangaSearchQueryCapabilities
get() = filterCapabilities.toMangaSearchQueryCapabilities()
public val config: MangaSourceConfig by lazy { context.getConfig(source) }
public open val sourceLocale: Locale public val searchQueryCapabilities: MangaSearchQueryCapabilities
get() = if (source.locale.isEmpty()) Locale.ROOT else Locale(source.locale)
protected val isNsfwSource: Boolean = source.contentType == ContentType.HENTAI public val config: MangaSourceConfig
/**
* Provide default domain and available alternatives, if any.
*
* Never hardcode domain in requests, use [domain] instead.
*/
@InternalParsersApi
public abstract val configKeyDomain: ConfigKey.Domain
protected open val userAgentKey: ConfigKey.UserAgent = ConfigKey.UserAgent(context.getDefaultUserAgent()) public val domain: String
public open fun getRequestHeaders(): Headers = Headers.Builder() public suspend fun queryManga(searchQuery: MangaSearchQuery): List<Manga>
.add("User-Agent", config[userAgentKey])
.build()
/**
* Used as fallback if value of `order` passed to [getList] is null
*/
public open val defaultSortOrder: SortOrder
get() {
val supported = availableSortOrders
return SortOrder.entries.first { it in supported }
}
@JvmField
protected val webClient: WebClient = OkHttpWebClient(context.httpClient, source)
/**
* Search list of manga by specified searchQuery
*
* @param searchQuery searchQuery
*/
public suspend fun queryManga(searchQuery: MangaSearchQuery): List<Manga> {
if (!searchQuery.skipValidation) {
searchQueryCapabilities.validate(searchQuery)
}
return getList(searchQuery)
}
/**
* Search list of manga by specified searchQuery
*
* @param query searchQuery
*/
protected open suspend fun getList(query: MangaSearchQuery): List<Manga> = getList(
offset = query.offset,
order = query.order ?: defaultSortOrder,
filter = convertToMangaListFilter(query),
)
/**
* Parse list of manga by specified criteria
*
* @param offset starting from 0 and used for pagination.
* Note than passed value may not be divisible by internal page size, so you should adjust it manually.
* @param order one of [availableSortOrders] or [defaultSortOrder] for default value
* @param filter is a set of filter rules
*
* @deprecated New [getList] should be preferred.
*/
@Deprecated("New getList(query: MangaSearchQuery) method should be preferred")
public abstract suspend fun getList(offset: Int, order: SortOrder, filter: MangaListFilter): List<Manga>
/** /**
* Parse details for [Manga]: chapters list, description, large cover, etc. * Parse details for [Manga]: chapters list, description, large cover, etc.
* Must return the same manga, may change any fields excepts id, url and source * Must return the same manga, may change any fields excepts id, url and source
* @see Manga.copy * @see Manga.copy
*/ */
public abstract suspend fun getDetails(manga: Manga): Manga public suspend fun getDetails(manga: Manga): Manga
/** /**
* Parse pages list for specified chapter. * Parse pages list for specified chapter.
* @see MangaPage for details * @see MangaPage for details
*/ */
public abstract suspend fun getPages(chapter: MangaChapter): List<MangaPage> public suspend fun getPages(chapter: MangaChapter): List<MangaPage>
/** /**
* Fetch direct link to the page image. * Fetch direct link to the page image.
*/ */
public open suspend fun getPageUrl(page: MangaPage): String = page.url.toAbsoluteUrl(domain) public suspend fun getPageUrl(page: MangaPage): String
public abstract suspend fun getFilterOptions(): MangaListFilterOptions public suspend fun getFilterOptions(): MangaListFilterOptions
/** /**
* Parse favicons from the main page of the source`s website * Parse favicons from the main page of the source`s website
*/ */
public open suspend fun getFavicons(): Favicons { public suspend fun getFavicons(): Favicons
return FaviconParser(webClient, domain).parseFavicons()
}
@CallSuper public fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>)
public open fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) {
keys.add(configKeyDomain)
}
public open suspend fun getRelatedManga(seed: Manga): List<Manga> { public suspend fun getRelatedManga(seed: Manga): List<Manga>
return RelatedMangaFinder(listOf(this)).invoke(seed)
}
/**
* Return [Manga] object by web link to it
* @see [Manga.publicUrl]
*/
internal open suspend fun resolveLink(resolver: LinkResolver, link: HttpUrl): Manga? = null
public fun getRequestHeaders(): Headers
} }

@ -16,7 +16,7 @@ public abstract class PagedMangaParser(
source: MangaParserSource, source: MangaParserSource,
@VisibleForTesting(otherwise = VisibleForTesting.PROTECTED) @JvmField public val pageSize: Int, @VisibleForTesting(otherwise = VisibleForTesting.PROTECTED) @JvmField public val pageSize: Int,
searchPageSize: Int = pageSize, searchPageSize: Int = pageSize,
) : MangaParser(context, source) { ) : AbstractMangaParser(context, source) {
@JvmField @JvmField
protected val paginator: Paginator = Paginator(pageSize) protected val paginator: Paginator = Paginator(pageSize)

@ -11,7 +11,7 @@ import org.koitharu.kotatsu.parsers.util.convertToMangaListFilter
public abstract class SinglePageMangaParser( public abstract class SinglePageMangaParser(
context: MangaLoaderContext, context: MangaLoaderContext,
source: MangaParserSource, source: MangaParserSource,
) : MangaParser(context, source) { ) : AbstractMangaParser(context, source) {
final override suspend fun getList(query: MangaSearchQuery): List<Manga> { final override suspend fun getList(query: MangaSearchQuery): List<Manga> {

@ -81,4 +81,9 @@ public data class MangaSearchQuery private constructor(
return uniqueCriteria.values.toSet() return uniqueCriteria.values.toSet()
} }
} }
public companion object {
public val EMPTY: MangaSearchQuery = MangaSearchQuery(emptySet(), null, 0, false)
}
} }

@ -11,7 +11,7 @@ import org.json.JSONArray
import org.json.JSONObject import org.json.JSONObject
import org.jsoup.Jsoup import org.jsoup.Jsoup
import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser import org.koitharu.kotatsu.parsers.AbstractMangaParser
import org.koitharu.kotatsu.parsers.MangaSourceParser import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.config.ConfigKey import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.* import org.koitharu.kotatsu.parsers.model.*
@ -28,7 +28,7 @@ import kotlin.math.min
@OptIn(ExperimentalUnsignedTypes::class) @OptIn(ExperimentalUnsignedTypes::class)
@MangaSourceParser("HITOMILA", "Hitomi.La", type = ContentType.HENTAI) @MangaSourceParser("HITOMILA", "Hitomi.La", type = ContentType.HENTAI)
internal class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaParserSource.HITOMILA) { internal class HitomiLaParser(context: MangaLoaderContext) : AbstractMangaParser(context, MangaParserSource.HITOMILA) {
override val configKeyDomain = ConfigKey.Domain("hitomi.la") override val configKeyDomain = ConfigKey.Domain("hitomi.la")
override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) {

@ -6,7 +6,7 @@ import okhttp3.HttpUrl
import okhttp3.HttpUrl.Companion.toHttpUrl import okhttp3.HttpUrl.Companion.toHttpUrl
import org.json.JSONObject import org.json.JSONObject
import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser import org.koitharu.kotatsu.parsers.AbstractMangaParser
import org.koitharu.kotatsu.parsers.MangaSourceParser import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.config.ConfigKey import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.exception.NotFoundException import org.koitharu.kotatsu.parsers.exception.NotFoundException
@ -21,7 +21,7 @@ import javax.crypto.spec.SecretKeySpec
internal abstract class LineWebtoonsParser( internal abstract class LineWebtoonsParser(
context: MangaLoaderContext, context: MangaLoaderContext,
source: MangaParserSource, source: MangaParserSource,
) : MangaParser(context, source) { ) : AbstractMangaParser(context, source) {
override val filterCapabilities: MangaListFilterCapabilities override val filterCapabilities: MangaListFilterCapabilities
get() = MangaListFilterCapabilities( get() = MangaListFilterCapabilities(

@ -8,7 +8,7 @@ import okhttp3.HttpUrl
import org.json.JSONArray import org.json.JSONArray
import org.json.JSONObject import org.json.JSONObject
import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser import org.koitharu.kotatsu.parsers.AbstractMangaParser
import org.koitharu.kotatsu.parsers.MangaSourceParser import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.config.ConfigKey import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.exception.ParseException import org.koitharu.kotatsu.parsers.exception.ParseException
@ -34,7 +34,7 @@ private const val SERVER_DATA = "data"
private const val SERVER_DATA_SAVER = "data-saver" private const val SERVER_DATA_SAVER = "data-saver"
@MangaSourceParser("MANGADEX", "MangaDex") @MangaSourceParser("MANGADEX", "MangaDex")
internal class MangaDexParser(context: MangaLoaderContext) : MangaParser(context, MangaParserSource.MANGADEX) { internal class MangaDexParser(context: MangaLoaderContext) : AbstractMangaParser(context, MangaParserSource.MANGADEX) {
override val configKeyDomain = ConfigKey.Domain("mangadex.org") override val configKeyDomain = ConfigKey.Domain("mangadex.org")

@ -8,7 +8,7 @@ import okhttp3.HttpUrl.Companion.toHttpUrl
import org.json.JSONArray import org.json.JSONArray
import org.json.JSONObject import org.json.JSONObject
import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser import org.koitharu.kotatsu.parsers.AbstractMangaParser
import org.koitharu.kotatsu.parsers.MangaSourceParser import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.config.ConfigKey import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.exception.NotFoundException import org.koitharu.kotatsu.parsers.exception.NotFoundException
@ -24,7 +24,7 @@ import javax.crypto.spec.SecretKeySpec
internal abstract class WebtoonsParser( internal abstract class WebtoonsParser(
context: MangaLoaderContext, context: MangaLoaderContext,
source: MangaParserSource, source: MangaParserSource,
) : MangaParser(context, source) { ) : AbstractMangaParser(context, source) {
private val signer by lazy { private val signer by lazy {
WebtoonsUrlSigner("gUtPzJFZch4ZyAGviiyH94P99lQ3pFdRTwpJWDlSGFfwgpr6ses5ALOxWHOIT7R1") WebtoonsUrlSigner("gUtPzJFZch4ZyAGviiyH94P99lQ3pFdRTwpJWDlSGFfwgpr6ses5ALOxWHOIT7R1")

@ -5,7 +5,7 @@ import org.json.JSONArray
import org.json.JSONObject import org.json.JSONObject
import org.koitharu.kotatsu.parsers.Broken import org.koitharu.kotatsu.parsers.Broken
import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser import org.koitharu.kotatsu.parsers.AbstractMangaParser
import org.koitharu.kotatsu.parsers.MangaSourceParser import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.config.ConfigKey import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.* import org.koitharu.kotatsu.parsers.model.*
@ -18,7 +18,7 @@ import java.util.*
@Broken @Broken
@MangaSourceParser("ANIBEL", "Anibel", "be") @MangaSourceParser("ANIBEL", "Anibel", "be")
internal class AnibelParser(context: MangaLoaderContext) : MangaParser(context, MangaParserSource.ANIBEL) { internal class AnibelParser(context: MangaLoaderContext) : AbstractMangaParser(context, MangaParserSource.ANIBEL) {
override val configKeyDomain = ConfigKey.Domain("anibel.net") override val configKeyDomain = ConfigKey.Domain("anibel.net")

@ -16,7 +16,7 @@ import java.util.EnumSet
import java.util.Locale import java.util.Locale
@MangaSourceParser("WEEBCENTRAL", "Weeb Central", "en") @MangaSourceParser("WEEBCENTRAL", "Weeb Central", "en")
internal class WeebCentral(context: MangaLoaderContext) : MangaParser(context, MangaParserSource.WEEBCENTRAL), internal class WeebCentral(context: MangaLoaderContext) : AbstractMangaParser(context, MangaParserSource.WEEBCENTRAL),
MangaParserAuthProvider { MangaParserAuthProvider {
override val configKeyDomain = ConfigKey.Domain("weebcentral.com") override val configKeyDomain = ConfigKey.Domain("weebcentral.com")

@ -1,7 +1,7 @@
package org.koitharu.kotatsu.parsers.site.ja package org.koitharu.kotatsu.parsers.site.ja
import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser import org.koitharu.kotatsu.parsers.AbstractMangaParser
import org.koitharu.kotatsu.parsers.MangaParserAuthProvider import org.koitharu.kotatsu.parsers.MangaParserAuthProvider
import org.koitharu.kotatsu.parsers.MangaSourceParser import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.config.ConfigKey import org.koitharu.kotatsu.parsers.config.ConfigKey
@ -16,7 +16,7 @@ private const val STATUS_FINISHED = "完結"
@MangaSourceParser("NICOVIDEO_SEIGA", "NicoVideo Seiga", "ja") @MangaSourceParser("NICOVIDEO_SEIGA", "NicoVideo Seiga", "ja")
internal class NicovideoSeigaParser(context: MangaLoaderContext) : internal class NicovideoSeigaParser(context: MangaLoaderContext) :
MangaParser(context, MangaParserSource.NICOVIDEO_SEIGA), AbstractMangaParser(context, MangaParserSource.NICOVIDEO_SEIGA),
MangaParserAuthProvider { MangaParserAuthProvider {
override val configKeyDomain: ConfigKey.Domain = ConfigKey.Domain("nicovideo.jp") override val configKeyDomain: ConfigKey.Domain = ConfigKey.Domain("nicovideo.jp")

@ -3,7 +3,7 @@ package org.koitharu.kotatsu.parsers.site.nepnep
import org.json.JSONArray import org.json.JSONArray
import org.json.JSONObject import org.json.JSONObject
import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser import org.koitharu.kotatsu.parsers.AbstractMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.* import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.network.UserAgents import org.koitharu.kotatsu.parsers.network.UserAgents
@ -20,7 +20,7 @@ internal abstract class NepnepParser(
context: MangaLoaderContext, context: MangaLoaderContext,
source: MangaParserSource, source: MangaParserSource,
domain: String, domain: String,
) : MangaParser(context, source) { ) : AbstractMangaParser(context, source) {
override val configKeyDomain = ConfigKey.Domain(domain) override val configKeyDomain = ConfigKey.Domain(domain)

@ -1,7 +1,7 @@
package org.koitharu.kotatsu.parsers.site.ru package org.koitharu.kotatsu.parsers.site.ru
import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser import org.koitharu.kotatsu.parsers.AbstractMangaParser
import org.koitharu.kotatsu.parsers.MangaParserAuthProvider import org.koitharu.kotatsu.parsers.MangaParserAuthProvider
import org.koitharu.kotatsu.parsers.MangaSourceParser import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.config.ConfigKey import org.koitharu.kotatsu.parsers.config.ConfigKey
@ -14,7 +14,7 @@ import java.util.*
@MangaSourceParser("NUDEMOON", "Nude-Moon", "ru", type = ContentType.HENTAI) @MangaSourceParser("NUDEMOON", "Nude-Moon", "ru", type = ContentType.HENTAI)
internal class NudeMoonParser( internal class NudeMoonParser(
context: MangaLoaderContext, context: MangaLoaderContext,
) : MangaParser(context, MangaParserSource.NUDEMOON), MangaParserAuthProvider { ) : AbstractMangaParser(context, MangaParserSource.NUDEMOON), MangaParserAuthProvider {
override val configKeyDomain = ConfigKey.Domain( override val configKeyDomain = ConfigKey.Domain(
"b.nude-moon.fun", "b.nude-moon.fun",

@ -17,7 +17,7 @@ import okio.IOException
import org.json.JSONArray import org.json.JSONArray
import org.jsoup.nodes.Element import org.jsoup.nodes.Element
import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser import org.koitharu.kotatsu.parsers.AbstractMangaParser
import org.koitharu.kotatsu.parsers.MangaParserAuthProvider import org.koitharu.kotatsu.parsers.MangaParserAuthProvider
import org.koitharu.kotatsu.parsers.config.ConfigKey import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.exception.AuthRequiredException import org.koitharu.kotatsu.parsers.exception.AuthRequiredException
@ -43,7 +43,7 @@ internal abstract class GroupleParser(
context: MangaLoaderContext, context: MangaLoaderContext,
source: MangaParserSource, source: MangaParserSource,
private val siteId: Int, private val siteId: Int,
) : MangaParser(context, source), MangaParserAuthProvider, Interceptor { ) : AbstractMangaParser(context, source), MangaParserAuthProvider, Interceptor {
@Volatile @Volatile
private var cachedPagesServer: String? = null private var cachedPagesServer: String? = null

@ -3,7 +3,7 @@ package org.koitharu.kotatsu.parsers.site.ru.multichan
import okhttp3.HttpUrl import okhttp3.HttpUrl
import org.jsoup.internal.StringUtil import org.jsoup.internal.StringUtil
import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser import org.koitharu.kotatsu.parsers.AbstractMangaParser
import org.koitharu.kotatsu.parsers.MangaParserAuthProvider import org.koitharu.kotatsu.parsers.MangaParserAuthProvider
import org.koitharu.kotatsu.parsers.exception.AuthRequiredException import org.koitharu.kotatsu.parsers.exception.AuthRequiredException
import org.koitharu.kotatsu.parsers.model.* import org.koitharu.kotatsu.parsers.model.*
@ -14,7 +14,7 @@ import java.util.*
internal abstract class ChanParser( internal abstract class ChanParser(
context: MangaLoaderContext, context: MangaLoaderContext,
source: MangaParserSource, source: MangaParserSource,
) : MangaParser(context, source), MangaParserAuthProvider { ) : AbstractMangaParser(context, source), MangaParserAuthProvider {
override val availableSortOrders: Set<SortOrder> = EnumSet.of( override val availableSortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.NEWEST, SortOrder.NEWEST,

@ -8,7 +8,7 @@ import okhttp3.Response
import org.json.JSONArray import org.json.JSONArray
import org.json.JSONObject import org.json.JSONObject
import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser import org.koitharu.kotatsu.parsers.AbstractMangaParser
import org.koitharu.kotatsu.parsers.MangaSourceParser import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.config.ConfigKey import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.* import org.koitharu.kotatsu.parsers.model.*
@ -25,7 +25,7 @@ private const val PAGE_SIZE = 60
// NOTE High profile focus // NOTE High profile focus
@MangaSourceParser("HENTAIUKR", "HentaiUkr", "uk", ContentType.HENTAI) @MangaSourceParser("HENTAIUKR", "HentaiUkr", "uk", ContentType.HENTAI)
internal class HentaiUkrParser(context: MangaLoaderContext) : MangaParser(context, MangaParserSource.HENTAIUKR), internal class HentaiUkrParser(context: MangaLoaderContext) : AbstractMangaParser(context, MangaParserSource.HENTAIUKR),
Interceptor { Interceptor {
private val date = SimpleDateFormat("yyyy-MM-dd", Locale.US) private val date = SimpleDateFormat("yyyy-MM-dd", Locale.US)

@ -8,7 +8,7 @@ import kotlinx.coroutines.sync.withLock
import org.jsoup.nodes.Document import org.jsoup.nodes.Document
import org.koitharu.kotatsu.parsers.Broken import org.koitharu.kotatsu.parsers.Broken
import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser import org.koitharu.kotatsu.parsers.AbstractMangaParser
import org.koitharu.kotatsu.parsers.MangaSourceParser import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.config.ConfigKey import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.* import org.koitharu.kotatsu.parsers.model.*
@ -21,7 +21,7 @@ private const val SEARCH_PAGE_SIZE = 10
@Broken @Broken
@MangaSourceParser("HENTAIVN", "HentaiVN", "vi", type = ContentType.HENTAI) @MangaSourceParser("HENTAIVN", "HentaiVN", "vi", type = ContentType.HENTAI)
internal class HentaiVNParser(context: MangaLoaderContext) : MangaParser(context, MangaParserSource.HENTAIVN) { internal class HentaiVNParser(context: MangaLoaderContext) : AbstractMangaParser(context, MangaParserSource.HENTAIVN) {
override val configKeyDomain: ConfigKey.Domain = ConfigKey.Domain("hentaihvn.tv") override val configKeyDomain: ConfigKey.Domain = ConfigKey.Domain("hentaihvn.tv")

@ -4,8 +4,8 @@ import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.runInterruptible import kotlinx.coroutines.runInterruptible
import okhttp3.HttpUrl import okhttp3.HttpUrl
import okhttp3.HttpUrl.Companion.toHttpUrlOrNull import okhttp3.HttpUrl.Companion.toHttpUrlOrNull
import org.koitharu.kotatsu.parsers.AbstractMangaParser
import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.model.* import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.suspendlazy.suspendLazy import org.koitharu.kotatsu.parsers.util.suspendlazy.suspendLazy
@ -19,14 +19,15 @@ public class LinkResolver internal constructor(
public suspend fun getSource(): MangaParserSource? = source.get() public suspend fun getSource(): MangaParserSource? = source.get()
public suspend fun getManga(): Manga? { public suspend fun getManga(): Manga? {
val parser = context.newParserInstance(source.get() ?: return null) val parser = context.newParserInstance(source.get() ?: return null) as? AbstractMangaParser
?: return null
return parser.resolveLink(this, link) ?: resolveManga(parser) return parser.resolveLink(this, link) ?: resolveManga(parser)
} }
private suspend fun resolveSource(): MangaParserSource? = runInterruptible(Dispatchers.Default) { private suspend fun resolveSource(): MangaParserSource? = runInterruptible(Dispatchers.Default) {
val domains = setOfNotNull(link.host, link.topPrivateDomain()) val domains = setOfNotNull(link.host, link.topPrivateDomain())
for (s in MangaParserSource.entries) { for (s in MangaParserSource.entries) {
val parser = context.newParserInstance(s) val parser = context.newParserInstance(s) as AbstractMangaParser
for (d in parser.configKeyDomain.presetValues) { for (d in parser.configKeyDomain.presetValues) {
if (d in domains) { if (d in domains) {
return@runInterruptible s return@runInterruptible s
@ -37,7 +38,7 @@ public class LinkResolver internal constructor(
} }
internal suspend fun resolveManga( internal suspend fun resolveManga(
parser: MangaParser, parser: AbstractMangaParser,
url: String = link.toString().toRelativeUrl(link.host), url: String = link.toString().toRelativeUrl(link.host),
id: Long = parser.generateUid(url), id: Long = parser.generateUid(url),
title: String = STUB_TITLE, title: String = STUB_TITLE,
@ -62,7 +63,7 @@ public class LinkResolver internal constructor(
), ),
) )
private suspend fun resolveBySeed(parser: MangaParser, s: Manga): Manga? { private suspend fun resolveBySeed(parser: AbstractMangaParser, s: Manga): Manga? {
val seed = parser.getDetails(s) val seed = parser.getDetails(s)
if (!parser.filterCapabilities.isSearchSupported) { if (!parser.filterCapabilities.isSearchSupported) {
return seed.takeUnless { it.chapters.isNullOrEmpty() } return seed.takeUnless { it.chapters.isNullOrEmpty() }

@ -2,6 +2,7 @@ package org.koitharu.kotatsu.parsers.util
import okhttp3.HttpUrl import okhttp3.HttpUrl
import org.jsoup.nodes.Element import org.jsoup.nodes.Element
import org.koitharu.kotatsu.parsers.AbstractMangaParser
import org.koitharu.kotatsu.parsers.ErrorMessages import org.koitharu.kotatsu.parsers.ErrorMessages
import org.koitharu.kotatsu.parsers.InternalParsersApi import org.koitharu.kotatsu.parsers.InternalParsersApi
import org.koitharu.kotatsu.parsers.MangaParser import org.koitharu.kotatsu.parsers.MangaParser
@ -81,17 +82,17 @@ private fun <T> Set<T>?.oneOrThrowIfMany(msg: String): T? = when {
else -> throw IllegalArgumentException(msg) else -> throw IllegalArgumentException(msg)
} }
public val MangaParser.domain: String public val AbstractMangaParser.domain: String
get() = config[configKeyDomain] get() = config[configKeyDomain]
@InternalParsersApi @InternalParsersApi
public fun MangaParser.getDomain(subdomain: String): String { public fun AbstractMangaParser.getDomain(subdomain: String): String {
val domain = domain val domain = domain
return subdomain + "." + domain.removePrefix("www.") return subdomain + "." + domain.removePrefix("www.")
} }
@InternalParsersApi @InternalParsersApi
public fun MangaParser.urlBuilder(subdomain: String? = null): HttpUrl.Builder { public fun AbstractMangaParser.urlBuilder(subdomain: String? = null): HttpUrl.Builder {
return HttpUrl.Builder() return HttpUrl.Builder()
.scheme(SCHEME_HTTPS) .scheme(SCHEME_HTTPS)
.host(if (subdomain == null) domain else "$subdomain.$domain") .host(if (subdomain == null) domain else "$subdomain.$domain")

@ -6,8 +6,10 @@ import kotlinx.coroutines.awaitAll
import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.coroutineScope
import org.koitharu.kotatsu.parsers.MangaParser import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.model.Manga import org.koitharu.kotatsu.parsers.model.Manga
import org.koitharu.kotatsu.parsers.model.MangaListFilter
import org.koitharu.kotatsu.parsers.model.SortOrder import org.koitharu.kotatsu.parsers.model.SortOrder
import org.koitharu.kotatsu.parsers.model.search.MangaSearchQuery
import org.koitharu.kotatsu.parsers.model.search.QueryCriteria
import org.koitharu.kotatsu.parsers.model.search.SearchableField
public class RelatedMangaFinder( public class RelatedMangaFinder(
private val parsers: Collection<MangaParser>, private val parsers: Collection<MangaParser>,
@ -34,7 +36,12 @@ public class RelatedMangaFinder(
} }
val results = words.map { keyword -> val results = words.map { keyword ->
scope.async { scope.async {
val result = parser.getList(0, SortOrder.RELEVANCE, MangaListFilter(query = keyword)) val result = parser.queryManga(
MangaSearchQuery.Builder()
.order(SortOrder.RELEVANCE)
.criterion(QueryCriteria.Match(SearchableField.TITLE_NAME, keyword))
.build(),
)
result.filter { it.id != seed.id && it.containKeyword(keyword) } result.filter { it.id != seed.id && it.containKeyword(keyword) }
} }
}.awaitAll() }.awaitAll()

@ -5,8 +5,6 @@ import okhttp3.Request
import okhttp3.Response import okhttp3.Response
import org.koitharu.kotatsu.parsers.model.MangaParserSource import org.koitharu.kotatsu.parsers.model.MangaParserSource
import org.koitharu.kotatsu.parsers.model.MangaSource import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.util.domain
import org.koitharu.kotatsu.parsers.util.mergeWith
private const val HEADER_REFERER = "Referer" private const val HEADER_REFERER = "Referer"
@ -20,11 +18,7 @@ internal class CommonHeadersInterceptor : Interceptor {
} else { } else {
null null
} }
val sourceHeaders = parser?.getRequestHeaders()
val headersBuilder = request.headers.newBuilder() val headersBuilder = request.headers.newBuilder()
if (sourceHeaders != null) {
headersBuilder.mergeWith(sourceHeaders, replaceExisting = false)
}
if (headersBuilder[HEADER_REFERER] == null && parser != null) { if (headersBuilder[HEADER_REFERER] == null && parser != null) {
headersBuilder[HEADER_REFERER] = "https://${parser.domain}/" headersBuilder[HEADER_REFERER] = "https://${parser.domain}/"
} }

@ -10,7 +10,6 @@ import org.koitharu.kotatsu.parsers.model.search.MangaSearchQuery
import org.koitharu.kotatsu.parsers.model.search.QueryCriteria import org.koitharu.kotatsu.parsers.model.search.QueryCriteria
import org.koitharu.kotatsu.parsers.model.search.QueryCriteria.Include import org.koitharu.kotatsu.parsers.model.search.QueryCriteria.Include
import org.koitharu.kotatsu.parsers.model.search.SearchableField.* import org.koitharu.kotatsu.parsers.model.search.SearchableField.*
import org.koitharu.kotatsu.parsers.util.domain
import org.koitharu.kotatsu.parsers.util.medianOrNull import org.koitharu.kotatsu.parsers.util.medianOrNull
import org.koitharu.kotatsu.parsers.util.mimeType import org.koitharu.kotatsu.parsers.util.mimeType
import org.koitharu.kotatsu.test_util.* import org.koitharu.kotatsu.test_util.*
@ -26,7 +25,7 @@ internal class MangaParserTest {
@MangaSources @MangaSources
fun list(source: MangaParserSource) = runTest(timeout = timeout) { fun list(source: MangaParserSource) = runTest(timeout = timeout) {
val parser = context.newParserInstance(source) val parser = context.newParserInstance(source)
val list = parser.getList(0, parser.defaultSortOrder, MangaListFilter.EMPTY) val list = parser.queryManga(MangaSearchQuery.Builder().build())
checkMangaList(list, "list") checkMangaList(list, "list")
assert(list.all { it.source == source }) assert(list.all { it.source == source })
} }
@ -38,9 +37,9 @@ internal class MangaParserTest {
if (parser is SinglePageMangaParser) { if (parser is SinglePageMangaParser) {
return@runTest return@runTest
} }
val page1 = parser.queryManga(MangaSearchQuery.Builder().offset(0).order(parser.defaultSortOrder).build()) val page1 = parser.queryManga(MangaSearchQuery.EMPTY)
val page2 = val page2 =
parser.queryManga(MangaSearchQuery.Builder().offset(page1.size).order(parser.defaultSortOrder).build()) parser.queryManga(MangaSearchQuery.Builder().offset(page1.size).build())
if (parser is PagedMangaParser) { if (parser is PagedMangaParser) {
assert(parser.pageSize >= page1.size) { assert(parser.pageSize >= page1.size) {
"Page size is ${page1.size} but ${parser.pageSize} expected" "Page size is ${page1.size} but ${parser.pageSize} expected"
@ -59,12 +58,7 @@ internal class MangaParserTest {
@MangaSources @MangaSources
fun searchByTitleName(source: MangaParserSource) = runTest(timeout = timeout) { fun searchByTitleName(source: MangaParserSource) = runTest(timeout = timeout) {
val parser = context.newParserInstance(source) val parser = context.newParserInstance(source)
val subject = parser.queryManga( val subject = parser.queryManga(MangaSearchQuery.EMPTY).minByOrNull {
MangaSearchQuery.Builder()
.offset(0)
.order(parser.defaultSortOrder)
.build(),
).minByOrNull {
it.title.length it.title.length
} ?: error("No manga found") } ?: error("No manga found")
@ -106,7 +100,6 @@ internal class MangaParserTest {
val list = parser.queryManga( val list = parser.queryManga(
MangaSearchQuery.Builder() MangaSearchQuery.Builder()
.offset(0) .offset(0)
.order(parser.defaultSortOrder)
.criterion(Include(TAG, setOf(tag))) .criterion(Include(TAG, setOf(tag)))
.build(), .build(),
) )
@ -118,13 +111,12 @@ internal class MangaParserTest {
@MangaSources @MangaSources
fun tagsMultiple(source: MangaParserSource) = runTest(timeout = timeout) { fun tagsMultiple(source: MangaParserSource) = runTest(timeout = timeout) {
val parser = context.newParserInstance(source) val parser = context.newParserInstance(source)
if (!parser.filterCapabilities.isMultipleTagsSupported) return@runTest // if (!parser.filterCapabilities.isMultipleTagsSupported) return@runTest
val tags = parser.getFilterOptions().availableTags.shuffled().take(2).toSet() val tags = parser.getFilterOptions().availableTags.shuffled().take(2).toSet()
val list = parser.queryManga( val list = parser.queryManga(
MangaSearchQuery.Builder() MangaSearchQuery.Builder()
.offset(0) .offset(0)
.order(parser.defaultSortOrder)
.criterion(Include(TAG, tags)) .criterion(Include(TAG, tags))
.build(), .build(),
) )
@ -144,8 +136,6 @@ internal class MangaParserTest {
val locale = locales.random() val locale = locales.random()
val list = parser.queryManga( val list = parser.queryManga(
MangaSearchQuery.Builder() MangaSearchQuery.Builder()
.offset(0)
.order(parser.defaultSortOrder)
.criterion(Include(LANGUAGE, setOf(locale))) .criterion(Include(LANGUAGE, setOf(locale)))
.criterion(Include(LANGUAGE, setOf(locale))) .criterion(Include(LANGUAGE, setOf(locale)))
.criterion(Include(ORIGINAL_LANGUAGE, setOf(locales.random()))) .criterion(Include(ORIGINAL_LANGUAGE, setOf(locales.random())))
@ -160,7 +150,7 @@ internal class MangaParserTest {
@MangaSources @MangaSources
fun details(source: MangaParserSource) = runTest(timeout = timeout) { fun details(source: MangaParserSource) = runTest(timeout = timeout) {
val parser = context.newParserInstance(source) val parser = context.newParserInstance(source)
val list = parser.queryManga(MangaSearchQuery.Builder().offset(0).order(parser.defaultSortOrder).build()) val list = parser.queryManga(MangaSearchQuery.EMPTY)
val manga = list[0] val manga = list[0]
parser.getDetails(manga).apply { parser.getDetails(manga).apply {
@ -191,7 +181,7 @@ internal class MangaParserTest {
@MangaSources @MangaSources
fun pages(source: MangaParserSource) = runTest(timeout = timeout) { fun pages(source: MangaParserSource) = runTest(timeout = timeout) {
val parser = context.newParserInstance(source) val parser = context.newParserInstance(source)
val list = parser.queryManga(MangaSearchQuery.Builder().offset(0).order(parser.defaultSortOrder).build()) val list = parser.queryManga(MangaSearchQuery.EMPTY)
val manga = list.first() val manga = list.first()
val chapter = parser.getDetails(manga).chapters?.firstOrNull() ?: error("Chapter is null at ${manga.publicUrl}") val chapter = parser.getDetails(manga).chapters?.firstOrNull() ?: error("Chapter is null at ${manga.publicUrl}")
val pages = parser.getPages(chapter) val pages = parser.getPages(chapter)
@ -246,8 +236,7 @@ internal class MangaParserTest {
@MangaSources @MangaSources
fun link(source: MangaParserSource) = runTest(timeout = timeout) { fun link(source: MangaParserSource) = runTest(timeout = timeout) {
val parser = context.newParserInstance(source) val parser = context.newParserInstance(source)
val manga = val manga = parser.queryManga(MangaSearchQuery.Builder().build()).first()
parser.queryManga(MangaSearchQuery.Builder().offset(0).order(parser.defaultSortOrder).build()).first()
val resolved = context.newLinkResolver(manga.publicUrl).getManga() val resolved = context.newLinkResolver(manga.publicUrl).getManga()
Assertions.assertNotNull(resolved) Assertions.assertNotNull(resolved)
resolved ?: return@runTest resolved ?: return@runTest

@ -1,6 +1,7 @@
package org.koitharu.kotatsu.parsers.util package org.koitharu.kotatsu.parsers.util
import org.junit.jupiter.api.Test import org.junit.jupiter.api.Test
import org.koitharu.kotatsu.parsers.AbstractMangaParser
import org.koitharu.kotatsu.parsers.MangaLoaderContextMock import org.koitharu.kotatsu.parsers.MangaLoaderContextMock
import org.koitharu.kotatsu.parsers.model.MangaParserSource import org.koitharu.kotatsu.parsers.model.MangaParserSource
import org.koitharu.kotatsu.parsers.newParser import org.koitharu.kotatsu.parsers.newParser
@ -25,7 +26,7 @@ class IntentFilterGenerator {
if (source == MangaParserSource.DUMMY) { if (source == MangaParserSource.DUMMY) {
continue continue
} }
val parser = source.newParser(MangaLoaderContextMock) val parser = source.newParser(MangaLoaderContextMock) as AbstractMangaParser
parser.configKeyDomain.presetValues.forEach { domain -> parser.configKeyDomain.presetValues.forEach { domain ->
writer.appendTab().append("<data android:host=\"").append(domain).appendLine("\" />") writer.appendTab().append("<data android:host=\"").append(domain).appendLine("\" />")
} }

Loading…
Cancel
Save