KotatsuApp
/
Kotatsu
2
0
Fork 0
Code Issues 124 Pull Requests Packages Projects Releases 163 Wiki Activity

Merge pull request #146 from KotatsuApp/feature/refactor2

Browse Source 
Improve network requests processing
Koitharu 3 years ago committed by GitHub
parent 91109bf6dc 05d705ac03
commit 624a3dd991
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
57 changed files with 6535 additions and 6669 deletions
Show Stats Download Patch File Download Diff File

4
.editorconfig
Unescape Escape View File

@ -6,8 +6,8 @@ end_of_line = lf
indent_style = tab
max_line_length = 120
tab_width = 4
insert_final_newline = false
disabled_rules=no-wildcard-imports,no-unused-imports
insert_final_newline = true
disabled_rules = no-wildcard-imports, no-unused-imports
[{*.kt,*.kts}]
ij_kotlin_allow_trailing_comma = true

4
.github/workflows/test-parsers.yml vendored
Unescape Escape View File

@ -4,7 +4,7 @@ on:
workflow_dispatch:
pull_request:
paths:
- 'src/main/kotlin/org/koitharu/kotatsu/parsers/site/*'
- 'src/main/kotlin/org/koitharu/kotatsu/parsers/site/**'
permissions:
contents: read
@ -24,4 +24,4 @@ jobs:
- uses: actions/upload-artifact@v3
with:
name: Report
path: build/test-results-html/TEST-org.koitharu.kotatsu.parsers.MangaParserTest.htm
path: build/test-results-html/TEST-org.koitharu.kotatsu.parsers.MangaParserTest.htm

6
CONTRIBUTING.md
Unescape Escape View File

@ -59,13 +59,13 @@ All functions in `MangaParser` class are documented. Pay attention to some pecul
- `sortOrders` set should not be empty. If your source is not support sorting, specify one most relevance value.
- If you cannot obtain direct links to pages images inside `getPages` method, it is ok to use an intermediate url
as `Page.url` and fetch a direct link at `getPageUrl` function.
- `getFaviconUrl` function is deprecated, but Kotatsu application before v4 still use it, so it is recommended to
override it too.
- You can use _asserts_ to check some optional fields. For example. `Manga.author` field is not required, but if your
source provide such information, add `assert(it != null)`. This will not have any effect on production but help to
find issues during unit testing.
- If your source website (or it's api) uses pages for pagination instead of offset you should extend `PagedMangaParser`
instead of `MangaParser`.
- Your parser may also implement the `Interceptor` interface for additional manipulation of all network requests and/or
responses, including image loading.
## Development process
@ -87,4 +87,4 @@ It is recommended to run unit tests before submitting a PR.
## Help
If you need a help or have some questions, ask a community in our [Discord server](https://discord.gg/NNJ5RgVBC5).
If you need a help or have some questions, ask a community in our [Discord server](https://discord.gg/NNJ5RgVBC5).

167
src/main/kotlin/org/koitharu/kotatsu/parsers/MangaLoaderContext.kt
Unescape Escape View File

@ -1,166 +1,29 @@
package org.koitharu.kotatsu.parsers
import okhttp3.*
import okhttp3.HttpUrl.Companion.toHttpUrl
import okhttp3.MediaType.Companion.toMediaType
import okhttp3.RequestBody.Companion.toRequestBody
import org.json.JSONObject
import org.jsoup.HttpStatusException
import okhttp3.CookieJar
import okhttp3.OkHttpClient
import org.koitharu.kotatsu.parsers.config.MangaSourceConfig
import org.koitharu.kotatsu.parsers.exception.GraphQLException
import org.koitharu.kotatsu.parsers.exception.NotFoundException
import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.util.await
import org.koitharu.kotatsu.parsers.util.parseJson
import java.util.*
abstract class MangaLoaderContext {
protected abstract val httpClient: OkHttpClient
abstract val httpClient: OkHttpClient
abstract val cookieJar: CookieJar
abstract val cookieJar: CookieJar
/**
* Do a GET http request to specific url
* @param url
* @param headers an additional headers for request, may be null
*/
suspend fun httpGet(url: HttpUrl, headers: Headers? = null): Response {
val request = Request.Builder()
.get()
.url(url)
if (headers != null) {
request.headers(headers)
}
return httpClient.newCall(request.build()).await().ensureSuccess()
}
open fun encodeBase64(data: ByteArray): String = Base64.getEncoder().encodeToString(data)
suspend fun httpGet(url: String, headers: Headers? = null): Response {
return httpGet(url.toHttpUrl(), headers)
}
open fun decodeBase64(data: String): ByteArray = Base64.getDecoder().decode(data)
open fun getPreferredLocales(): List<Locale> = listOf(Locale.getDefault())
/**
* Do a HEAD http request to specific url
* @param url
* @param headers an additional headers for request, may be null
*/
suspend fun httpHead(url: String, headers: Headers? = null): Response {
val request = Request.Builder()
.head()
.url(url)
if (headers != null) {
request.headers(headers)
}
return httpClient.newCall(request.build()).await().ensureSuccess()
}
/**
* Execute JavaScript code and return result
* @param script JavaScript source code
* @return execution result as string, may be null
*/
abstract suspend fun evaluateJs(script: String): String?
/**
* Do a POST http request to specific url with `multipart/form-data` payload
* @param url
* @param form payload as key=>value map
* @param headers an additional headers for request, may be null
*/
suspend fun httpPost(
url: String,
form: Map<String, String>,
headers: Headers? = null,
): Response {
val body = FormBody.Builder()
form.forEach { (k, v) ->
body.addEncoded(k, v)
}
val request = Request.Builder()
.post(body.build())
.url(url)
if (headers != null) {
request.headers(headers)
}
return httpClient.newCall(request.build()).await().ensureSuccess()
}
/**
* Do a POST http request to specific url with `multipart/form-data` payload
* @param url
* @param payload payload as `key=value` string with `&` separator
* @param headers an additional headers for request, may be null
*/
suspend fun httpPost(
url: String,
payload: String,
headers: Headers?,
): Response {
val body = FormBody.Builder()
payload.split('&').forEach {
val pos = it.indexOf('=')
if (pos != -1) {
val k = it.substring(0, pos)
val v = it.substring(pos + 1)
body.addEncoded(k, v)
}
}
val request = Request.Builder()
.post(body.build())
.url(url)
if (headers != null) {
request.headers(headers)
}
return httpClient.newCall(request.build()).await().ensureSuccess()
}
/**
* Do a GraphQL request to specific url
* @param endpoint an url
* @param query GraphQL request payload
*/
suspend fun graphQLQuery(endpoint: String, query: String): JSONObject {
val body = JSONObject()
body.put("operationName", null as Any?)
body.put("variables", JSONObject())
body.put("query", "{$query}")
val mediaType = "application/json; charset=utf-8".toMediaType()
val requestBody = body.toString().toRequestBody(mediaType)
val request = Request.Builder()
.post(requestBody)
.url(endpoint)
val json = httpClient.newCall(request.build()).await().ensureSuccess().parseJson()
json.optJSONArray("errors")?.let {
if (it.length() != 0) {
throw GraphQLException(it)
}
}
return json
}
open fun encodeBase64(data: ByteArray): String = Base64.getEncoder().encodeToString(data)
open fun decodeBase64(data: String): ByteArray = Base64.getDecoder().decode(data)
open fun getPreferredLocales(): List<Locale> = listOf(Locale.getDefault())
/**
* Execute JavaScript code and return result
* @param script JavaScript source code
* @return execution result as string, may be null
*/
abstract suspend fun evaluateJs(script: String): String?
abstract fun getConfig(source: MangaSource): MangaSourceConfig
private fun Response.ensureSuccess(): Response {
val exception: Exception? = when (code) { // Catch some error codes, not all
404 -> NotFoundException(message, request.url.toString())
in 500..599 -> HttpStatusException(message, code, request.url.toString())
else -> null
}
if (exception != null) {
runCatching {
close()
}.onFailure {
exception.addSuppressed(it)
}
throw exception
}
return this
}
}
abstract fun getConfig(source: MangaSource): MangaSourceConfig
}

304
src/main/kotlin/org/koitharu/kotatsu/parsers/MangaParser.kt
Unescape Escape View File

@ -2,197 +2,125 @@ package org.koitharu.kotatsu.parsers
import androidx.annotation.CallSuper
import okhttp3.Headers
import okhttp3.HttpUrl
import org.jsoup.nodes.Element
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.exception.ParseException
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.network.OkHttpWebClient
import org.koitharu.kotatsu.parsers.network.WebClient
import org.koitharu.kotatsu.parsers.util.FaviconParser
import org.koitharu.kotatsu.parsers.util.domain
import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl
import java.util.*
abstract class MangaParser @InternalParsersApi constructor(val source: MangaSource) {
protected abstract val context: MangaLoaderContext
/**
* Supported [SortOrder] variants. Must not be empty.
*
* For better performance use [EnumSet] for more than one item.
*/
abstract val sortOrders: Set<SortOrder>
val config by lazy { context.getConfig(source) }
open val sourceLocale: Locale?
get() = source.locale?.let { Locale(it) }
/**
* Provide default domain and available alternatives, if any.
*
* Never hardcode domain in requests, use [getDomain] instead.
*/
protected abstract val configKeyDomain: ConfigKey.Domain
open val headers: Headers? = null
/**
* Used as fallback if value of `sortOrder` passed to [getList] is null
*/
protected open val defaultSortOrder: SortOrder
get() {
val supported = sortOrders
return SortOrder.values().first { it in supported }
}
/**
* Parse list of manga by specified criteria
*
* @param offset starting from 0 and used for pagination.
* Note than passed value may not be divisible by internal page size, so you should adjust it manually.
* @param query search query, may be null or empty if no search needed
* @param tags genres for filtering, values from [getTags] and [Manga.tags]. May be null or empty
* @param sortOrder one of [sortOrders] or null for default value
*/
@JvmSynthetic
@InternalParsersApi
abstract suspend fun getList(
offset: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga>
/**
* Parse list of manga with search by text query
*
* @param offset starting from 0 and used for pagination.
* @param query search query
*/
open suspend fun getList(offset: Int, query: String): List<Manga> {
return getList(offset, query, null, defaultSortOrder)
}
/**
* Parse list of manga by specified criteria
*
* @param offset starting from 0 and used for pagination.
* Note than passed value may not be divisible by internal page size, so you should adjust it manually.
* @param tags genres for filtering, values from [getTags] and [Manga.tags]. May be null or empty
* @param sortOrder one of [sortOrders] or null for default value
*/
open suspend fun getList(offset: Int, tags: Set<MangaTag>?, sortOrder: SortOrder?): List<Manga> {
return getList(offset, null, tags, sortOrder ?: defaultSortOrder)
}
/**
* Parse details for [Manga]: chapters list, description, large cover, etc.
* Must return the same manga, may change any fields excepts id, url and source
* @see Manga.copy
*/
abstract suspend fun getDetails(manga: Manga): Manga
/**
* Parse pages list for specified chapter.
* @see MangaPage for details
*/
abstract suspend fun getPages(chapter: MangaChapter): List<MangaPage>
/**
* Fetch direct link to the page image.
*/
open suspend fun getPageUrl(page: MangaPage): String = page.url.toAbsoluteUrl(getDomain())
/**
* Fetch available tags (genres) for source
*/
abstract suspend fun getTags(): Set<MangaTag>
/**
* Returns direct link to the website favicon
*/
@Deprecated(
message = "Use parseFavicons() to get multiple favicons with different size",
replaceWith = ReplaceWith("parseFavicons()"),
)
open fun getFaviconUrl() = "https://${getDomain()}/favicon.ico"
/**
* Parse favicons from the main page of the source`s website
*/
open suspend fun getFavicons(): Favicons {
return FaviconParser(context, getDomain(), headers).parseFavicons()
}
@CallSuper
open fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) {
keys.add(configKeyDomain)
}
/* Utils */
fun getDomain(): String {
return config[configKeyDomain]
}
fun getDomain(subdomain: String): String {
val domain = getDomain()
return subdomain + "." + domain.removePrefix("www.")
}
fun urlBuilder(): HttpUrl.Builder {
return HttpUrl.Builder()
.scheme("https")
.host(getDomain())
}
/**
* Create a unique id for [Manga]/[MangaChapter]/[MangaPage].
* @param url must be relative url, without a domain
* @see [Manga.id]
* @see [MangaChapter.id]
* @see [MangaPage.id]
*/
@InternalParsersApi
protected fun generateUid(url: String): Long {
var h = 1125899906842597L
source.name.forEach { c ->
h = 31 * h + c.code
}
url.forEach { c ->
h = 31 * h + c.code
}
return h
}
/**
* Create a unique id for [Manga]/[MangaChapter]/[MangaPage].
* @param id an internal identifier
* @see [Manga.id]
* @see [MangaChapter.id]
* @see [MangaPage.id]
*/
@InternalParsersApi
protected fun generateUid(id: Long): Long {
var h = 1125899906842597L
source.name.forEach { c ->
h = 31 * h + c.code
}
h = 31 * h + id
return h
}
@InternalParsersApi
protected fun Element.parseFailed(message: String? = null): Nothing {
throw ParseException(message, ownerDocument()?.location() ?: baseUri(), null)
}
@InternalParsersApi
protected fun Set<MangaTag>?.oneOrThrowIfMany(): MangaTag? {
return when {
isNullOrEmpty() -> null
size == 1 -> first()
else -> throw IllegalArgumentException("Multiple genres are not supported by this source")
}
}
}
abstract class MangaParser @InternalParsersApi constructor(
@property:InternalParsersApi val context: MangaLoaderContext,
val source: MangaSource,
) {
/**
* Supported [SortOrder] variants. Must not be empty.
*
* For better performance use [EnumSet] for more than one item.
*/
abstract val sortOrders: Set<SortOrder>
val config by lazy { context.getConfig(source) }
open val sourceLocale: Locale?
get() = source.locale?.let { Locale(it) }
/**
* Provide default domain and available alternatives, if any.
*
* Never hardcode domain in requests, use [getDomain] instead.
*/
@InternalParsersApi
abstract val configKeyDomain: ConfigKey.Domain
open val headers: Headers? = null
/**
* Used as fallback if value of `sortOrder` passed to [getList] is null
*/
protected open val defaultSortOrder: SortOrder
get() {
val supported = sortOrders
return SortOrder.values().first { it in supported }
}
protected val webClient: WebClient = OkHttpWebClient(context.httpClient, source)
/**
* Parse list of manga by specified criteria
*
* @param offset starting from 0 and used for pagination.
* Note than passed value may not be divisible by internal page size, so you should adjust it manually.
* @param query search query, may be null or empty if no search needed
* @param tags genres for filtering, values from [getTags] and [Manga.tags]. May be null or empty
* @param sortOrder one of [sortOrders] or null for default value
*/
@JvmSynthetic
@InternalParsersApi
abstract suspend fun getList(
offset: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga>
/**
* Parse list of manga with search by text query
*
* @param offset starting from 0 and used for pagination.
* @param query search query
*/
open suspend fun getList(offset: Int, query: String): List<Manga> {
return getList(offset, query, null, defaultSortOrder)
}
/**
* Parse list of manga by specified criteria
*
* @param offset starting from 0 and used for pagination.
* Note than passed value may not be divisible by internal page size, so you should adjust it manually.
* @param tags genres for filtering, values from [getTags] and [Manga.tags]. May be null or empty
* @param sortOrder one of [sortOrders] or null for default value
*/
open suspend fun getList(offset: Int, tags: Set<MangaTag>?, sortOrder: SortOrder?): List<Manga> {
return getList(offset, null, tags, sortOrder ?: defaultSortOrder)
}
/**
* Parse details for [Manga]: chapters list, description, large cover, etc.
* Must return the same manga, may change any fields excepts id, url and source
* @see Manga.copy
*/
abstract suspend fun getDetails(manga: Manga): Manga
/**
* Parse pages list for specified chapter.
* @see MangaPage for details
*/
abstract suspend fun getPages(chapter: MangaChapter): List<MangaPage>
/**
* Fetch direct link to the page image.
*/
open suspend fun getPageUrl(page: MangaPage): String = page.url.toAbsoluteUrl(domain)
/**
* Fetch available tags (genres) for source
*/
abstract suspend fun getTags(): Set<MangaTag>
/**
* Parse favicons from the main page of the source`s website
*/
open suspend fun getFavicons(): Favicons {
return FaviconParser(webClient, domain).parseFavicons()
}
@CallSuper
open fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) {
keys.add(configKeyDomain)
}
}

81
src/main/kotlin/org/koitharu/kotatsu/parsers/PagedMangaParser.kt
Unescape Escape View File

@ -8,43 +8,44 @@ import org.koitharu.kotatsu.parsers.util.Paginator
@InternalParsersApi
abstract class PagedMangaParser(
source: MangaSource,
pageSize: Int,
searchPageSize: Int = pageSize,
) : MangaParser(source) {
protected val paginator = Paginator(pageSize)
protected val searchPaginator = Paginator(searchPageSize)
override suspend fun getList(offset: Int, query: String): List<Manga> {
return getList(searchPaginator, offset, query, null, defaultSortOrder)
}
override suspend fun getList(offset: Int, tags: Set<MangaTag>?, sortOrder: SortOrder?): List<Manga> {
return getList(paginator, offset, null, tags, sortOrder ?: defaultSortOrder)
}
@InternalParsersApi
@Deprecated("You should use getListPage for PagedMangaParser", level = DeprecationLevel.HIDDEN)
final override suspend fun getList(
offset: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> = throw UnsupportedOperationException("You should use getListPage for PagedMangaParser")
abstract suspend fun getListPage(page: Int, query: String?, tags: Set<MangaTag>?, sortOrder: SortOrder): List<Manga>
private suspend fun getList(
paginator: Paginator,
offset: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val page = paginator.getPage(offset)
val list = getListPage(page, query, tags, sortOrder)
paginator.onListReceived(offset, page, list.size)
return list
}
}
context: MangaLoaderContext,
source: MangaSource,
pageSize: Int,
searchPageSize: Int = pageSize,
) : MangaParser(context, source) {
protected val paginator = Paginator(pageSize)
protected val searchPaginator = Paginator(searchPageSize)
override suspend fun getList(offset: Int, query: String): List<Manga> {
return getList(searchPaginator, offset, query, null, defaultSortOrder)
}
override suspend fun getList(offset: Int, tags: Set<MangaTag>?, sortOrder: SortOrder?): List<Manga> {
return getList(paginator, offset, null, tags, sortOrder ?: defaultSortOrder)
}
@InternalParsersApi
@Deprecated("You should use getListPage for PagedMangaParser", level = DeprecationLevel.HIDDEN)
final override suspend fun getList(
offset: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> = throw UnsupportedOperationException("You should use getListPage for PagedMangaParser")
abstract suspend fun getListPage(page: Int, query: String?, tags: Set<MangaTag>?, sortOrder: SortOrder): List<Manga>
private suspend fun getList(
paginator: Paginator,
offset: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val page = paginator.getPage(offset)
val list = getListPage(page, query, tags, sortOrder)
paginator.onListReceived(offset, page, list.size)
return list
}
}

83
src/main/kotlin/org/koitharu/kotatsu/parsers/model/MangaPage.kt
Unescape Escape View File

@ -3,50 +3,49 @@ package org.koitharu.kotatsu.parsers.model
import org.koitharu.kotatsu.parsers.MangaParser
class MangaPage(
/**
* Unique identifier for manga
*/
val id: Long,
/**
* Relative url to page (**without** a domain) or any other uri.
* Used principally in parsers.
* May contain link to image or html page.
* @see MangaParser.getPageUrl
*/
val url: String,
/**
* Absolute link to the chapter or website home page.
* Used in Referer header
*/
val referer: String,
/**
* Absolute url of the small page image if exists, null otherwise
*/
val preview: String?,
val source: MangaSource,
/**
* Unique identifier for manga
*/
val id: Long,
/**
* Relative url to page (**without** a domain) or any other uri.
* Used principally in parsers.
* May contain link to image or html page.
* @see MangaParser.getPageUrl
*/
val url: String,
/**
* Absolute link to the chapter or website home page.
* Used in Referer header
*/
@Deprecated("Referer header should be added based on source domain")
val referer: String,
/**
* Absolute url of the small page image if exists, null otherwise
*/
val preview: String?,
val source: MangaSource,
) {
override fun equals(other: Any?): Boolean {
if (this === other) return true
if (javaClass != other?.javaClass) return false
override fun equals(other: Any?): Boolean {
if (this === other) return true
if (javaClass != other?.javaClass) return false
other as MangaPage
other as MangaPage
if (id != other.id) return false
if (url != other.url) return false
if (referer != other.referer) return false
if (preview != other.preview) return false
if (source != other.source) return false
if (id != other.id) return false
if (url != other.url) return false
if (referer != other.referer) return false
if (preview != other.preview) return false
return source == other.source
}
return true
}
override fun hashCode(): Int {
var result = id.hashCode()
result = 31 * result + url.hashCode()
result = 31 * result + referer.hashCode()
result = 31 * result + (preview?.hashCode() ?: 0)
result = 31 * result + source.hashCode()
return result
}
}
override fun hashCode(): Int {
var result = id.hashCode()
result = 31 * result + url.hashCode()
result = 31 * result + referer.hashCode()
result = 31 * result + (preview?.hashCode() ?: 0)
result = 31 * result + source.hashCode()
return result
}
}

107
src/main/kotlin/org/koitharu/kotatsu/parsers/network/OkHttpWebClient.kt
Unescape Escape View File

@ -0,0 +1,107 @@
package org.koitharu.kotatsu.parsers.network
import okhttp3.*
import okhttp3.MediaType.Companion.toMediaType
import okhttp3.RequestBody.Companion.toRequestBody
import org.json.JSONObject
import org.jsoup.HttpStatusException
import org.koitharu.kotatsu.parsers.exception.GraphQLException
import org.koitharu.kotatsu.parsers.exception.NotFoundException
import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.util.await
import org.koitharu.kotatsu.parsers.util.parseJson
class OkHttpWebClient(
private val httpClient: OkHttpClient,
private val mangaSource: MangaSource,
) : WebClient {
override suspend fun httpGet(url: HttpUrl): Response {
val request = Request.Builder()
.get()
.url(url)
.addTags()
return httpClient.newCall(request.build()).await().ensureSuccess()
}
override suspend fun httpHead(url: HttpUrl): Response {
val request = Request.Builder()
.head()
.url(url)
.addTags()
return httpClient.newCall(request.build()).await().ensureSuccess()
}
override suspend fun httpPost(url: HttpUrl, form: Map<String, String>): Response {
val body = FormBody.Builder()
form.forEach { (k, v) ->
body.addEncoded(k, v)
}
val request = Request.Builder()
.post(body.build())
.url(url)
.addTags()
return httpClient.newCall(request.build()).await().ensureSuccess()
}
override suspend fun httpPost(url: HttpUrl, payload: String): Response {
val body = FormBody.Builder()
payload.split('&').forEach {
val pos = it.indexOf('=')
if (pos != -1) {
val k = it.substring(0, pos)
val v = it.substring(pos + 1)
body.addEncoded(k, v)
}
}
val request = Request.Builder()
.post(body.build())
.url(url)
.addTags()
return httpClient.newCall(request.build()).await().ensureSuccess()
}
override suspend fun graphQLQuery(endpoint: String, query: String): JSONObject {
val body = JSONObject()
body.put("operationName", null as Any?)
body.put("variables", JSONObject())
body.put("query", "{$query}")
val mediaType = "application/json; charset=utf-8".toMediaType()
val requestBody = body.toString().toRequestBody(mediaType)
val request = Request.Builder()
.post(requestBody)
.url(endpoint)
.addTags()
val json = httpClient.newCall(request.build()).await().ensureSuccess().parseJson()
json.optJSONArray("errors")?.let {
if (it.length() != 0) {
throw GraphQLException(it)
}
}
return json
}
private fun Request.Builder.addTags(): Request.Builder {
if (mangaSource != null) {
tag(MangaSource::class.java, mangaSource)
}
return this
}
private fun Response.ensureSuccess(): Response {
val exception: Exception? = when (code) { // Catch some error codes, not all
404 -> NotFoundException(message, request.url.toString())
in 500..599 -> HttpStatusException(message, code, request.url.toString())
else -> null
}
if (exception != null) {
runCatching {
close()
}.onFailure {
exception.addSuppressed(it)
}
throw exception
}
return this
}
}

68
src/main/kotlin/org/koitharu/kotatsu/parsers/network/WebClient.kt
Unescape Escape View File

@ -0,0 +1,68 @@
package org.koitharu.kotatsu.parsers.network
import okhttp3.HttpUrl
import okhttp3.HttpUrl.Companion.toHttpUrl
import okhttp3.Response
import org.json.JSONObject
interface WebClient {
/**
* Do a GET http request to specific url
* @param url
*/
suspend fun httpGet(url: String): Response = httpGet(url.toHttpUrl())
/**
* Do a GET http request to specific url
* @param url
*/
suspend fun httpGet(url: HttpUrl): Response
/**
* Do a HEAD http request to specific url
* @param url
*/
suspend fun httpHead(url: String): Response = httpHead(url.toHttpUrl())
/**
* Do a HEAD http request to specific url
* @param url
*/
suspend fun httpHead(url: HttpUrl): Response
/**
* Do a POST http request to specific url with `multipart/form-data` payload
* @param url
* @param form payload as key=>value map
*/
suspend fun httpPost(url: String, form: Map<String, String>): Response = httpPost(url.toHttpUrl(), form)
/**
* Do a POST http request to specific url with `multipart/form-data` payload
* @param url
* @param form payload as key=>value map
*/
suspend fun httpPost(url: HttpUrl, form: Map<String, String>): Response
/**
* Do a POST http request to specific url with `multipart/form-data` payload
* @param url
* @param payload payload as `key=value` string with `&` separator
*/
suspend fun httpPost(url: String, payload: String): Response = httpPost(url.toHttpUrl(), payload)
/**
* Do a POST http request to specific url with `multipart/form-data` payload
* @param url
* @param payload payload as `key=value` string with `&` separator
*/
suspend fun httpPost(url: HttpUrl, payload: String): Response
/**
* Do a GraphQL request to specific url
* @param endpoint an url
* @param query GraphQL request payload
*/
suspend fun graphQLQuery(endpoint: String, query: String): JSONObject
}

356
src/main/kotlin/org/koitharu/kotatsu/parsers/site/AnibelParser.kt
Unescape Escape View File

@ -8,6 +8,9 @@ import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.domain
import org.koitharu.kotatsu.parsers.util.generateUid
import org.koitharu.kotatsu.parsers.util.getDomain
import org.koitharu.kotatsu.parsers.util.json.mapJSON
import org.koitharu.kotatsu.parsers.util.json.mapJSONIndexed
import org.koitharu.kotatsu.parsers.util.json.stringIterator
@ -15,38 +18,34 @@ import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl
import java.util.*
@MangaSourceParser("ANIBEL", "Anibel", "be")
internal class AnibelParser(override val context: MangaLoaderContext) : MangaParser(MangaSource.ANIBEL) {
internal class AnibelParser(context: MangaLoaderContext) : MangaParser(context, MangaSource.ANIBEL) {
override val configKeyDomain = ConfigKey.Domain("anibel.net", null)
override val configKeyDomain = ConfigKey.Domain("anibel.net", null)
override val sortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.NEWEST,
)
override val sortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.NEWEST,
)
override fun getFaviconUrl(): String {
return "https://cdn.${getDomain()}/favicons/favicon.png"
}
override suspend fun getList(
offset: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
if (!query.isNullOrEmpty()) {
return if (offset == 0) {
search(query)
} else {
emptyList()
}
}
val filters = tags?.takeUnless { it.isEmpty() }?.joinToString(
separator = ",",
prefix = "genres: [",
postfix = "]",
) { "\"${it.key}\"" }.orEmpty()
val array = apiCall(
"""
override suspend fun getList(
offset: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
if (!query.isNullOrEmpty()) {
return if (offset == 0) {
search(query)
} else {
emptyList()
}
}
val filters = tags?.takeUnless { it.isEmpty() }?.joinToString(
separator = ",",
prefix = "genres: [",
postfix = "]",
) { "\"${it.key}\"" }.orEmpty()
val array = apiCall(
"""
getMediaList(offset: $offset, limit: 20, mediaType: manga, filters: {$filters}) {
docs {
mediaId
@ -63,37 +62,37 @@ internal class AnibelParser(override val context: MangaLoaderContext) : MangaPar
}
}
""".trimIndent(),
).getJSONObject("getMediaList").getJSONArray("docs")
return array.mapJSON { jo ->
val mediaId = jo.getString("mediaId")
val title = jo.getJSONObject("title")
val href = "${jo.getString("mediaType")}/${jo.getString("slug")}"
Manga(
id = generateUid(mediaId),
title = title.getString("be"),
coverUrl = jo.getString("poster").removePrefix("/cdn")
.toAbsoluteUrl(getDomain("cdn")) + "?width=200&height=280",
altTitle = title.getString("alt").takeUnless(String::isEmpty),
author = null,
isNsfw = false,
rating = jo.getDouble("rating").toFloat() / 10f,
url = href,
publicUrl = "https://${getDomain()}/$href",
tags = jo.getJSONArray("genres").mapToTags(),
state = when (jo.getString("status")) {
"ongoing" -> MangaState.ONGOING
"finished" -> MangaState.FINISHED
else -> null
},
source = source,
)
}
}
).getJSONObject("getMediaList").getJSONArray("docs")
return array.mapJSON { jo ->
val mediaId = jo.getString("mediaId")
val title = jo.getJSONObject("title")
val href = "${jo.getString("mediaType")}/${jo.getString("slug")}"
Manga(
id = generateUid(mediaId),
title = title.getString("be"),
coverUrl = jo.getString("poster").removePrefix("/cdn")
.toAbsoluteUrl(getDomain("cdn")) + "?width=200&height=280",
altTitle = title.getString("alt").takeUnless(String::isEmpty),
author = null,
isNsfw = false,
rating = jo.getDouble("rating").toFloat() / 10f,
url = href,
publicUrl = "https://${domain}/$href",
tags = jo.getJSONArray("genres").mapToTags(),
state = when (jo.getString("status")) {
"ongoing" -> MangaState.ONGOING
"finished" -> MangaState.FINISHED
else -> null
},
source = source,
)
}
}
override suspend fun getDetails(manga: Manga): Manga {
val (type, slug) = manga.url.split('/')
val details = apiCall(
"""
override suspend fun getDetails(manga: Manga): Manga {
val (type, slug) = manga.url.split('/')
val details = apiCall(
"""
media(mediaType: $type, slug: "$slug") {
mediaId
title {
@ -109,52 +108,52 @@ internal class AnibelParser(override val context: MangaLoaderContext) : MangaPar
genres
}
""".trimIndent(),
).getJSONObject("media")
val title = details.getJSONObject("title")
val poster = details.getString("poster").removePrefix("/cdn")
.toAbsoluteUrl(getDomain("cdn"))
val chapters = apiCall(
"""
).getJSONObject("media")
val title = details.getJSONObject("title")
val poster = details.getString("poster").removePrefix("/cdn")
.toAbsoluteUrl(getDomain("cdn"))
val chapters = apiCall(
"""
chapters(mediaId: "${details.getString("mediaId")}") {
id
chapter
released
}
""".trimIndent(),
).getJSONArray("chapters")
return manga.copy(
title = title.getString("be"),
altTitle = title.getString("alt"),
coverUrl = "$poster?width=200&height=280",
largeCoverUrl = poster,
description = details.getJSONObject("description").getString("be"),
rating = details.getDouble("rating").toFloat() / 10f,
tags = details.getJSONArray("genres").mapToTags(),
state = when (details.getString("status")) {
"ongoing" -> MangaState.ONGOING
"finished" -> MangaState.FINISHED
else -> null
},
chapters = chapters.mapJSON { jo ->
val number = jo.getInt("chapter")
MangaChapter(
id = generateUid(jo.getString("id")),
name = "Глава $number",
number = number,
url = "${manga.url}/read/$number",
scanlator = null,
uploadDate = jo.getLong("released"),
branch = null,
source = source,
)
},
)
}
).getJSONArray("chapters")
return manga.copy(
title = title.getString("be"),
altTitle = title.getString("alt"),
coverUrl = "$poster?width=200&height=280",
largeCoverUrl = poster,
description = details.getJSONObject("description").getString("be"),
rating = details.getDouble("rating").toFloat() / 10f,
tags = details.getJSONArray("genres").mapToTags(),
state = when (details.getString("status")) {
"ongoing" -> MangaState.ONGOING
"finished" -> MangaState.FINISHED
else -> null
},
chapters = chapters.mapJSON { jo ->
val number = jo.getInt("chapter")
MangaChapter(
id = generateUid(jo.getString("id")),
name = "Глава $number",
number = number,
url = "${manga.url}/read/$number",
scanlator = null,
uploadDate = jo.getLong("released"),
branch = null,
source = source,
)
},
)
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val (_, slug, _, number) = chapter.url.split('/')
val chapterJson = apiCall(
"""
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val (_, slug, _, number) = chapter.url.split('/')
val chapterJson = apiCall(
"""
chapter(slug: "$slug", chapter: $number) {
id
images {
@ -163,35 +162,35 @@ internal class AnibelParser(override val context: MangaLoaderContext) : MangaPar
}
}
""".trimIndent(),
).getJSONObject("chapter")
val pages = chapterJson.getJSONArray("images")
val chapterUrl = "https://${getDomain()}/${chapter.url}"
return pages.mapJSONIndexed { i, jo ->
MangaPage(
id = generateUid("${chapter.url}/$i"),
url = jo.getString("large"),
referer = chapterUrl,
preview = jo.getString("thumbnail"),
source = source,
)
}
}
).getJSONObject("chapter")
val pages = chapterJson.getJSONArray("images")
val chapterUrl = "https://${domain}/${chapter.url}"
return pages.mapJSONIndexed { i, jo ->
MangaPage(
id = generateUid("${chapter.url}/$i"),
url = jo.getString("large"),
referer = chapterUrl,
preview = jo.getString("thumbnail"),
source = source,
)
}
}
override suspend fun getTags(): Set<MangaTag> {
val json = apiCall(
"""
override suspend fun getTags(): Set<MangaTag> {
val json = apiCall(
"""
getFilters(mediaType: manga) {
genres
}
""".trimIndent(),
)
val array = json.getJSONObject("getFilters").getJSONArray("genres")
return array.mapToTags()
}
)
val array = json.getJSONObject("getFilters").getJSONArray("genres")
return array.mapToTags()
}
private suspend fun search(query: String): List<Manga> {
val json = apiCall(
"""
private suspend fun search(query: String): List<Manga> {
val json = apiCall(
"""
search(query: "$query", limit: 40) {
id
title {
@ -203,64 +202,65 @@ internal class AnibelParser(override val context: MangaLoaderContext) : MangaPar
type
}
""".trimIndent(),
)
val array = json.getJSONArray("search")
return array.mapJSON { jo ->
val mediaId = jo.getString("id")
val title = jo.getJSONObject("title")
val href = "${jo.getString("type").lowercase()}/${jo.getString("url")}"
Manga(
id = generateUid(mediaId),
title = title.getString("be"),
coverUrl = jo.getString("poster").removePrefix("/cdn")
.toAbsoluteUrl(getDomain("cdn")) + "?width=200&height=280",
altTitle = title.getString("en").takeUnless(String::isEmpty),
author = null,
isNsfw = false,
rating = RATING_UNKNOWN,
url = href,
publicUrl = "https://${getDomain()}/$href",
tags = emptySet(),
state = null,
source = source,
)
}
}
)
val array = json.getJSONArray("search")
return array.mapJSON { jo ->
val mediaId = jo.getString("id")
val title = jo.getJSONObject("title")
val href = "${jo.getString("type").lowercase()}/${jo.getString("url")}"
Manga(
id = generateUid(mediaId),
title = title.getString("be"),
coverUrl = jo.getString("poster").removePrefix("/cdn")
.toAbsoluteUrl(getDomain("cdn")) + "?width=200&height=280",
altTitle = title.getString("en").takeUnless(String::isEmpty),
author = null,
isNsfw = false,
rating = RATING_UNKNOWN,
url = href,
publicUrl = "https://${domain}/$href",
tags = emptySet(),
state = null,
source = source,
)
}
}
private suspend fun apiCall(request: String): JSONObject {
return context.graphQLQuery("https://api.${getDomain()}/graphql", request)
.getJSONObject("data")
}
private suspend fun apiCall(request: String): JSONObject {
return webClient.graphQLQuery("https://api.${domain}/graphql", request)
.getJSONObject("data")
}
private fun JSONArray.mapToTags(): Set<MangaTag> {
private fun JSONArray.mapToTags(): Set<MangaTag> {
fun toTitle(slug: String): String {
val builder = StringBuilder(slug)
var capitalize = true
for ((i, c) in builder.withIndex()) {
when {
c == '-' -> {
builder.setCharAt(i, ' ')
}
capitalize -> {
builder.setCharAt(i, c.uppercaseChar())
capitalize = false
}
}
}
return builder.toString()
}
fun toTitle(slug: String): String {
val builder = StringBuilder(slug)
var capitalize = true
for ((i, c) in builder.withIndex()) {
when {
c == '-' -> {
builder.setCharAt(i, ' ')
}
capitalize -> {
builder.setCharAt(i, c.uppercaseChar())
capitalize = false
}
}
}
return builder.toString()
}
val result = ArraySet<MangaTag>(length())
stringIterator().forEach {
result.add(
MangaTag(
title = toTitle(it),
key = it,
source = source,
),
)
}
return result
}
}
val result = ArraySet<MangaTag>(length())
stringIterator().forEach {
result.add(
MangaTag(
title = toTitle(it),
key = it,
source = source,
),
)
}
return result
}
}

543
src/main/kotlin/org/koitharu/kotatsu/parsers/site/BatoToParser.kt
Unescape Escape View File

@ -19,292 +19,291 @@ import javax.crypto.spec.IvParameterSpec
import javax.crypto.spec.SecretKeySpec
@MangaSourceParser("BATOTO", "Bato.To")
internal class BatoToParser(override val context: MangaLoaderContext) : PagedMangaParser(
source = MangaSource.BATOTO,
pageSize = 60,
searchPageSize = 20,
internal class BatoToParser(context: MangaLoaderContext) : PagedMangaParser(
context = context,
source = MangaSource.BATOTO,
pageSize = 60,
searchPageSize = 20,
) {
override val sortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.NEWEST,
SortOrder.UPDATED,
SortOrder.POPULARITY,
SortOrder.ALPHABETICAL,
)
override val sortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.NEWEST,
SortOrder.UPDATED,
SortOrder.POPULARITY,
SortOrder.ALPHABETICAL,
)
override val configKeyDomain = ConfigKey.Domain(
"bato.to",
arrayOf("bato.to", "mto.to", "mangatoto.com", "battwo.com", "batotwo.com", "comiko.net", "batotoo.com"),
)
override val configKeyDomain = ConfigKey.Domain(
"bato.to",
arrayOf("bato.to", "mto.to", "mangatoto.com", "battwo.com", "batotwo.com", "comiko.net", "batotoo.com"),
)
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
if (!query.isNullOrEmpty()) {
return search(page, query)
}
@Suppress("NON_EXHAUSTIVE_WHEN_STATEMENT")
val url = buildString {
append("https://")
append(getDomain())
append("/browse?sort=")
when (sortOrder) {
SortOrder.UPDATED,
-> append("update.za")
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
if (!query.isNullOrEmpty()) {
return search(page, query)
}
@Suppress("NON_EXHAUSTIVE_WHEN_STATEMENT")
val url = buildString {
append("https://")
append(domain)
append("/browse?sort=")
when (sortOrder) {
SortOrder.UPDATED,
-> append("update.za")
SortOrder.POPULARITY -> append("views_a.za")
SortOrder.NEWEST -> append("create.za")
SortOrder.ALPHABETICAL -> append("title.az")
}
if (!tags.isNullOrEmpty()) {
append("&genres=")
appendAll(tags, ",") { it.key }
}
append("&page=")
append(page)
}
return parseList(url, page)
}
SortOrder.POPULARITY -> append("views_a.za")
SortOrder.NEWEST -> append("create.za")
SortOrder.ALPHABETICAL -> append("title.az")
}
if (!tags.isNullOrEmpty()) {
append("&genres=")
appendAll(tags, ",") { it.key }
}
append("&page=")
append(page)
}
return parseList(url, page)
}
override suspend fun getDetails(manga: Manga): Manga {
val root = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml()
.requireElementById("mainer")
val details = root.selectFirstOrThrow(".detail-set")
val attrs = details.selectFirst(".attr-main")?.select(".attr-item")?.associate {
it.child(0).text().trim() to it.child(1)
}.orEmpty()
return manga.copy(
title = root.selectFirst("h3.item-title")?.text() ?: manga.title,
isNsfw = !root.selectFirst("alert")?.getElementsContainingOwnText("NSFW").isNullOrEmpty(),
largeCoverUrl = details.selectFirst("img[src]")?.absUrl("src"),
description = details.getElementById("limit-height-body-summary")
?.selectFirst(".limit-html")
?.html(),
tags = manga.tags + attrs["Genres:"]?.parseTags().orEmpty(),
state = when (attrs["Release status:"]?.text()) {
"Ongoing" -> MangaState.ONGOING
"Completed" -> MangaState.FINISHED
else -> manga.state
},
author = attrs["Authors:"]?.text()?.trim() ?: manga.author,
chapters = root.selectFirst(".episode-list")
?.selectFirst(".main")
?.children()
?.reversed()
?.mapChapters { i, div ->
div.parseChapter(i)
}.orEmpty(),
)
}
override suspend fun getDetails(manga: Manga): Manga {
val root = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml()
.requireElementById("mainer")
val details = root.selectFirstOrThrow(".detail-set")
val attrs = details.selectFirst(".attr-main")?.select(".attr-item")?.associate {
it.child(0).text().trim() to it.child(1)
}.orEmpty()
return manga.copy(
title = root.selectFirst("h3.item-title")?.text() ?: manga.title,
isNsfw = !root.selectFirst("alert")?.getElementsContainingOwnText("NSFW").isNullOrEmpty(),
largeCoverUrl = details.selectFirst("img[src]")?.absUrl("src"),
description = details.getElementById("limit-height-body-summary")
?.selectFirst(".limit-html")
?.html(),
tags = manga.tags + attrs["Genres:"]?.parseTags().orEmpty(),
state = when (attrs["Release status:"]?.text()) {
"Ongoing" -> MangaState.ONGOING
"Completed" -> MangaState.FINISHED
else -> manga.state
},
author = attrs["Authors:"]?.text()?.trim() ?: manga.author,
chapters = root.selectFirst(".episode-list")
?.selectFirst(".main")
?.children()
?.reversed()
?.mapChapters { i, div ->
div.parseChapter(i)
}.orEmpty(),
)
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(getDomain())
val scripts = context.httpGet(fullUrl).parseHtml().select("script")
for (script in scripts) {
val scriptSrc = script.html()
val p = scriptSrc.indexOf("const imgHttpLis =")
if (p == -1) continue
val start = scriptSrc.indexOf('[', p)
val end = scriptSrc.indexOf(';', start)
if (start == -1 || end == -1) {
continue
}
val images = JSONArray(scriptSrc.substring(start, end))
val batoPass = scriptSrc.substringBetweenFirst("batoPass =", ";")?.trim(' ', '"', '\n')
?: script.parseFailed("Cannot find batoPass")
val batoWord = scriptSrc.substringBetweenFirst("batoWord =", ";")?.trim(' ', '"', '\n')
?: script.parseFailed("Cannot find batoWord")
val password = context.evaluateJs(batoPass)?.removeSurrounding('"')
?: script.parseFailed("Cannot evaluate batoPass")
val args = JSONArray(decryptAES(batoWord, password))
val result = ArrayList<MangaPage>(images.length())
repeat(images.length()) { i ->
val url = images.getString(i)
result += MangaPage(
id = generateUid(url),
url = url + "?" + args.getString(i),
referer = fullUrl,
preview = null,
source = source,
)
}
return result
}
throw ParseException("Cannot find images list", fullUrl)
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(domain)
val scripts = webClient.httpGet(fullUrl).parseHtml().select("script")
for (script in scripts) {
val scriptSrc = script.html()
val p = scriptSrc.indexOf("const imgHttpLis =")
if (p == -1) continue
val start = scriptSrc.indexOf('[', p)
val end = scriptSrc.indexOf(';', start)
if (start == -1 || end == -1) {
continue
}
val images = JSONArray(scriptSrc.substring(start, end))
val batoPass = scriptSrc.substringBetweenFirst("batoPass =", ";")?.trim(' ', '"', '\n')
?: script.parseFailed("Cannot find batoPass")
val batoWord = scriptSrc.substringBetweenFirst("batoWord =", ";")?.trim(' ', '"', '\n')
?: script.parseFailed("Cannot find batoWord")
val password = context.evaluateJs(batoPass)?.removeSurrounding('"')
?: script.parseFailed("Cannot evaluate batoPass")
val args = JSONArray(decryptAES(batoWord, password))
val result = ArrayList<MangaPage>(images.length())
repeat(images.length()) { i ->
val url = images.getString(i)
result += MangaPage(
id = generateUid(url),
url = url + "?" + args.getString(i),
referer = fullUrl,
preview = null,
source = source,
)
}
return result
}
throw ParseException("Cannot find images list", fullUrl)
}
override suspend fun getTags(): Set<MangaTag> {
val scripts = context.httpGet(
"https://${getDomain()}/browse",
).parseHtml().selectOrThrow("script")
for (script in scripts) {
val genres = script.html().substringBetweenFirst("const _genres =", ";") ?: continue
val jo = JSONObject(genres)
val result = ArraySet<MangaTag>(jo.length())
jo.keys().forEach { key ->
val item = jo.getJSONObject(key)
result += MangaTag(
title = item.getString("text").toTitleCase(),
key = item.getString("file"),
source = source,
)
}
return result
}
throw ParseException("Cannot find gernes list", scripts[0].baseUri())
}
override suspend fun getTags(): Set<MangaTag> {
val scripts = webClient.httpGet(
"https://${domain}/browse",
).parseHtml().selectOrThrow("script")
for (script in scripts) {
val genres = script.html().substringBetweenFirst("const _genres =", ";") ?: continue
val jo = JSONObject(genres)
val result = ArraySet<MangaTag>(jo.length())
jo.keys().forEach { key ->
val item = jo.getJSONObject(key)
result += MangaTag(
title = item.getString("text").toTitleCase(),
key = item.getString("file"),
source = source,
)
}
return result
}
throw ParseException("Cannot find gernes list", scripts[0].baseUri())
}
override fun getFaviconUrl(): String = "https://styles.amarkcdn.com/img/batoto/favicon.ico?v0"
private suspend fun search(page: Int, query: String): List<Manga> {
val url = buildString {
append("https://")
append(domain)
append("/search?word=")
append(query.replace(' ', '+'))
append("&page=")
append(page)
}
return parseList(url, page)
}
private suspend fun search(page: Int, query: String): List<Manga> {
val url = buildString {
append("https://")
append(getDomain())
append("/search?word=")
append(query.replace(' ', '+'))
append("&page=")
append(page)
}
return parseList(url, page)
}
private fun getActivePage(body: Element): Int = body.select("nav ul.pagination > li.page-item.active")
.lastOrNull()
?.text()
?.toIntOrNull() ?: body.parseFailed("Cannot determine current page")
private fun getActivePage(body: Element): Int = body.select("nav ul.pagination > li.page-item.active")
.lastOrNull()
?.text()
?.toIntOrNull() ?: body.parseFailed("Cannot determine current page")
private suspend fun parseList(url: String, page: Int): List<Manga> {
val body = webClient.httpGet(url).parseHtml().body()
if (body.selectFirst(".browse-no-matches") != null) {
return emptyList()
}
val activePage = getActivePage(body)
if (activePage != page) {
return emptyList()
}
val root = body.requireElementById("series-list")
return root.children().map { div ->
val a = div.selectFirstOrThrow("a")
val href = a.attrAsRelativeUrl("href")
val title = div.selectFirstOrThrow(".item-title").text()
Manga(
id = generateUid(href),
title = title,
altTitle = div.selectFirst(".item-alias")?.text()?.takeUnless { it == title },
url = href,
publicUrl = a.absUrl("href"),
rating = RATING_UNKNOWN,
isNsfw = false,
coverUrl = div.selectFirst("img[src]")?.absUrl("src").orEmpty(),
largeCoverUrl = null,
description = null,
tags = div.selectFirst(".item-genre")?.parseTags().orEmpty(),
state = null,
author = null,
source = source,
)
}
}
private suspend fun parseList(url: String, page: Int): List<Manga> {
val body = context.httpGet(url).parseHtml().body()
if (body.selectFirst(".browse-no-matches") != null) {
return emptyList()
}
val activePage = getActivePage(body)
if (activePage != page) {
return emptyList()
}
val root = body.requireElementById("series-list")
return root.children().map { div ->
val a = div.selectFirstOrThrow("a")
val href = a.attrAsRelativeUrl("href")
val title = div.selectFirstOrThrow(".item-title").text()
Manga(
id = generateUid(href),
title = title,
altTitle = div.selectFirst(".item-alias")?.text()?.takeUnless { it == title },
url = href,
publicUrl = a.absUrl("href"),
rating = RATING_UNKNOWN,
isNsfw = false,
coverUrl = div.selectFirst("img[src]")?.absUrl("src").orEmpty(),
largeCoverUrl = null,
description = null,
tags = div.selectFirst(".item-genre")?.parseTags().orEmpty(),
state = null,
author = null,
source = source,
)
}
}
private fun Element.parseTags() = children().mapToSet { span ->
val text = span.ownText()
MangaTag(
title = text.toTitleCase(),
key = text.lowercase(Locale.ENGLISH).replace(' ', '_'),
source = source,
)
}
private fun Element.parseTags() = children().mapToSet { span ->
val text = span.ownText()
MangaTag(
title = text.toTitleCase(),
key = text.lowercase(Locale.ENGLISH).replace(' ', '_'),
source = source,
)
}
private fun Element.parseChapter(index: Int): MangaChapter? {
val a = selectFirst("a.chapt") ?: return null
val extra = selectFirst(".extra")
val href = a.attrAsRelativeUrl("href")
return MangaChapter(
id = generateUid(href),
name = a.text(),
number = index + 1,
url = href,
scanlator = extra?.getElementsByAttributeValueContaining("href", "/group/")?.text(),
uploadDate = runCatching {
parseChapterDate(extra?.select("i")?.lastOrNull()?.ownText())
}.getOrDefault(0),
branch = null,
source = source,
)
}
private fun Element.parseChapter(index: Int): MangaChapter? {
val a = selectFirst("a.chapt") ?: return null
val extra = selectFirst(".extra")
val href = a.attrAsRelativeUrl("href")
return MangaChapter(
id = generateUid(href),
name = a.text(),
number = index + 1,
url = href,
scanlator = extra?.getElementsByAttributeValueContaining("href", "/group/")?.text(),
uploadDate = runCatching {
parseChapterDate(extra?.select("i")?.lastOrNull()?.ownText())
}.getOrDefault(0),
branch = null,
source = source,
)
}
private fun parseChapterDate(date: String?): Long {
if (date.isNullOrEmpty()) {
return 0
}
val value = date.substringBefore(' ').toInt()
val field = when {
"sec" in date -> Calendar.SECOND
"min" in date -> Calendar.MINUTE
"hour" in date -> Calendar.HOUR
"day" in date -> Calendar.DAY_OF_MONTH
"week" in date -> Calendar.WEEK_OF_YEAR
"month" in date -> Calendar.MONTH
"year" in date -> Calendar.YEAR
else -> return 0
}
val calendar = Calendar.getInstance()
calendar.add(field, -value)
return calendar.timeInMillis
}
private fun parseChapterDate(date: String?): Long {
if (date.isNullOrEmpty()) {
return 0
}
val value = date.substringBefore(' ').toInt()
val field = when {
"sec" in date -> Calendar.SECOND
"min" in date -> Calendar.MINUTE
"hour" in date -> Calendar.HOUR
"day" in date -> Calendar.DAY_OF_MONTH
"week" in date -> Calendar.WEEK_OF_YEAR
"month" in date -> Calendar.MONTH
"year" in date -> Calendar.YEAR
else -> return 0
}
val calendar = Calendar.getInstance()
calendar.add(field, -value)
return calendar.timeInMillis
}
private fun decryptAES(encrypted: String, password: String): String {
val cipherData = context.decodeBase64(encrypted)
val saltData = cipherData.copyOfRange(8, 16)
val (key, iv) = generateKeyAndIV(
keyLength = 32,
ivLength = 16,
iterations = 1,
salt = saltData,
password = password.toByteArray(StandardCharsets.UTF_8),
md = MessageDigest.getInstance("MD5"),
)
val encryptedData = cipherData.copyOfRange(16, cipherData.size)
val cipher = Cipher.getInstance("AES/CBC/PKCS5Padding")
cipher.init(Cipher.DECRYPT_MODE, key, iv)
return cipher.doFinal(encryptedData).toString(Charsets.UTF_8)
}
private fun decryptAES(encrypted: String, password: String): String {
val cipherData = context.decodeBase64(encrypted)
val saltData = cipherData.copyOfRange(8, 16)
val (key, iv) = generateKeyAndIV(
keyLength = 32,
ivLength = 16,
iterations = 1,
salt = saltData,
password = password.toByteArray(StandardCharsets.UTF_8),
md = MessageDigest.getInstance("MD5"),
)
val encryptedData = cipherData.copyOfRange(16, cipherData.size)
val cipher = Cipher.getInstance("AES/CBC/PKCS5Padding")
cipher.init(Cipher.DECRYPT_MODE, key, iv)
return cipher.doFinal(encryptedData).toString(Charsets.UTF_8)
}
@Suppress("SameParameterValue")
private fun generateKeyAndIV(
keyLength: Int,
ivLength: Int,
iterations: Int,
salt: ByteArray,
password: ByteArray,
md: MessageDigest,
): Pair<SecretKeySpec, IvParameterSpec> {
val digestLength = md.digestLength
val requiredLength = (keyLength + ivLength + digestLength - 1) / digestLength * digestLength
val generatedData = ByteArray(requiredLength)
var generatedLength = 0
md.reset()
while (generatedLength < keyLength + ivLength) {
if (generatedLength > 0) {
md.update(generatedData, generatedLength - digestLength, digestLength)
}
md.update(password)
md.update(salt, 0, 8)
md.digest(generatedData, generatedLength, digestLength)
repeat(iterations - 1) {
md.update(generatedData, generatedLength, digestLength)
md.digest(generatedData, generatedLength, digestLength)
}
generatedLength += digestLength
}
@Suppress("SameParameterValue")
private fun generateKeyAndIV(
keyLength: Int,
ivLength: Int,
iterations: Int,
salt: ByteArray,
password: ByteArray,
md: MessageDigest,
): Pair<SecretKeySpec, IvParameterSpec> {
val digestLength = md.digestLength
val requiredLength = (keyLength + ivLength + digestLength - 1) / digestLength * digestLength
val generatedData = ByteArray(requiredLength)
var generatedLength = 0
md.reset()
while (generatedLength < keyLength + ivLength) {
if (generatedLength > 0) {
md.update(generatedData, generatedLength - digestLength, digestLength)
}
md.update(password)
md.update(salt, 0, 8)
md.digest(generatedData, generatedLength, digestLength)
repeat(iterations - 1) {
md.update(generatedData, generatedLength, digestLength)
md.digest(generatedData, generatedLength, digestLength)
}
generatedLength += digestLength
}
return SecretKeySpec(generatedData.copyOfRange(0, keyLength), "AES") to IvParameterSpec(
if (ivLength > 0) {
generatedData.copyOfRange(keyLength, keyLength + ivLength)
} else byteArrayOf(),
)
}
}
return SecretKeySpec(generatedData.copyOfRange(0, keyLength), "AES") to IvParameterSpec(
if (ivLength > 0) {
generatedData.copyOfRange(keyLength, keyLength + ivLength)
} else byteArrayOf(),
)
}
}

469
src/main/kotlin/org/koitharu/kotatsu/parsers/site/BlogTruyenParser.kt
Unescape Escape View File

@ -14,240 +14,241 @@ import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import java.text.SimpleDateFormat
import java.util.*
import kotlin.collections.HashSet
@MangaSourceParser("BLOGTRUYEN", "BlogTruyen", "vi")
class BlogTruyenParser(override val context: MangaLoaderContext) :
PagedMangaParser(MangaSource.BLOGTRUYEN, pageSize = 20) {
override val configKeyDomain: ConfigKey.Domain
get() = ConfigKey.Domain("blogtruyen.vn", null)
override val sortOrders: Set<SortOrder>
get() = EnumSet.of(SortOrder.UPDATED)
private val mutex = Mutex()
private val dateFormat = SimpleDateFormat("dd/MM/yyyy HH:mm", Locale.US)
private var cacheTags: ArrayMap<String, MangaTag>? = null
override suspend fun getDetails(manga: Manga): Manga {
val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml()
val descriptionElement = doc.selectFirstOrThrow("div.description")
val statusText = descriptionElement
.selectFirst("p:contains(Trạng thái) > span.color-red")
?.text()
val state = when (statusText) {
"Đang tiến hành" -> MangaState.ONGOING
"Đã hoàn thành" -> MangaState.FINISHED
else -> null
}
val rating = doc.selectFirst("span.total-vote")?.attr("ng-init")?.let { text ->
val like = text.substringAfter("TotalLike=")
.substringBefore(';')
.toIntOrNull() ?: return@let RATING_UNKNOWN
val dislike = text.substringAfter("TotalDisLike=")
.toIntOrNull() ?: return@let RATING_UNKNOWN
when {
like == 0 && dislike == 0 -> RATING_UNKNOWN
else -> like.toFloat() / (like + dislike)
}
}
val tagMap = getOrCreateTagMap()
val tags = descriptionElement.select("p > span.category").mapNotNullToSet {
val tagName = it.selectFirst("a")?.text()?.trim() ?: return@mapNotNullToSet null
tagMap[tagName]
}
return manga.copy(
tags = tags,
author = descriptionElement.selectFirst("p:contains(Tác giả) > a")?.text(),
description = doc.selectFirst(".detail .content")?.html(),
chapters = parseChapterList(doc),
largeCoverUrl = doc.selectLast("div.thumbnail > img")?.imageUrl(),
state = state,
rating = rating ?: RATING_UNKNOWN,
isNsfw = doc.getElementById("warningCategory") != null
)
}
private fun parseChapterList(doc: Document): List<MangaChapter> {
val chapterList = doc.select("#list-chapters > p")
return chapterList.asReversed().mapChapters { index, element ->
val titleElement = element.selectFirst("span.title > a") ?: return@mapChapters null
val name = titleElement.text()
val relativeUrl = titleElement.attrAsRelativeUrl("href")
val id = relativeUrl.substringAfter('/').substringBefore('/')
val uploadDate = dateFormat.tryParse(element.select("span.publishedDate").text())
MangaChapter(
id = generateUid(id),
name = name,
number = index + 1,
url = relativeUrl,
scanlator = null,
uploadDate = uploadDate,
branch = null,
source = source
)
}
}
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
return when {
!query.isNullOrEmpty() -> {
val searchUrl = "https://${getDomain()}/timkiem/nangcao/1/0/-1/-1?txt=${query.urlEncoded()}&p=$page"
val searchContent = context.httpGet(searchUrl).parseHtml()
.selectFirst("section.list-manga-bycate > div.list")
parseMangaList(searchContent)
}
!tags.isNullOrEmpty() -> {
val tag = tags.oneOrThrowIfMany()!!
val categoryAjax = "https://${getDomain()}/ajax/Category/AjaxLoadMangaByCategory?id=${tag.key}&orderBy=5&p=$page"
val listContent = context.httpGet(categoryAjax).parseHtml().selectFirst("div.list")
parseMangaList(listContent)
}
else -> getNormalList(page)
}
}
private suspend fun getNormalList(page: Int): List<Manga> {
val pageLink = "https://${getDomain()}/page-$page"
val doc = context.httpGet(pageLink).parseHtml()
val listElements = doc.selectFirstOrThrow("section.list-mainpage.listview")
.select("div.bg-white.storyitem")
return listElements.mapNotNull {
val linkTag = it.selectFirst("div.fl-l > a") ?: return@mapNotNull null
val relativeUrl = linkTag.attrAsRelativeUrl("href")
val tagMap = getOrCreateTagMap()
val tags = it.select("footer > div.category > a").mapNotNullToSet { a ->
tagMap[a.text()]
}
Manga(
id = generateUid(relativeUrl),
title = linkTag.attr("title"),
altTitle = null,
description = it.selectFirst("p.al-j.break.line-height-15")?.text(),
url = relativeUrl,
publicUrl = relativeUrl.toAbsoluteUrl(getDomain()),
coverUrl = linkTag.selectLast("img")?.imageUrl().orEmpty(),
source = source,
tags = tags,
isNsfw = false,
rating = RATING_UNKNOWN,
author = null,
state = null,
)
}
}
private fun parseMangaList(listElement: Element?): List<Manga> {
listElement ?: return emptyList()
return listElement.select("span.tiptip[data-tiptip]").mapNotNull {
val mangaInfo = listElement.getElementById(it.attr("data-tiptip")) ?: return@mapNotNull null
val a = it.selectFirst("a") ?: return@mapNotNull null
val relativeUrl = a.attrAsRelativeUrl("href")
Manga(
id = generateUid(relativeUrl),
title = a.text(),
altTitle = null,
description = mangaInfo.select("div.al-j.fs-12").text(),
url = relativeUrl,
publicUrl = relativeUrl.toAbsoluteUrl(getDomain()),
coverUrl = mangaInfo.selectFirst("div > img.img")?.imageUrl().orEmpty(),
isNsfw = false,
rating = RATING_UNKNOWN,
tags = emptySet(),
author = null,
state = null,
source = source,
)
}
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
fun generateImageId(index: Int) = generateUid("${chapter.url}/$index")
val doc = context.httpGet(chapter.url.toAbsoluteUrl(getDomain())).parseHtml()
val pages = ArrayList<MangaPage>()
val referer = chapter.url.toAbsoluteUrl(getDomain())
doc.select("#content > img").forEach { img ->
pages.add(
MangaPage(
id = generateImageId(pages.size),
url = img.imageUrl(),
referer = referer,
preview = null,
source = source,
)
)
}
// Some chapters use js script to render images
val script = doc.selectLast("#content > script")
if (script != null && script.data().contains("listImageCaption")) {
val imagesStr = script.data().substringBefore(';').substringAfterLast('=').trim()
val imageArr = JSONArray(imagesStr)
for (i in 0 until imageArr.length()) {
val imageUrl = imageArr.getJSONObject(i).getString("url")
pages.add(
MangaPage(
id = generateImageId(pages.size),
url = imageUrl,
referer = referer,
preview = null,
source = source
)
)
}
}
return pages
}
override suspend fun getTags(): Set<MangaTag> {
val map = getOrCreateTagMap()
val tags = HashSet<MangaTag>(map.size)
for (entry in map) {
tags.add(entry.value)
}
return tags
}
private suspend fun getOrCreateTagMap(): ArrayMap<String, MangaTag> = mutex.withLock {
cacheTags?.let { return@withLock it }
val doc = context.httpGet("/timkiem/nangcao".toAbsoluteUrl(getDomain())).parseHtml()
val tagItems = doc.select("li[data-id]")
val tagMap = ArrayMap<String, MangaTag>(tagItems.size)
for (tag in tagItems) {
val title = tag.text().trim()
tagMap[tag.text().trim()] = MangaTag(
title = title,
key = tag.attr("data-id"),
source = source
)
}
cacheTags = tagMap
tagMap
}
private fun Element.imageUrl(): String {
return attrAsAbsoluteUrlOrNull("src")
?: attrAsAbsoluteUrlOrNull("data-cfsrc")
?: ""
}
class BlogTruyenParser(context: MangaLoaderContext) :
PagedMangaParser(context, MangaSource.BLOGTRUYEN, pageSize = 20) {
override val configKeyDomain: ConfigKey.Domain
get() = ConfigKey.Domain("blogtruyen.vn", null)
override val sortOrders: Set<SortOrder>
get() = EnumSet.of(SortOrder.UPDATED)
private val mutex = Mutex()
private val dateFormat = SimpleDateFormat("dd/MM/yyyy HH:mm", Locale.US)
private var cacheTags: ArrayMap<String, MangaTag>? = null
override suspend fun getDetails(manga: Manga): Manga {
val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml()
val descriptionElement = doc.selectFirstOrThrow("div.description")
val statusText = descriptionElement
.selectFirst("p:contains(Trạng thái) > span.color-red")
?.text()
val state = when (statusText) {
"Đang tiến hành" -> MangaState.ONGOING
"Đã hoàn thành" -> MangaState.FINISHED
else -> null
}
val rating = doc.selectFirst("span.total-vote")?.attr("ng-init")?.let { text ->
val like = text.substringAfter("TotalLike=")
.substringBefore(';')
.toIntOrNull() ?: return@let RATING_UNKNOWN
val dislike = text.substringAfter("TotalDisLike=")
.toIntOrNull() ?: return@let RATING_UNKNOWN
when {
like == 0 && dislike == 0 -> RATING_UNKNOWN
else -> like.toFloat() / (like + dislike)
}
}
val tagMap = getOrCreateTagMap()
val tags = descriptionElement.select("p > span.category").mapNotNullToSet {
val tagName = it.selectFirst("a")?.text()?.trim() ?: return@mapNotNullToSet null
tagMap[tagName]
}
return manga.copy(
tags = tags,
author = descriptionElement.selectFirst("p:contains(Tác giả) > a")?.text(),
description = doc.selectFirst(".detail .content")?.html(),
chapters = parseChapterList(doc),
largeCoverUrl = doc.selectLast("div.thumbnail > img")?.imageUrl(),
state = state,
rating = rating ?: RATING_UNKNOWN,
isNsfw = doc.getElementById("warningCategory") != null,
)
}
private fun parseChapterList(doc: Document): List<MangaChapter> {
val chapterList = doc.select("#list-chapters > p")
return chapterList.asReversed().mapChapters { index, element ->
val titleElement = element.selectFirst("span.title > a") ?: return@mapChapters null
val name = titleElement.text()
val relativeUrl = titleElement.attrAsRelativeUrl("href")
val id = relativeUrl.substringAfter('/').substringBefore('/')
val uploadDate = dateFormat.tryParse(element.select("span.publishedDate").text())
MangaChapter(
id = generateUid(id),
name = name,
number = index + 1,
url = relativeUrl,
scanlator = null,
uploadDate = uploadDate,
branch = null,
source = source,
)
}
}
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
return when {
!query.isNullOrEmpty() -> {
val searchUrl = "https://${domain}/timkiem/nangcao/1/0/-1/-1?txt=${query.urlEncoded()}&p=$page"
val searchContent = webClient.httpGet(searchUrl).parseHtml()
.selectFirst("section.list-manga-bycate > div.list")
parseMangaList(searchContent)
}
!tags.isNullOrEmpty() -> {
val tag = tags.oneOrThrowIfMany()!!
val categoryAjax =
"https://${domain}/ajax/Category/AjaxLoadMangaByCategory?id=${tag.key}&orderBy=5&p=$page"
val listContent = webClient.httpGet(categoryAjax).parseHtml().selectFirst("div.list")
parseMangaList(listContent)
}
else -> getNormalList(page)
}
}
private suspend fun getNormalList(page: Int): List<Manga> {
val pageLink = "https://${domain}/page-$page"
val doc = webClient.httpGet(pageLink).parseHtml()
val listElements = doc.selectFirstOrThrow("section.list-mainpage.listview")
.select("div.bg-white.storyitem")
return listElements.mapNotNull {
val linkTag = it.selectFirst("div.fl-l > a") ?: return@mapNotNull null
val relativeUrl = linkTag.attrAsRelativeUrl("href")
val tagMap = getOrCreateTagMap()
val tags = it.select("footer > div.category > a").mapNotNullToSet { a ->
tagMap[a.text()]
}
Manga(
id = generateUid(relativeUrl),
title = linkTag.attr("title"),
altTitle = null,
description = it.selectFirst("p.al-j.break.line-height-15")?.text(),
url = relativeUrl,
publicUrl = relativeUrl.toAbsoluteUrl(domain),
coverUrl = linkTag.selectLast("img")?.imageUrl().orEmpty(),
source = source,
tags = tags,
isNsfw = false,
rating = RATING_UNKNOWN,
author = null,
state = null,
)
}
}
private fun parseMangaList(listElement: Element?): List<Manga> {
listElement ?: return emptyList()
return listElement.select("span.tiptip[data-tiptip]").mapNotNull {
val mangaInfo = listElement.getElementById(it.attr("data-tiptip")) ?: return@mapNotNull null
val a = it.selectFirst("a") ?: return@mapNotNull null
val relativeUrl = a.attrAsRelativeUrl("href")
Manga(
id = generateUid(relativeUrl),
title = a.text(),
altTitle = null,
description = mangaInfo.select("div.al-j.fs-12").text(),
url = relativeUrl,
publicUrl = relativeUrl.toAbsoluteUrl(domain),
coverUrl = mangaInfo.selectFirst("div > img.img")?.imageUrl().orEmpty(),
isNsfw = false,
rating = RATING_UNKNOWN,
tags = emptySet(),
author = null,
state = null,
source = source,
)
}
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
fun generateImageId(index: Int) = generateUid("${chapter.url}/$index")
val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml()
val pages = ArrayList<MangaPage>()
val referer = chapter.url.toAbsoluteUrl(domain)
doc.select("#content > img").forEach { img ->
pages.add(
MangaPage(
id = generateImageId(pages.size),
url = img.imageUrl(),
referer = referer,
preview = null,
source = source,
),
)
}
// Some chapters use js script to render images
val script = doc.selectLast("#content > script")
if (script != null && script.data().contains("listImageCaption")) {
val imagesStr = script.data().substringBefore(';').substringAfterLast('=').trim()
val imageArr = JSONArray(imagesStr)
for (i in 0 until imageArr.length()) {
val imageUrl = imageArr.getJSONObject(i).getString("url")
pages.add(
MangaPage(
id = generateImageId(pages.size),
url = imageUrl,
referer = referer,
preview = null,
source = source,
),
)
}
}
return pages
}
override suspend fun getTags(): Set<MangaTag> {
val map = getOrCreateTagMap()
val tags = HashSet<MangaTag>(map.size)
for (entry in map) {
tags.add(entry.value)
}
return tags
}
private suspend fun getOrCreateTagMap(): ArrayMap<String, MangaTag> = mutex.withLock {
cacheTags?.let { return@withLock it }
val doc = webClient.httpGet("/timkiem/nangcao".toAbsoluteUrl(domain)).parseHtml()
val tagItems = doc.select("li[data-id]")
val tagMap = ArrayMap<String, MangaTag>(tagItems.size)
for (tag in tagItems) {
val title = tag.text().trim()
tagMap[tag.text().trim()] = MangaTag(
title = title,
key = tag.attr("data-id"),
source = source,
)
}
cacheTags = tagMap
tagMap
}
private fun Element.imageUrl(): String {
return attrAsAbsoluteUrlOrNull("src")
?: attrAsAbsoluteUrlOrNull("data-cfsrc")
?: ""
}
}

177
src/main/kotlin/org/koitharu/kotatsu/parsers/site/CloneMangaParser.kt
Unescape Escape View File

@ -1,104 +1,103 @@
package org.koitharu.kotatsu.parsers.site
import org.koitharu.kotatsu.parsers.*
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import java.util.*
@MangaSourceParser("CLONEMANGA", "CloneManga", "en")
internal class CloneMangaParser(override val context: MangaLoaderContext) : PagedMangaParser(
MangaSource.CLONEMANGA,
pageSize = 1,
internal class CloneMangaParser(context: MangaLoaderContext) : PagedMangaParser(
context,
MangaSource.CLONEMANGA,
pageSize = 1,
) {
override val sortOrders: Set<SortOrder> = Collections.singleton(
SortOrder.POPULARITY,
)
override val sortOrders: Set<SortOrder> = Collections.singleton(
SortOrder.POPULARITY,
)
override val configKeyDomain = ConfigKey.Domain("manga.clone-army.org", null)
override val configKeyDomain = ConfigKey.Domain("manga.clone-army.org", null)
override fun getFaviconUrl(): String {
return "https://pbs.twimg.com/profile_images/458758466346029056/Ys93EANp_400x400.png"
}
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
if (query != null || page > 1) {
return emptyList()
}
val link = "https://${domain}/viewer_landing.php"
val doc = webClient.httpGet(link).parseHtml()
val mangas = doc.getElementsByClass("comicPreviewContainer")
return mangas.mapNotNull { item ->
val attr = item.getElementsByClass("comicPreview").attr("style")
val href = item.selectFirst("a")?.attrAsAbsoluteUrl("href") ?: return@mapNotNull null
val cover = attr.substring(attr.indexOf("site/themes"), attr.indexOf(")"))
Manga(
id = generateUid(href),
title = item.selectFirst("h3")?.text() ?: return@mapNotNull null,
coverUrl = "https://${domain}/$cover",
altTitle = null,
author = "Dan Kim",
rating = RATING_UNKNOWN,
url = href,
isNsfw = false,
tags = emptySet(),
state = null,
publicUrl = href.toAbsoluteUrl(domain),
source = source,
)
}
}
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
if (query != null || page > 1) {
return emptyList()
}
val link = "https://${getDomain()}/viewer_landing.php"
val doc = context.httpGet(link).parseHtml()
val mangas = doc.getElementsByClass("comicPreviewContainer")
return mangas.mapNotNull { item ->
val attr = item.getElementsByClass("comicPreview").attr("style")
val href = item.selectFirst("a")?.attrAsAbsoluteUrl("href") ?: return@mapNotNull null
val cover = attr.substring(attr.indexOf("site/themes"), attr.indexOf(")"))
Manga(
id = generateUid(href),
title = item.selectFirst("h3")?.text() ?: return@mapNotNull null,
coverUrl = "https://${getDomain()}/$cover",
altTitle = null,
author = "Dan Kim",
rating = RATING_UNKNOWN,
url = href,
isNsfw = false,
tags = emptySet(),
state = null,
publicUrl = href.toAbsoluteUrl(getDomain()),
source = source,
)
}
}
override suspend fun getDetails(manga: Manga): Manga {
val doc = webClient.httpGet(manga.publicUrl).parseHtml()
val series = doc.location()
val numChapters = Regex(
pattern = "&page=(.*)&lang=",
).findAll(
input = doc.getElementsByTag("script")[3].toString(),
)
.elementAt(3).destructured.component1()
.toInt()
val chapters = ArrayList<MangaChapter>()
for (i in 0..numChapters) {
val chapter = MangaChapter(
id = generateUid("$series&page=$i"),
name = "Chapter ${i + 1}",
number = i + 1,
url = "$series&page=$i",
scanlator = null,
branch = null,
uploadDate = 0L,
source = MangaSource.DUMMY,
)
chapters.add(chapter)
}
return manga.copy(chapters = chapters)
}
override suspend fun getDetails(manga: Manga): Manga {
val doc = context.httpGet(manga.publicUrl).parseHtml()
val series = doc.location()
val numChapters = Regex(
pattern = "&page=(.*)&lang=",
).findAll(
input = doc.getElementsByTag("script")[3].toString(),
)
.elementAt(3).destructured.component1()
.toInt()
val chapters = ArrayList<MangaChapter>()
for (i in 0..numChapters) {
val chapter = MangaChapter(
id = generateUid("$series&page=$i"),
name = "Chapter ${i + 1}",
number = i + 1,
url = "$series&page=$i",
scanlator = null,
branch = null,
uploadDate = 0L,
source = MangaSource.DUMMY,
)
chapters.add(chapter)
}
return manga.copy(chapters = chapters)
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml()
val imgUrl = doc.getElementsByClass("subsectionContainer")[0]
.selectFirst("img")
?.attrAsAbsoluteUrlOrNull("src") ?: doc.parseFailed("Something broken")
return listOf(
MangaPage(
id = generateUid(imgUrl),
url = imgUrl,
referer = imgUrl,
preview = null,
source = MangaSource.DUMMY,
),
)
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val doc = context.httpGet(chapter.url.toAbsoluteUrl(getDomain())).parseHtml()
val imgUrl = doc.getElementsByClass("subsectionContainer")[0]
.selectFirst("img")
?.attrAsAbsoluteUrlOrNull("src") ?: doc.parseFailed("Something broken")
return listOf(
MangaPage(
id = generateUid(imgUrl),
url = imgUrl,
referer = imgUrl,
preview = null,
source = MangaSource.DUMMY,
),
)
}
override suspend fun getTags(): Set<MangaTag> {
return emptySet()
}
}
override suspend fun getTags(): Set<MangaTag> {
return emptySet()
}
}

362
src/main/kotlin/org/koitharu/kotatsu/parsers/site/ComickFunParser.kt
Unescape Escape View File

@ -22,195 +22,195 @@ private const val PAGE_SIZE = 20
private const val CHAPTERS_LIMIT = 99999
@MangaSourceParser("COMICK_FUN", "ComicK")
internal class ComickFunParser(override val context: MangaLoaderContext) : MangaParser(MangaSource.COMICK_FUN) {
internal class ComickFunParser(context: MangaLoaderContext) : MangaParser(context, MangaSource.COMICK_FUN) {
override val configKeyDomain = ConfigKey.Domain("comick.app", null)
override val configKeyDomain = ConfigKey.Domain("comick.app", null)
override val sortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.POPULARITY,
SortOrder.UPDATED,
SortOrder.RATING,
)
override val sortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.POPULARITY,
SortOrder.UPDATED,
SortOrder.RATING,
)
@Volatile
private var cachedTags: SparseArrayCompat<MangaTag>? = null
@Volatile
private var cachedTags: SparseArrayCompat<MangaTag>? = null
override suspend fun getList(
offset: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val domain = getDomain()
val url = buildString {
append("https://api.")
append(domain)
append("/search?tachiyomi=true")
if (!query.isNullOrEmpty()) {
if (offset > 0) {
return emptyList()
}
append("&q=")
append(query.urlEncoded())
} else {
append("&limit=")
append(PAGE_SIZE)
append("&page=")
append((offset / PAGE_SIZE) + 1)
if (!tags.isNullOrEmpty()) {
append("&genres=")
appendAll(tags, "&genres=", MangaTag::key)
}
append("&sort=") // view, uploaded, rating, follow, user_follow_count
append(
when (sortOrder) {
SortOrder.POPULARITY -> "view"
SortOrder.RATING -> "rating"
else -> "uploaded"
},
)
}
}
val ja = context.httpGet(url).parseJsonArray()
val tagsMap = cachedTags ?: loadTags()
return ja.mapJSON { jo ->
val slug = jo.getString("slug")
Manga(
id = generateUid(slug),
title = jo.getString("title"),
altTitle = null,
url = slug,
publicUrl = "https://$domain/comic/$slug",
rating = jo.getDoubleOrDefault("rating", -10.0).toFloat() / 10f,
isNsfw = false,
coverUrl = jo.getString("cover_url"),
largeCoverUrl = null,
description = jo.getStringOrNull("desc"),
tags = jo.selectGenres("genres", tagsMap),
state = runCatching {
if (jo.getBoolean("translation_completed")) {
MangaState.FINISHED
} else {
MangaState.ONGOING
}
}.getOrNull(),
author = null,
source = source,
)
}
}
override suspend fun getList(
offset: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val domain = domain
val url = buildString {
append("https://api.")
append(domain)
append("/search?tachiyomi=true")
if (!query.isNullOrEmpty()) {
if (offset > 0) {
return emptyList()
}
append("&q=")
append(query.urlEncoded())
} else {
append("&limit=")
append(PAGE_SIZE)
append("&page=")
append((offset / PAGE_SIZE) + 1)
if (!tags.isNullOrEmpty()) {
append("&genres=")
appendAll(tags, "&genres=", MangaTag::key)
}
append("&sort=") // view, uploaded, rating, follow, user_follow_count
append(
when (sortOrder) {
SortOrder.POPULARITY -> "view"
SortOrder.RATING -> "rating"
else -> "uploaded"
},
)
}
}
val ja = webClient.httpGet(url).parseJsonArray()
val tagsMap = cachedTags ?: loadTags()
return ja.mapJSON { jo ->
val slug = jo.getString("slug")
Manga(
id = generateUid(slug),
title = jo.getString("title"),
altTitle = null,
url = slug,
publicUrl = "https://$domain/comic/$slug",
rating = jo.getDoubleOrDefault("rating", -10.0).toFloat() / 10f,
isNsfw = false,
coverUrl = jo.getString("cover_url"),
largeCoverUrl = null,
description = jo.getStringOrNull("desc"),
tags = jo.selectGenres("genres", tagsMap),
state = runCatching {
if (jo.getBoolean("translation_completed")) {
MangaState.FINISHED
} else {
MangaState.ONGOING
}
}.getOrNull(),
author = null,
source = source,
)
}
}
override suspend fun getDetails(manga: Manga): Manga {
val domain = getDomain()
val url = "https://api.$domain/comic/${manga.url}?tachiyomi=true"
val jo = context.httpGet(url).parseJson()
val comic = jo.getJSONObject("comic")
return manga.copy(
title = comic.getString("title"),
altTitle = null, // TODO
isNsfw = jo.getBoolean("matureContent") || comic.getBoolean("hentai"),
description = comic.getStringOrNull("parsed") ?: comic.getString("desc"),
tags = manga.tags + jo.getJSONArray("genres").mapJSONToSet {
MangaTag(
title = it.getString("name"),
key = it.getString("slug"),
source = source,
)
},
author = jo.getJSONArray("artists").optJSONObject(0)?.getString("name"),
chapters = getChapters(comic.getLong("id")),
)
}
override suspend fun getDetails(manga: Manga): Manga {
val domain = domain
val url = "https://api.$domain/comic/${manga.url}?tachiyomi=true"
val jo = webClient.httpGet(url).parseJson()
val comic = jo.getJSONObject("comic")
return manga.copy(
title = comic.getString("title"),
altTitle = null, // TODO
isNsfw = jo.getBoolean("matureContent") || comic.getBoolean("hentai"),
description = comic.getStringOrNull("parsed") ?: comic.getString("desc"),
tags = manga.tags + jo.getJSONArray("genres").mapJSONToSet {
MangaTag(
title = it.getString("name"),
key = it.getString("slug"),
source = source,
)
},
author = jo.getJSONArray("artists").optJSONObject(0)?.getString("name"),
chapters = getChapters(comic.getLong("id")),
)
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val jo = context.httpGet(
"https://api.${getDomain()}/chapter/${chapter.url}?tachiyomi=true",
).parseJson().getJSONObject("chapter")
val referer = "https://${getDomain()}/"
return jo.getJSONArray("images").mapJSON {
val url = it.getString("url")
MangaPage(
id = generateUid(url),
url = url,
referer = referer,
preview = null,
source = source,
)
}
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val jo = webClient.httpGet(
"https://api.${domain}/chapter/${chapter.url}?tachiyomi=true",
).parseJson().getJSONObject("chapter")
val referer = "https://${domain}/"
return jo.getJSONArray("images").mapJSON {
val url = it.getString("url")
MangaPage(
id = generateUid(url),
url = url,
referer = referer,
preview = null,
source = source,
)
}
}
override suspend fun getTags(): Set<MangaTag> {
val sparseArray = cachedTags ?: loadTags()
val set = ArraySet<MangaTag>(sparseArray.size())
for (i in 0 until sparseArray.size()) {
set.add(sparseArray.valueAt(i))
}
return set
}
override suspend fun getTags(): Set<MangaTag> {
val sparseArray = cachedTags ?: loadTags()
val set = ArraySet<MangaTag>(sparseArray.size())
for (i in 0 until sparseArray.size()) {
set.add(sparseArray.valueAt(i))
}
return set
}
private suspend fun loadTags(): SparseArrayCompat<MangaTag> {
val ja = context.httpGet("https://api.${getDomain()}/genre").parseJsonArray()
val tags = SparseArrayCompat<MangaTag>(ja.length())
for (jo in ja.JSONIterator()) {
tags.append(
jo.getInt("id"),
MangaTag(
title = jo.getString("name"),
key = jo.getString("slug"),
source = source,
),
)
}
cachedTags = tags
return tags
}
private suspend fun loadTags(): SparseArrayCompat<MangaTag> {
val ja = webClient.httpGet("https://api.${domain}/genre").parseJsonArray()
val tags = SparseArrayCompat<MangaTag>(ja.length())
for (jo in ja.JSONIterator()) {
tags.append(
jo.getInt("id"),
MangaTag(
title = jo.getString("name"),
key = jo.getString("slug"),
source = source,
),
)
}
cachedTags = tags
return tags
}
private suspend fun getChapters(id: Long): List<MangaChapter> {
val ja = context.httpGet(
url = "https://api.${getDomain()}/comic/$id/chapter?tachiyomi=true&limit=$CHAPTERS_LIMIT",
).parseJson().getJSONArray("chapters")
val dateFormat = SimpleDateFormat("yyyy-MM-dd")
val counters = HashMap<Locale, Int>()
return ja.mapReversed { jo ->
val locale = Locale.forLanguageTag(jo.getString("lang"))
var number = counters[locale] ?: 0
number++
counters[locale] = number
MangaChapter(
id = generateUid(jo.getLong("id")),
name = buildString {
jo.getStringOrNull("vol")?.let { append("Vol ").append(it).append(' ') }
jo.getStringOrNull("chap")?.let { append("Chap ").append(it) }
jo.getStringOrNull("title")?.let { append(": ").append(it) }
},
number = number,
url = jo.getString("hid"),
scanlator = jo.optJSONArray("group_name")?.optString(0),
uploadDate = dateFormat.tryParse(jo.getString("created_at").substringBefore('T')),
branch = locale.getDisplayName(locale).toTitleCase(locale),
source = source,
)
}
}
private suspend fun getChapters(id: Long): List<MangaChapter> {
val ja = webClient.httpGet(
url = "https://api.${domain}/comic/$id/chapter?tachiyomi=true&limit=$CHAPTERS_LIMIT",
).parseJson().getJSONArray("chapters")
val dateFormat = SimpleDateFormat("yyyy-MM-dd")
val counters = HashMap<Locale, Int>()
return ja.mapReversed { jo ->
val locale = Locale.forLanguageTag(jo.getString("lang"))
var number = counters[locale] ?: 0
number++
counters[locale] = number
MangaChapter(
id = generateUid(jo.getLong("id")),
name = buildString {
jo.getStringOrNull("vol")?.let { append("Vol ").append(it).append(' ') }
jo.getStringOrNull("chap")?.let { append("Chap ").append(it) }
jo.getStringOrNull("title")?.let { append(": ").append(it) }
},
number = number,
url = jo.getString("hid"),
scanlator = jo.optJSONArray("group_name")?.optString(0),
uploadDate = dateFormat.tryParse(jo.getString("created_at").substringBefore('T')),
branch = locale.getDisplayName(locale).toTitleCase(locale),
source = source,
)
}
}
private inline fun <R> JSONArray.mapReversed(block: (JSONObject) -> R): List<R> {
val len = length()
val destination = ArrayList<R>(len)
for (i in (0 until len).reversed()) {
val jo = getJSONObject(i)
destination.add(block(jo))
}
return destination
}
private inline fun <R> JSONArray.mapReversed(block: (JSONObject) -> R): List<R> {
val len = length()
val destination = ArrayList<R>(len)
for (i in (0 until len).reversed()) {
val jo = getJSONObject(i)
destination.add(block(jo))
}
return destination
}
private fun JSONObject.selectGenres(name: String, tags: SparseArrayCompat<MangaTag>): Set<MangaTag> {
val array = optJSONArray(name) ?: return emptySet()
val res = ArraySet<MangaTag>(array.length())
for (i in 0 until array.length()) {
val id = array.getInt(i)
val tag = tags.get(id) ?: continue
res.add(tag)
}
return res
}
}
private fun JSONObject.selectGenres(name: String, tags: SparseArrayCompat<MangaTag>): Set<MangaTag> {
val array = optJSONArray(name) ?: return emptySet()
val res = ArraySet<MangaTag>(array.length())
for (i in 0 until array.length()) {
val id = array.getInt(i)
val tag = tags.get(id) ?: continue
res.add(tag)
}
return res
}
}

278
src/main/kotlin/org/koitharu/kotatsu/parsers/site/DesuMeParser.kt
Unescape Escape View File

@ -13,149 +13,149 @@ import org.koitharu.kotatsu.parsers.util.json.mapJSONToSet
import java.util.*
@MangaSourceParser("DESUME", "Desu.me", "ru")
internal class DesuMeParser(override val context: MangaLoaderContext) : PagedMangaParser(MangaSource.DESUME, 20) {
internal class DesuMeParser(context: MangaLoaderContext) : PagedMangaParser(context, MangaSource.DESUME, 20) {
override val configKeyDomain = ConfigKey.Domain("desu.me", null)
override val configKeyDomain = ConfigKey.Domain("desu.me", null)
override val sortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.UPDATED,
SortOrder.POPULARITY,
SortOrder.NEWEST,
SortOrder.ALPHABETICAL,
)
override val sortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.UPDATED,
SortOrder.POPULARITY,
SortOrder.NEWEST,
SortOrder.ALPHABETICAL,
)
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
if (query != null && page != searchPaginator.firstPage) {
return emptyList()
}
val domain = getDomain()
val url = buildString {
append("https://")
append(domain)
append("/manga/api/?limit=20&order=")
append(getSortKey(sortOrder))
append("&page=")
append(page)
if (!tags.isNullOrEmpty()) {
append("&genres=")
appendAll(tags, ",") { it.key }
}
if (query != null) {
append("&search=")
append(query)
}
}
val json = context.httpGet(url).parseJson().getJSONArray("response")
?: throw ParseException("Invalid response", url)
val total = json.length()
val list = ArrayList<Manga>(total)
for (i in 0 until total) {
val jo = json.getJSONObject(i)
val cover = jo.getJSONObject("image")
val id = jo.getLong("id")
list += Manga(
url = "/manga/api/$id",
publicUrl = jo.getString("url"),
source = MangaSource.DESUME,
title = jo.getString("russian"),
altTitle = jo.getString("name"),
coverUrl = cover.getString("preview"),
largeCoverUrl = cover.getString("original"),
state = when {
jo.getInt("ongoing") == 1 -> MangaState.ONGOING
else -> null
},
rating = jo.getDouble("score").toFloat().coerceIn(0f, 1f),
id = generateUid(id),
isNsfw = false,
tags = emptySet(),
author = null,
description = jo.getString("description"),
)
}
return list
}
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
if (query != null && page != searchPaginator.firstPage) {
return emptyList()
}
val domain = domain
val url = buildString {
append("https://")
append(domain)
append("/manga/api/?limit=20&order=")
append(getSortKey(sortOrder))
append("&page=")
append(page)
if (!tags.isNullOrEmpty()) {
append("&genres=")
appendAll(tags, ",") { it.key }
}
if (query != null) {
append("&search=")
append(query)
}
}
val json = webClient.httpGet(url).parseJson().getJSONArray("response")
?: throw ParseException("Invalid response", url)
val total = json.length()
val list = ArrayList<Manga>(total)
for (i in 0 until total) {
val jo = json.getJSONObject(i)
val cover = jo.getJSONObject("image")
val id = jo.getLong("id")
list += Manga(
url = "/manga/api/$id",
publicUrl = jo.getString("url"),
source = MangaSource.DESUME,
title = jo.getString("russian"),
altTitle = jo.getString("name"),
coverUrl = cover.getString("preview"),
largeCoverUrl = cover.getString("original"),
state = when {
jo.getInt("ongoing") == 1 -> MangaState.ONGOING
else -> null
},
rating = jo.getDouble("score").toFloat().coerceIn(0f, 1f),
id = generateUid(id),
isNsfw = false,
tags = emptySet(),
author = null,
description = jo.getString("description"),
)
}
return list
}
override suspend fun getDetails(manga: Manga): Manga {
val url = manga.url.toAbsoluteUrl(getDomain())
val json = context.httpGet(url).parseJson().getJSONObject("response")
?: throw ParseException("Invalid response", url)
val baseChapterUrl = manga.url + "/chapter/"
val chaptersList = json.getJSONObject("chapters").getJSONArray("list")
val totalChapters = chaptersList.length()
return manga.copy(
tags = json.getJSONArray("genres").mapJSONToSet {
MangaTag(
key = it.getString("text"),
title = it.getString("russian").toTitleCase(),
source = manga.source,
)
},
publicUrl = json.getString("url"),
description = json.getString("description"),
chapters = chaptersList.mapJSONIndexed { i, it ->
val chid = it.getLong("id")
val volChap = "Том " + it.optString("vol", "0") + ". " + "Глава " + it.optString("ch", "0")
val title = it.optString("title", "null").takeUnless { it == "null" }
MangaChapter(
id = generateUid(chid),
source = manga.source,
url = "$baseChapterUrl$chid",
uploadDate = it.getLong("date") * 1000,
name = if (title.isNullOrEmpty()) volChap else "$volChap: $title",
number = totalChapters - i,
scanlator = null,
branch = null,
)
}.reversed(),
)
}
override suspend fun getDetails(manga: Manga): Manga {
val url = manga.url.toAbsoluteUrl(domain)
val json = webClient.httpGet(url).parseJson().getJSONObject("response")
?: throw ParseException("Invalid response", url)
val baseChapterUrl = manga.url + "/chapter/"
val chaptersList = json.getJSONObject("chapters").getJSONArray("list")
val totalChapters = chaptersList.length()
return manga.copy(
tags = json.getJSONArray("genres").mapJSONToSet {
MangaTag(
key = it.getString("text"),
title = it.getString("russian").toTitleCase(),
source = manga.source,
)
},
publicUrl = json.getString("url"),
description = json.getString("description"),
chapters = chaptersList.mapJSONIndexed { i, it ->
val chid = it.getLong("id")
val volChap = "Том " + it.optString("vol", "0") + ". " + "Глава " + it.optString("ch", "0")
val title = it.optString("title", "null").takeUnless { it == "null" }
MangaChapter(
id = generateUid(chid),
source = manga.source,
url = "$baseChapterUrl$chid",
uploadDate = it.getLong("date") * 1000,
name = if (title.isNullOrEmpty()) volChap else "$volChap: $title",
number = totalChapters - i,
scanlator = null,
branch = null,
)
}.reversed(),
)
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(getDomain())
val json = context.httpGet(fullUrl)
.parseJson()
.getJSONObject("response") ?: throw ParseException("Invalid response", fullUrl)
return json.getJSONObject("pages").getJSONArray("list").mapJSON { jo ->
MangaPage(
id = generateUid(jo.getLong("id")),
referer = fullUrl,
preview = null,
source = chapter.source,
url = jo.getString("img"),
)
}
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(domain)
val json = webClient.httpGet(fullUrl)
.parseJson()
.getJSONObject("response") ?: throw ParseException("Invalid response", fullUrl)
return json.getJSONObject("pages").getJSONArray("list").mapJSON { jo ->
MangaPage(
id = generateUid(jo.getLong("id")),
referer = fullUrl,
preview = null,
source = chapter.source,
url = jo.getString("img"),
)
}
}
override suspend fun getTags(): Set<MangaTag> {
val doc = context.httpGet("https://${getDomain()}/manga/").parseHtml()
val root = doc.body().requireElementById("animeFilter")
.selectFirstOrThrow(".catalog-genres")
return root.select("li").mapToSet {
val input = it.selectFirstOrThrow("input")
MangaTag(
source = source,
key = input.attr("data-genre-slug").ifEmpty {
it.parseFailed("data-genre-slug is empty")
},
title = input.attr("data-genre-name").toTitleCase().ifEmpty {
it.parseFailed("data-genre-name is empty")
},
)
}
}
override suspend fun getTags(): Set<MangaTag> {
val doc = webClient.httpGet("https://${domain}/manga/").parseHtml()
val root = doc.body().requireElementById("animeFilter")
.selectFirstOrThrow(".catalog-genres")
return root.select("li").mapToSet {
val input = it.selectFirstOrThrow("input")
MangaTag(
source = source,
key = input.attr("data-genre-slug").ifEmpty {
it.parseFailed("data-genre-slug is empty")
},
title = input.attr("data-genre-name").toTitleCase().ifEmpty {
it.parseFailed("data-genre-name is empty")
},
)
}
}
private fun getSortKey(sortOrder: SortOrder) =
when (sortOrder) {
SortOrder.ALPHABETICAL -> "name"
SortOrder.POPULARITY -> "popular"
SortOrder.UPDATED -> "updated"
SortOrder.NEWEST -> "id"
else -> "updated"
}
}
private fun getSortKey(sortOrder: SortOrder) =
when (sortOrder) {
SortOrder.ALPHABETICAL -> "name"
SortOrder.POPULARITY -> "popular"
SortOrder.UPDATED -> "updated"
SortOrder.NEWEST -> "id"
else -> "updated"
}
}

534
src/main/kotlin/org/koitharu/kotatsu/parsers/site/ExHentaiParser.kt
Unescape Escape View File

@ -20,289 +20,289 @@ private const val DOMAIN_AUTHORIZED = "exhentai.org"
@MangaSourceParser("EXHENTAI", "ExHentai")
internal class ExHentaiParser(
override val context: MangaLoaderContext,
) : PagedMangaParser(MangaSource.EXHENTAI, pageSize = 25), MangaParserAuthProvider {
context: MangaLoaderContext,
) : PagedMangaParser(context, MangaSource.EXHENTAI, pageSize = 25), MangaParserAuthProvider {
override val sortOrders: Set<SortOrder> = Collections.singleton(
SortOrder.NEWEST,
)
override val sortOrders: Set<SortOrder> = Collections.singleton(
SortOrder.NEWEST,
)
override val configKeyDomain: ConfigKey.Domain
get() = ConfigKey.Domain(if (isAuthorized) DOMAIN_AUTHORIZED else DOMAIN_UNAUTHORIZED, null)
override val configKeyDomain: ConfigKey.Domain
get() = ConfigKey.Domain(if (isAuthorized) DOMAIN_AUTHORIZED else DOMAIN_UNAUTHORIZED, null)
override val authUrl: String
get() = "https://${getDomain()}/bounce_login.php"
override val authUrl: String
get() = "https://${domain}/bounce_login.php"
private val ratingPattern = Regex("-?[0-9]+px")
private val authCookies = arrayOf("ipb_member_id", "ipb_pass_hash")
private var updateDm = false
private val nextPages = SparseArrayCompat<Long>()
private val suspiciousContentKey = ConfigKey.ShowSuspiciousContent(true)
private val ratingPattern = Regex("-?[0-9]+px")
private val authCookies = arrayOf("ipb_member_id", "ipb_pass_hash")
private var updateDm = false
private val nextPages = SparseArrayCompat<Long>()
private val suspiciousContentKey = ConfigKey.ShowSuspiciousContent(true)
override val isAuthorized: Boolean
get() {
val authorized = isAuthorized(DOMAIN_UNAUTHORIZED)
if (authorized) {
if (!isAuthorized(DOMAIN_AUTHORIZED)) {
context.cookieJar.copyCookies(
DOMAIN_UNAUTHORIZED,
DOMAIN_AUTHORIZED,
authCookies,
)
context.cookieJar.insertCookies(DOMAIN_AUTHORIZED, "yay=louder")
}
return true
}
return false
}
override val isAuthorized: Boolean
get() {
val authorized = isAuthorized(DOMAIN_UNAUTHORIZED)
if (authorized) {
if (!isAuthorized(DOMAIN_AUTHORIZED)) {
context.cookieJar.copyCookies(
DOMAIN_UNAUTHORIZED,
DOMAIN_AUTHORIZED,
authCookies,
)
context.cookieJar.insertCookies(DOMAIN_AUTHORIZED, "yay=louder")
}
return true
}
return false
}
init {
context.cookieJar.insertCookies(DOMAIN_AUTHORIZED, "nw=1", "sl=dm_2")
context.cookieJar.insertCookies(DOMAIN_UNAUTHORIZED, "nw=1", "sl=dm_2")
paginator.firstPage = 0
}
init {
context.cookieJar.insertCookies(DOMAIN_AUTHORIZED, "nw=1", "sl=dm_2")
context.cookieJar.insertCookies(DOMAIN_UNAUTHORIZED, "nw=1", "sl=dm_2")
paginator.firstPage = 0
}
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
var search = query?.urlEncoded().orEmpty()
val next = nextPages.get(page, 0L)
if (page > 0 && next == 0L) {
assert(false) { "Page timestamp not found" }
return emptyList()
}
val url = buildString {
append("https://")
append(getDomain())
append("/?next=")
append(next)
if (!tags.isNullOrEmpty()) {
var fCats = 0
for (tag in tags) {
tag.key.toIntOrNull()?.let { fCats = fCats or it } ?: run {
search += tag.key + " "
}
}
if (fCats != 0) {
append("&f_cats=")
append(1023 - fCats)
}
}
if (search.isNotEmpty()) {
append("&f_search=")
append(search.trim().replace(' ', '+'))
}
// by unknown reason cookie "sl=dm_2" is ignored, so, we should request it again
if (updateDm) {
append("&inline_set=dm_e")
}
append("&advsearch=1")
if (config[suspiciousContentKey]) {
append("&f_sh=on")
}
}
val body = context.httpGet(url).parseHtml().body()
val root = body.selectFirst("table.itg")
?.selectFirst("tbody")
?: if (updateDm) {
body.parseFailed("Cannot find root")
} else {
updateDm = true
return getListPage(page, query, tags, sortOrder)
}
updateDm = false
nextPages[page + 1] = getNextTimestamp(body)
return root.children().mapNotNull { tr ->
if (tr.childrenSize() != 2) return@mapNotNull null
val (td1, td2) = tr.children()
val glink = td2.selectFirstOrThrow("div.glink")
val a = glink.parents().select("a").first() ?: glink.parseFailed("link not found")
val href = a.attrAsRelativeUrl("href")
val tagsDiv = glink.nextElementSibling() ?: glink.parseFailed("tags div not found")
val mainTag = td2.selectFirst("div.cn")?.let { div ->
MangaTag(
title = div.text().toTitleCase(),
key = tagIdByClass(div.classNames()) ?: return@let null,
source = source,
)
}
Manga(
id = generateUid(href),
title = glink.text().cleanupTitle(),
altTitle = null,
url = href,
publicUrl = a.absUrl("href"),
rating = td2.selectFirst("div.ir")?.parseRating() ?: RATING_UNKNOWN,
isNsfw = true,
coverUrl = td1.selectFirst("img")?.absUrl("src").orEmpty(),
tags = setOfNotNull(mainTag),
state = null,
author = tagsDiv.getElementsContainingOwnText("artist:").first()
?.nextElementSibling()?.text(),
source = source,
)
}
}
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
var search = query?.urlEncoded().orEmpty()
val next = nextPages.get(page, 0L)
if (page > 0 && next == 0L) {
assert(false) { "Page timestamp not found" }
return emptyList()
}
val url = buildString {
append("https://")
append(domain)
append("/?next=")
append(next)
if (!tags.isNullOrEmpty()) {
var fCats = 0
for (tag in tags) {
tag.key.toIntOrNull()?.let { fCats = fCats or it } ?: run {
search += tag.key + " "
}
}
if (fCats != 0) {
append("&f_cats=")
append(1023 - fCats)
}
}
if (search.isNotEmpty()) {
append("&f_search=")
append(search.trim().replace(' ', '+'))
}
// by unknown reason cookie "sl=dm_2" is ignored, so, we should request it again
if (updateDm) {
append("&inline_set=dm_e")
}
append("&advsearch=1")
if (config[suspiciousContentKey]) {
append("&f_sh=on")
}
}
val body = webClient.httpGet(url).parseHtml().body()
val root = body.selectFirst("table.itg")
?.selectFirst("tbody")
?: if (updateDm) {
body.parseFailed("Cannot find root")
} else {
updateDm = true
return getListPage(page, query, tags, sortOrder)
}
updateDm = false
nextPages[page + 1] = getNextTimestamp(body)
return root.children().mapNotNull { tr ->
if (tr.childrenSize() != 2) return@mapNotNull null
val (td1, td2) = tr.children()
val glink = td2.selectFirstOrThrow("div.glink")
val a = glink.parents().select("a").first() ?: glink.parseFailed("link not found")
val href = a.attrAsRelativeUrl("href")
val tagsDiv = glink.nextElementSibling() ?: glink.parseFailed("tags div not found")
val mainTag = td2.selectFirst("div.cn")?.let { div ->
MangaTag(
title = div.text().toTitleCase(),
key = tagIdByClass(div.classNames()) ?: return@let null,
source = source,
)
}
Manga(
id = generateUid(href),
title = glink.text().cleanupTitle(),
altTitle = null,
url = href,
publicUrl = a.absUrl("href"),
rating = td2.selectFirst("div.ir")?.parseRating() ?: RATING_UNKNOWN,
isNsfw = true,
coverUrl = td1.selectFirst("img")?.absUrl("src").orEmpty(),
tags = setOfNotNull(mainTag),
state = null,
author = tagsDiv.getElementsContainingOwnText("artist:").first()
?.nextElementSibling()?.text(),
source = source,
)
}
}
override suspend fun getDetails(manga: Manga): Manga {
val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml()
val root = doc.body().selectFirstOrThrow("div.gm")
val cover = root.getElementById("gd1")?.children()?.first()
val title = root.getElementById("gd2")
val taglist = root.getElementById("taglist")
val tabs = doc.body().selectFirst("table.ptt")?.selectFirst("tr")
return manga.copy(
title = title?.getElementById("gn")?.text()?.cleanupTitle() ?: manga.title,
altTitle = title?.getElementById("gj")?.text()?.cleanupTitle() ?: manga.altTitle,
publicUrl = doc.baseUri().ifEmpty { manga.publicUrl },
rating = root.getElementById("rating_label")?.text()
?.substringAfterLast(' ')
?.toFloatOrNull()
?.div(5f) ?: manga.rating,
largeCoverUrl = cover?.styleValueOrNull("background")?.cssUrl(),
description = taglist?.select("tr")?.joinToString("<br>") { tr ->
val (tc, td) = tr.children()
val subtags = td.select("a").joinToString { it.html() }
"<b>${tc.html()}</b> $subtags"
},
chapters = tabs?.select("a")?.findLast { a ->
a.text().toIntOrNull() != null
}?.let { a ->
val count = a.text().toInt()
val chapters = ChaptersListBuilder(count)
for (i in 1..count) {
val url = "${manga.url}?p=${i - 1}"
chapters += MangaChapter(
id = generateUid(url),
name = "${manga.title} #$i",
number = i,
url = url,
uploadDate = 0L,
source = source,
scanlator = null,
branch = null,
)
}
chapters.toList()
},
)
}
override suspend fun getDetails(manga: Manga): Manga {
val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml()
val root = doc.body().selectFirstOrThrow("div.gm")
val cover = root.getElementById("gd1")?.children()?.first()
val title = root.getElementById("gd2")
val taglist = root.getElementById("taglist")
val tabs = doc.body().selectFirst("table.ptt")?.selectFirst("tr")
return manga.copy(
title = title?.getElementById("gn")?.text()?.cleanupTitle() ?: manga.title,
altTitle = title?.getElementById("gj")?.text()?.cleanupTitle() ?: manga.altTitle,
publicUrl = doc.baseUri().ifEmpty { manga.publicUrl },
rating = root.getElementById("rating_label")?.text()
?.substringAfterLast(' ')
?.toFloatOrNull()
?.div(5f) ?: manga.rating,
largeCoverUrl = cover?.styleValueOrNull("background")?.cssUrl(),
description = taglist?.select("tr")?.joinToString("<br>") { tr ->
val (tc, td) = tr.children()
val subtags = td.select("a").joinToString { it.html() }
"<b>${tc.html()}</b> $subtags"
},
chapters = tabs?.select("a")?.findLast { a ->
a.text().toIntOrNull() != null
}?.let { a ->
val count = a.text().toInt()
val chapters = ChaptersListBuilder(count)
for (i in 1..count) {
val url = "${manga.url}?p=${i - 1}"
chapters += MangaChapter(
id = generateUid(url),
name = "${manga.title} #$i",
number = i,
url = url,
uploadDate = 0L,
source = source,
scanlator = null,
branch = null,
)
}
chapters.toList()
},
)
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val doc = context.httpGet(chapter.url.toAbsoluteUrl(getDomain())).parseHtml()
val root = doc.body().requireElementById("gdt")
return root.select("a").map { a ->
val url = a.attrAsRelativeUrl("href")
MangaPage(
id = generateUid(url),
url = url,
referer = a.absUrl("href"),
preview = null,
source = source,
)
}
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml()
val root = doc.body().requireElementById("gdt")
return root.select("a").map { a ->
val url = a.attrAsRelativeUrl("href")
MangaPage(
id = generateUid(url),
url = url,
referer = a.absUrl("href"),
preview = null,
source = source,
)
}
}
override suspend fun getPageUrl(page: MangaPage): String {
val doc = context.httpGet(page.url.toAbsoluteUrl(getDomain())).parseHtml()
return doc.body().requireElementById("img").attrAsAbsoluteUrl("src")
}
override suspend fun getPageUrl(page: MangaPage): String {
val doc = webClient.httpGet(page.url.toAbsoluteUrl(domain)).parseHtml()
return doc.body().requireElementById("img").attrAsAbsoluteUrl("src")
}
override suspend fun getTags(): Set<MangaTag> {
val doc = context.httpGet("https://${getDomain()}").parseHtml()
val root = doc.body().requireElementById("searchbox").selectFirstOrThrow("table")
return root.select("div.cs").mapNotNullToSet { div ->
val id = div.id().substringAfterLast('_').toIntOrNull()
?: return@mapNotNullToSet null
MangaTag(
title = div.text().toTitleCase(),
key = id.toString(),
source = source,
)
}
}
override suspend fun getTags(): Set<MangaTag> {
val doc = webClient.httpGet("https://${domain}").parseHtml()
val root = doc.body().requireElementById("searchbox").selectFirstOrThrow("table")
return root.select("div.cs").mapNotNullToSet { div ->
val id = div.id().substringAfterLast('_').toIntOrNull()
?: return@mapNotNullToSet null
MangaTag(
title = div.text().toTitleCase(),
key = id.toString(),
source = source,
)
}
}
override suspend fun getUsername(): String {
val doc = context.httpGet("https://forums.$DOMAIN_UNAUTHORIZED/").parseHtml().body()
val username = doc.getElementById("userlinks")
?.getElementsByAttributeValueContaining("href", "showuser=")
?.firstOrNull()
?.ownText()
?: if (doc.getElementById("userlinksguest") != null) {
throw AuthRequiredException(source)
} else {
doc.parseFailed()
}
return username
}
override suspend fun getUsername(): String {
val doc = webClient.httpGet("https://forums.$DOMAIN_UNAUTHORIZED/").parseHtml().body()
val username = doc.getElementById("userlinks")
?.getElementsByAttributeValueContaining("href", "showuser=")
?.firstOrNull()
?.ownText()
?: if (doc.getElementById("userlinksguest") != null) {
throw AuthRequiredException(source)
} else {
doc.parseFailed()
}
return username
}
override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) {
super.onCreateConfig(keys)
keys.add(suspiciousContentKey)
}
override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) {
super.onCreateConfig(keys)
keys.add(suspiciousContentKey)
}
private fun isAuthorized(domain: String): Boolean {
val cookies = context.cookieJar.getCookies(domain).mapToSet { x -> x.name }
return authCookies.all { it in cookies }
}
private fun isAuthorized(domain: String): Boolean {
val cookies = context.cookieJar.getCookies(domain).mapToSet { x -> x.name }
return authCookies.all { it in cookies }
}
private fun Element.parseRating(): Float {
return runCatching {
val style = requireNotNull(attr("style"))
val (v1, v2) = ratingPattern.find(style)!!.destructured
var p1 = v1.dropLast(2).toInt()
val p2 = v2.dropLast(2).toInt()
if (p2 != -1) {
p1 += 8
}
(80 - p1) / 80f
}.getOrDefault(RATING_UNKNOWN)
}
private fun Element.parseRating(): Float {
return runCatching {
val style = requireNotNull(attr("style"))
val (v1, v2) = ratingPattern.find(style)!!.destructured
var p1 = v1.dropLast(2).toInt()
val p2 = v2.dropLast(2).toInt()
if (p2 != -1) {
p1 += 8
}
(80 - p1) / 80f
}.getOrDefault(RATING_UNKNOWN)
}
private fun String.cleanupTitle(): String {
val result = StringBuilder(length)
var skip = false
for (c in this) {
when {
c == '[' -> skip = true
c == ']' -> skip = false
c.isWhitespace() && result.isEmpty() -> continue
!skip -> result.append(c)
}
}
while (result.lastOrNull()?.isWhitespace() == true) {
result.deleteCharAt(result.lastIndex)
}
return result.toString()
}
private fun String.cleanupTitle(): String {
val result = StringBuilder(length)
var skip = false
for (c in this) {
when {
c == '[' -> skip = true
c == ']' -> skip = false
c.isWhitespace() && result.isEmpty() -> continue
!skip -> result.append(c)
}
}
while (result.lastOrNull()?.isWhitespace() == true) {
result.deleteCharAt(result.lastIndex)
}
return result.toString()
}
private fun String.cssUrl(): String? {
val fromIndex = indexOf("url(")
if (fromIndex == -1) {
return null
}
val toIndex = indexOf(')', startIndex = fromIndex)
return if (toIndex == -1) {
null
} else {
substring(fromIndex + 4, toIndex).trim()
}
}
private fun String.cssUrl(): String? {
val fromIndex = indexOf("url(")
if (fromIndex == -1) {
return null
}
val toIndex = indexOf(')', startIndex = fromIndex)
return if (toIndex == -1) {
null
} else {
substring(fromIndex + 4, toIndex).trim()
}
}
private fun tagIdByClass(classNames: Collection<String>): String? {
val className = classNames.find { x -> x.startsWith("ct") } ?: return null
val num = className.drop(2).toIntOrNull(16) ?: return null
return 2.0.pow(num).toInt().toString()
}
private fun tagIdByClass(classNames: Collection<String>): String? {
val className = classNames.find { x -> x.startsWith("ct") } ?: return null
val num = className.drop(2).toIntOrNull(16) ?: return null
return 2.0.pow(num).toInt().toString()
}
private fun getNextTimestamp(root: Element): Long {
return root.getElementById("unext")
?.attrAsAbsoluteUrlOrNull("href")
?.toHttpUrlOrNull()
?.queryParameter("next")
?.toLongOrNull() ?: 1
}
}
private fun getNextTimestamp(root: Element): Long {
return root.getElementById("unext")
?.attrAsAbsoluteUrlOrNull("href")
?.toHttpUrlOrNull()
?.queryParameter("next")
?.toLongOrNull() ?: 1
}
}

470
src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaDexParser.kt
Unescape Escape View File

@ -22,254 +22,254 @@ private const val CHAPTERS_MAX_PAGE_SIZE = 500
private const val CHAPTERS_PARALLELISM = 3
private const val CHAPTERS_MAX_COUNT = 10_000 // strange api behavior, looks like a bug
private const val CONTENT_RATING =
"contentRating[]=safe&contentRating[]=suggestive&contentRating[]=erotica&contentRating[]=pornographic"
"contentRating[]=safe&contentRating[]=suggestive&contentRating[]=erotica&contentRating[]=pornographic"
private const val LOCALE_FALLBACK = "en"
@MangaSourceParser("MANGADEX", "MangaDex")
internal class MangaDexParser(override val context: MangaLoaderContext) : MangaParser(MangaSource.MANGADEX) {
internal class MangaDexParser(context: MangaLoaderContext) : MangaParser(context, MangaSource.MANGADEX) {
override val configKeyDomain = ConfigKey.Domain("mangadex.org", null)
override val configKeyDomain = ConfigKey.Domain("mangadex.org", null)
override val sortOrders: EnumSet<SortOrder> = EnumSet.of(
SortOrder.UPDATED,
SortOrder.ALPHABETICAL,
SortOrder.NEWEST,
SortOrder.POPULARITY,
)
override val sortOrders: EnumSet<SortOrder> = EnumSet.of(
SortOrder.UPDATED,
SortOrder.ALPHABETICAL,
SortOrder.NEWEST,
SortOrder.POPULARITY,
)
override suspend fun getList(
offset: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val domain = getDomain()
val url = buildString {
append("https://api.")
append(domain)
append("/manga?limit=")
append(PAGE_SIZE)
append("&offset=")
append(offset)
append("&includes[]=cover_art&includes[]=author&includes[]=artist&")
tags?.forEach { tag ->
append("includedTags[]=")
append(tag.key)
append('&')
}
if (!query.isNullOrEmpty()) {
append("title=")
append(query.urlEncoded())
append('&')
}
append(CONTENT_RATING)
append("&order")
append(
when (sortOrder) {
SortOrder.UPDATED,
-> "[latestUploadedChapter]=desc"
override suspend fun getList(
offset: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val domain = domain
val url = buildString {
append("https://api.")
append(domain)
append("/manga?limit=")
append(PAGE_SIZE)
append("&offset=")
append(offset)
append("&includes[]=cover_art&includes[]=author&includes[]=artist&")
tags?.forEach { tag ->
append("includedTags[]=")
append(tag.key)
append('&')
}
if (!query.isNullOrEmpty()) {
append("title=")
append(query.urlEncoded())
append('&')
}
append(CONTENT_RATING)
append("&order")
append(
when (sortOrder) {
SortOrder.UPDATED,
-> "[latestUploadedChapter]=desc"
SortOrder.ALPHABETICAL -> "[title]=asc"
SortOrder.NEWEST -> "[createdAt]=desc"
SortOrder.POPULARITY -> "[followedCount]=desc"
else -> "[followedCount]=desc"
},
)
}
val json = context.httpGet(url).parseJson().getJSONArray("data")
return json.mapJSON { jo ->
val id = jo.getString("id")
val attrs = jo.getJSONObject("attributes")
val relations = jo.getJSONArray("relationships").associateByKey("type")
val cover = relations["cover_art"]
?.getJSONObject("attributes")
?.getString("fileName")
?.let {
"https://uploads.$domain/covers/$id/$it"
}
Manga(
id = generateUid(id),
title = requireNotNull(attrs.getJSONObject("title").selectByLocale()) {
"Title should not be null"
},
altTitle = attrs.optJSONObject("altTitles")?.selectByLocale(),
url = id,
publicUrl = "https://$domain/title/$id",
rating = RATING_UNKNOWN,
isNsfw = attrs.getStringOrNull("contentRating") == "erotica",
coverUrl = cover?.plus(".256.jpg").orEmpty(),
largeCoverUrl = cover,
description = attrs.optJSONObject("description")?.selectByLocale(),
tags = attrs.getJSONArray("tags").mapJSONToSet { tag ->
MangaTag(
title = tag.getJSONObject("attributes")
.getJSONObject("name")
.firstStringValue()
.toTitleCase(),
key = tag.getString("id"),
source = source,
)
},
state = when (jo.getStringOrNull("status")) {
"ongoing" -> MangaState.ONGOING
"completed" -> MangaState.FINISHED
else -> null
},
author = (relations["author"] ?: relations["artist"])
?.getJSONObject("attributes")
?.getStringOrNull("name"),
source = source,
)
}
}
SortOrder.ALPHABETICAL -> "[title]=asc"
SortOrder.NEWEST -> "[createdAt]=desc"
SortOrder.POPULARITY -> "[followedCount]=desc"
else -> "[followedCount]=desc"
},
)
}
val json = webClient.httpGet(url).parseJson().getJSONArray("data")
return json.mapJSON { jo ->
val id = jo.getString("id")
val attrs = jo.getJSONObject("attributes")
val relations = jo.getJSONArray("relationships").associateByKey("type")
val cover = relations["cover_art"]
?.getJSONObject("attributes")
?.getString("fileName")
?.let {
"https://uploads.$domain/covers/$id/$it"
}
Manga(
id = generateUid(id),
title = requireNotNull(attrs.getJSONObject("title").selectByLocale()) {
"Title should not be null"
},
altTitle = attrs.optJSONObject("altTitles")?.selectByLocale(),
url = id,
publicUrl = "https://$domain/title/$id",
rating = RATING_UNKNOWN,
isNsfw = attrs.getStringOrNull("contentRating") == "erotica",
coverUrl = cover?.plus(".256.jpg").orEmpty(),
largeCoverUrl = cover,
description = attrs.optJSONObject("description")?.selectByLocale(),
tags = attrs.getJSONArray("tags").mapJSONToSet { tag ->
MangaTag(
title = tag.getJSONObject("attributes")
.getJSONObject("name")
.firstStringValue()
.toTitleCase(),
key = tag.getString("id"),
source = source,
)
},
state = when (jo.getStringOrNull("status")) {
"ongoing" -> MangaState.ONGOING
"completed" -> MangaState.FINISHED
else -> null
},
author = (relations["author"] ?: relations["artist"])
?.getJSONObject("attributes")
?.getStringOrNull("name"),
source = source,
)
}
}
override suspend fun getDetails(manga: Manga): Manga = coroutineScope {
val domain = getDomain()
val mangaId = manga.url.removePrefix("/")
val attrsDeferred = async {
context.httpGet(
"https://api.$domain/manga/${mangaId}?includes[]=artist&includes[]=author&includes[]=cover_art",
).parseJson().getJSONObject("data").getJSONObject("attributes")
}
val feedDeferred = async { loadChapters(mangaId) }
val mangaAttrs = attrsDeferred.await()
val feed = feedDeferred.await()
// 2022-01-02T00:27:11+00:00
val dateFormat = SimpleDateFormat(
"yyyy-MM-dd'T'HH:mm:ss'+00:00'",
Locale.ROOT,
)
manga.copy(
description = mangaAttrs.optJSONObject("description")?.selectByLocale()
?: manga.description,
chapters = feed.mapChapters { _, jo ->
val id = jo.getString("id")
val attrs = jo.getJSONObject("attributes")
if (!attrs.isNull("externalUrl")) {
return@mapChapters null
}
val locale = attrs.getStringOrNull("translatedLanguage")?.let { Locale.forLanguageTag(it) }
val relations = jo.getJSONArray("relationships").associateByKey("type")
val number = attrs.getIntOrDefault("chapter", 0)
MangaChapter(
id = generateUid(id),
name = attrs.getStringOrNull("title")?.takeUnless(String::isEmpty)
?: "Chapter #$number",
number = number,
url = id,
scanlator = relations["scanlation_group"]?.getStringOrNull("name"),
uploadDate = dateFormat.tryParse(attrs.getString("publishAt")),
branch = locale?.getDisplayName(locale)?.toTitleCase(locale),
source = source,
)
},
)
}
override suspend fun getDetails(manga: Manga): Manga = coroutineScope {
val domain = domain
val mangaId = manga.url.removePrefix("/")
val attrsDeferred = async {
webClient.httpGet(
"https://api.$domain/manga/${mangaId}?includes[]=artist&includes[]=author&includes[]=cover_art",
).parseJson().getJSONObject("data").getJSONObject("attributes")
}
val feedDeferred = async { loadChapters(mangaId) }
val mangaAttrs = attrsDeferred.await()
val feed = feedDeferred.await()
// 2022-01-02T00:27:11+00:00
val dateFormat = SimpleDateFormat(
"yyyy-MM-dd'T'HH:mm:ss'+00:00'",
Locale.ROOT,
)
manga.copy(
description = mangaAttrs.optJSONObject("description")?.selectByLocale()
?: manga.description,
chapters = feed.mapChapters { _, jo ->
val id = jo.getString("id")
val attrs = jo.getJSONObject("attributes")
if (!attrs.isNull("externalUrl")) {
return@mapChapters null
}
val locale = attrs.getStringOrNull("translatedLanguage")?.let { Locale.forLanguageTag(it) }
val relations = jo.getJSONArray("relationships").associateByKey("type")
val number = attrs.getIntOrDefault("chapter", 0)
MangaChapter(
id = generateUid(id),
name = attrs.getStringOrNull("title")?.takeUnless(String::isEmpty)
?: "Chapter #$number",
number = number,
url = id,
scanlator = relations["scanlation_group"]?.getStringOrNull("name"),
uploadDate = dateFormat.tryParse(attrs.getString("publishAt")),
branch = locale?.getDisplayName(locale)?.toTitleCase(locale),
source = source,
)
},
)
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val domain = getDomain()
val chapterJson = context.httpGet("https://api.$domain/at-home/server/${chapter.url}?forcePort443=false")
.parseJson()
.getJSONObject("chapter")
val pages = chapterJson.getJSONArray("data")
val prefix = "https://uploads.$domain/data/${chapterJson.getString("hash")}/"
val referer = "https://$domain/"
return List(pages.length()) { i ->
val url = prefix + pages.getString(i)
MangaPage(
id = generateUid(url),
url = url,
referer = referer,
preview = null, // TODO prefix + dataSaver.getString(i),
source = source,
)
}
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val domain = domain
val chapterJson = webClient.httpGet("https://api.$domain/at-home/server/${chapter.url}?forcePort443=false")
.parseJson()
.getJSONObject("chapter")
val pages = chapterJson.getJSONArray("data")
val prefix = "https://uploads.$domain/data/${chapterJson.getString("hash")}/"
val referer = "https://$domain/"
return List(pages.length()) { i ->
val url = prefix + pages.getString(i)
MangaPage(
id = generateUid(url),
url = url,
referer = referer,
preview = null, // TODO prefix + dataSaver.getString(i),
source = source,
)
}
}
override suspend fun getTags(): Set<MangaTag> {
val tags = context.httpGet("https://api.${getDomain()}/manga/tag").parseJson()
.getJSONArray("data")
return tags.mapJSONToSet { jo ->
MangaTag(
title = jo.getJSONObject("attributes").getJSONObject("name").firstStringValue().toTitleCase(),
key = jo.getString("id"),
source = source,
)
}
}
override suspend fun getTags(): Set<MangaTag> {
val tags = webClient.httpGet("https://api.${domain}/manga/tag").parseJson()
.getJSONArray("data")
return tags.mapJSONToSet { jo ->
MangaTag(
title = jo.getJSONObject("attributes").getJSONObject("name").firstStringValue().toTitleCase(),
key = jo.getString("id"),
source = source,
)
}
}
private fun JSONObject.firstStringValue() = values().next() as String
private fun JSONObject.firstStringValue() = values().next() as String
private fun JSONObject.selectByLocale(): String? {
val preferredLocales = context.getPreferredLocales()
for (locale in preferredLocales) {
getStringOrNull(locale.language)?.let { return it }
getStringOrNull(locale.toLanguageTag())?.let { return it }
}
return getStringOrNull(LOCALE_FALLBACK) ?: values().nextOrNull() as? String
}
private fun JSONObject.selectByLocale(): String? {
val preferredLocales = context.getPreferredLocales()
for (locale in preferredLocales) {
getStringOrNull(locale.language)?.let { return it }
getStringOrNull(locale.toLanguageTag())?.let { return it }
}
return getStringOrNull(LOCALE_FALLBACK) ?: values().nextOrNull() as? String
}
private suspend fun loadChapters(mangaId: String): List<JSONObject> {
val firstPage = loadChapters(mangaId, offset = 0, limit = CHAPTERS_FIRST_PAGE_SIZE)
if (firstPage.size >= firstPage.total) {
return firstPage.data
}
val tail = coroutineScope {
val leftCount = firstPage.total.coerceAtMost(CHAPTERS_MAX_COUNT) - firstPage.size
val pages = (leftCount / CHAPTERS_MAX_PAGE_SIZE.toFloat()).toIntUp()
val dispatcher = Dispatchers.Default.limitedParallelism(CHAPTERS_PARALLELISM)
List(pages) { page ->
val offset = page * CHAPTERS_MAX_PAGE_SIZE + firstPage.size
async(dispatcher) {
loadChapters(mangaId, offset, CHAPTERS_MAX_PAGE_SIZE)
}
}.awaitAll()
}
val result = ArrayList<JSONObject>(firstPage.total)
result += firstPage.data
tail.flatMapTo(result) { it.data }
return result
}
private suspend fun loadChapters(mangaId: String): List<JSONObject> {
val firstPage = loadChapters(mangaId, offset = 0, limit = CHAPTERS_FIRST_PAGE_SIZE)
if (firstPage.size >= firstPage.total) {
return firstPage.data
}
val tail = coroutineScope {
val leftCount = firstPage.total.coerceAtMost(CHAPTERS_MAX_COUNT) - firstPage.size
val pages = (leftCount / CHAPTERS_MAX_PAGE_SIZE.toFloat()).toIntUp()
val dispatcher = Dispatchers.Default.limitedParallelism(CHAPTERS_PARALLELISM)
List(pages) { page ->
val offset = page * CHAPTERS_MAX_PAGE_SIZE + firstPage.size
async(dispatcher) {
loadChapters(mangaId, offset, CHAPTERS_MAX_PAGE_SIZE)
}
}.awaitAll()
}
val result = ArrayList<JSONObject>(firstPage.total)
result += firstPage.data
tail.flatMapTo(result) { it.data }
return result
}
private suspend fun loadChapters(mangaId: String, offset: Int, limit: Int): Chapters {
val limitedLimit = when {
offset >= CHAPTERS_MAX_COUNT -> return Chapters(emptyList(), CHAPTERS_MAX_COUNT)
offset + limit > CHAPTERS_MAX_COUNT -> CHAPTERS_MAX_COUNT - offset
else -> limit
}
val url = buildString {
append("https://api.")
append(getDomain())
append("/manga/")
append(mangaId)
append("/feed")
append("?limit=")
append(limitedLimit)
append("&includes[]=scanlation_group&order[volume]=asc&order[chapter]=asc&offset=")
append(offset)
append('&')
append(CONTENT_RATING)
}
val json = context.httpGet(url).parseJson()
if (json.getString("result") == "ok") {
return Chapters(
data = json.optJSONArray("data")?.toJSONList().orEmpty(),
total = json.getInt("total"),
)
} else {
val error = json.optJSONArray("errors").mapJSON { jo ->
jo.getString("detail")
}.joinToString("\n")
throw ParseException(error, url)
}
}
private suspend fun loadChapters(mangaId: String, offset: Int, limit: Int): Chapters {
val limitedLimit = when {
offset >= CHAPTERS_MAX_COUNT -> return Chapters(emptyList(), CHAPTERS_MAX_COUNT)
offset + limit > CHAPTERS_MAX_COUNT -> CHAPTERS_MAX_COUNT - offset
else -> limit
}
val url = buildString {
append("https://api.")
append(domain)
append("/manga/")
append(mangaId)
append("/feed")
append("?limit=")
append(limitedLimit)
append("&includes[]=scanlation_group&order[volume]=asc&order[chapter]=asc&offset=")
append(offset)
append('&')
append(CONTENT_RATING)
}
val json = webClient.httpGet(url).parseJson()
if (json.getString("result") == "ok") {
return Chapters(
data = json.optJSONArray("data")?.toJSONList().orEmpty(),
total = json.getInt("total"),
)
} else {
val error = json.optJSONArray("errors").mapJSON { jo ->
jo.getString("detail")
}.joinToString("\n")
throw ParseException(error, url)
}
}
private class Chapters(
val data: List<JSONObject>,
val total: Int,
) {
private class Chapters(
val data: List<JSONObject>,
val total: Int,
) {
val size: Int
get() = data.size
}
}
val size: Int
get() = data.size
}
}

261
src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaInUaParser.kt
Unescape Escape View File

@ -12,141 +12,142 @@ import java.util.*
private const val DEF_BRANCH_NAME = "Основний переклад"
@MangaSourceParser("MANGAINUA", "MANGA/in/UA", "uk")
class MangaInUaParser(override val context: MangaLoaderContext) : PagedMangaParser(
source = MangaSource.MANGAINUA,
pageSize = 24,
searchPageSize = 10,
class MangaInUaParser(context: MangaLoaderContext) : PagedMangaParser(
context = context,
source = MangaSource.MANGAINUA,
pageSize = 24,
searchPageSize = 10,
) {
override val sortOrders: Set<SortOrder>
get() = Collections.singleton(SortOrder.UPDATED)
override val sortOrders: Set<SortOrder>
get() = Collections.singleton(SortOrder.UPDATED)
override val configKeyDomain: ConfigKey.Domain = ConfigKey.Domain("manga.in.ua", null)
override val configKeyDomain: ConfigKey.Domain = ConfigKey.Domain("manga.in.ua", null)
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val url = when {
!query.isNullOrEmpty() -> (
"/index.php?do=search" +
"&subaction=search" +
"&search_start=$page" +
"&full_search=1" +
"&story=$query" +
"&titleonly=3"
).toAbsoluteUrl(getDomain())
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val url = when {
!query.isNullOrEmpty() -> (
"/index.php?do=search" +
"&subaction=search" +
"&search_start=$page" +
"&full_search=1" +
"&story=$query" +
"&titleonly=3"
).toAbsoluteUrl(domain)
tags.isNullOrEmpty() -> "/mangas/page/$page".toAbsoluteUrl(getDomain())
tags.size == 1 -> "${tags.first().key}/page/$page"
tags.size > 1 -> throw IllegalArgumentException("This source supports only 1 genre")
else -> "/mangas/page/$page".toAbsoluteUrl(getDomain())
}
val doc = context.httpGet(url).parseHtml()
val container = doc.body().requireElementById("site-content")
val items = container.select("div.col-6")
return items.mapNotNull { item ->
val href = item.selectFirst("a")?.attrAsRelativeUrl("href") ?: return@mapNotNull null
Manga(
id = generateUid(href),
title = item.selectFirst("h3.card__title")?.text() ?: return@mapNotNull null,
coverUrl = item.selectFirst("header.card__cover")?.selectFirst("img")?.run {
attrAsAbsoluteUrlOrNull("data-src") ?: attrAsAbsoluteUrlOrNull("src")
}.orEmpty(),
altTitle = null,
author = null,
rating = item.selectFirst("div.card__short-rate--num")
?.text()
?.toFloatOrNull()
?.div(10F) ?: RATING_UNKNOWN,
url = href,
isNsfw = item.selectFirst("ul.card__list")?.select("li")?.lastOrNull()?.text() == "18+",
tags = runCatching {
item.selectFirst("div.card__category")?.select("a")?.mapToSet {
MangaTag(
title = it.ownText(),
key = it.attr("href").removeSuffix("/"),
source = source,
)
}
}.getOrNull().orEmpty(),
state = null,
publicUrl = href.toAbsoluteUrl(container.host ?: getDomain()),
source = source,
)
}
}
tags.isNullOrEmpty() -> "/mangas/page/$page".toAbsoluteUrl(domain)
tags.size == 1 -> "${tags.first().key}/page/$page"
tags.size > 1 -> throw IllegalArgumentException("This source supports only 1 genre")
else -> "/mangas/page/$page".toAbsoluteUrl(domain)
}
val doc = webClient.httpGet(url).parseHtml()
val container = doc.body().requireElementById("site-content")
val items = container.select("div.col-6")
return items.mapNotNull { item ->
val href = item.selectFirst("a")?.attrAsRelativeUrl("href") ?: return@mapNotNull null
Manga(
id = generateUid(href),
title = item.selectFirst("h3.card__title")?.text() ?: return@mapNotNull null,
coverUrl = item.selectFirst("header.card__cover")?.selectFirst("img")?.run {
attrAsAbsoluteUrlOrNull("data-src") ?: attrAsAbsoluteUrlOrNull("src")
}.orEmpty(),
altTitle = null,
author = null,
rating = item.selectFirst("div.card__short-rate--num")
?.text()
?.toFloatOrNull()
?.div(10F) ?: RATING_UNKNOWN,
url = href,
isNsfw = item.selectFirst("ul.card__list")?.select("li")?.lastOrNull()?.text() == "18+",
tags = runCatching {
item.selectFirst("div.card__category")?.select("a")?.mapToSet {
MangaTag(
title = it.ownText(),
key = it.attr("href").removeSuffix("/"),
source = source,
)
}
}.getOrNull().orEmpty(),
state = null,
publicUrl = href.toAbsoluteUrl(container.host ?: domain),
source = source,
)
}
}
override suspend fun getDetails(manga: Manga): Manga {
val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml()
val root = doc.body().requireElementById("site-content")
val dateFormat = SimpleDateFormat("dd.MM.yyyy", Locale.US)
val chapterNodes = root.selectFirstOrThrow(".linkstocomics").select(".ltcitems")
var prevChapterName: String? = null
var i = 0
return manga.copy(
description = root.selectFirst("div.item__full-description")?.text(),
largeCoverUrl = root.selectFirst("div.item__full-sidebar--poster")?.selectFirst("img")
?.attrAsAbsoluteUrlOrNull("src"),
chapters = chapterNodes.mapChapters { _, item ->
val href = item?.selectFirst("a")?.attrAsRelativeUrlOrNull("href")
?: return@mapChapters null
val isAlternative = item.styleValueOrNull("background") != null
val name = item.selectFirst("a")?.text().orEmpty()
if (!isAlternative) i++
MangaChapter(
id = generateUid(href),
name = if (isAlternative) {
prevChapterName ?: return@mapChapters null
} else {
prevChapterName = name
name
},
number = i,
url = href,
scanlator = null,
branch = if (isAlternative) {
name.substringAfterLast(':').trim()
} else {
DEF_BRANCH_NAME
},
uploadDate = dateFormat.tryParse(item.selectFirst("div.ltcright")?.text()),
source = source,
)
},
)
}
override suspend fun getDetails(manga: Manga): Manga {
val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml()
val root = doc.body().requireElementById("site-content")
val dateFormat = SimpleDateFormat("dd.MM.yyyy", Locale.US)
val chapterNodes = root.selectFirstOrThrow(".linkstocomics").select(".ltcitems")
var prevChapterName: String? = null
var i = 0
return manga.copy(
description = root.selectFirst("div.item__full-description")?.text(),
largeCoverUrl = root.selectFirst("div.item__full-sidebar--poster")?.selectFirst("img")
?.attrAsAbsoluteUrlOrNull("src"),
chapters = chapterNodes.mapChapters { _, item ->
val href = item?.selectFirst("a")?.attrAsRelativeUrlOrNull("href")
?: return@mapChapters null
val isAlternative = item.styleValueOrNull("background") != null
val name = item.selectFirst("a")?.text().orEmpty()
if (!isAlternative) i++
MangaChapter(
id = generateUid(href),
name = if (isAlternative) {
prevChapterName ?: return@mapChapters null
} else {
prevChapterName = name
name
},
number = i,
url = href,
scanlator = null,
branch = if (isAlternative) {
name.substringAfterLast(':').trim()
} else {
DEF_BRANCH_NAME
},
uploadDate = dateFormat.tryParse(item.selectFirst("div.ltcright")?.text()),
source = source,
)
},
)
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(getDomain())
val doc = context.httpGet(fullUrl).parseHtml()
val root = doc.body().requireElementById("comics").selectFirstOrThrow("ul.xfieldimagegallery")
return root.select("li").map { ul ->
val img = ul.selectFirstOrThrow("img")
val url = img.attrAsAbsoluteUrl("data-src")
MangaPage(
id = generateUid(url),
url = url,
preview = null,
referer = fullUrl,
source = source,
)
}
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(domain)
val doc = webClient.httpGet(fullUrl).parseHtml()
val root = doc.body().requireElementById("comics").selectFirstOrThrow("ul.xfieldimagegallery")
return root.select("li").map { ul ->
val img = ul.selectFirstOrThrow("img")
val url = img.attrAsAbsoluteUrl("data-src")
MangaPage(
id = generateUid(url),
url = url,
preview = null,
referer = fullUrl,
source = source,
)
}
}
override suspend fun getTags(): Set<MangaTag> {
val domain = getDomain()
val doc = context.httpGet("https://$domain/mangas").parseHtml()
val root = doc.body().requireElementById("menu_1").selectFirstOrThrow("div.menu__wrapper")
return root.select("li").mapNotNullToSet { li ->
val a = li.selectFirst("a") ?: return@mapNotNullToSet null
MangaTag(
title = a.ownText(),
key = a.attr("href").removeSuffix("/"),
source = source,
)
}
}
}
override suspend fun getTags(): Set<MangaTag> {
val domain = domain
val doc = webClient.httpGet("https://$domain/mangas").parseHtml()
val root = doc.body().requireElementById("menu_1").selectFirstOrThrow("div.menu__wrapper")
return root.select("li").mapNotNullToSet { li ->
val a = li.selectFirst("a") ?: return@mapNotNullToSet null
MangaTag(
title = a.ownText(),
key = a.attr("href").removeSuffix("/"),
source = source,
)
}
}
}

187
src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaOwlParser.kt
Unescape Escape View File

@ -1,187 +0,0 @@
package org.koitharu.kotatsu.parsers.site
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import java.text.SimpleDateFormat
import java.util.*
@Deprecated("Permanently shutdown")
@MangaSourceParser("MANGAOWL", "MangaOwl", "en")
internal class MangaOwlParser(override val context: MangaLoaderContext) : MangaParser(MangaSource.MANGAOWL) {
override val configKeyDomain = ConfigKey.Domain("mangaowls.com", null)
override val sortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.POPULARITY,
SortOrder.NEWEST,
SortOrder.UPDATED,
)
private val regexNsfw = Regex("(yaoi)|(yuri)|(smut)|(mature)|(adult)", RegexOption.IGNORE_CASE)
override suspend fun getList(
offset: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val page = (offset / 36f).toIntUp().inc()
val link = buildString {
append("https://")
append(getDomain())
when {
!query.isNullOrEmpty() -> {
append("/search/$page?search=")
append(query.urlEncoded())
}
!tags.isNullOrEmpty() -> {
for (tag in tags) {
append(tag.key)
}
append("/$page?type=${getAlternativeSortKey(sortOrder)}")
}
else -> {
append("/${getSortKey(sortOrder)}/$page")
}
}
}
val doc = context.httpGet(link).parseHtml()
val slides = doc.body().selectOrThrow("ul.slides")
val items = slides.select("div.col-md-2")
return items.mapNotNull { item ->
val href = item.selectFirst("h6 a")?.attrAsRelativeUrlOrNull("href") ?: return@mapNotNull null
Manga(
id = generateUid(href),
title = item.selectFirst("h6 a")?.text() ?: return@mapNotNull null,
coverUrl = item.select("div.img-responsive").attr("abs:data-background-image"),
altTitle = null,
author = null,
rating = runCatching {
item.selectFirst("div.block-stars")
?.text()
?.toFloatOrNull()
?.div(10f)
}.getOrNull() ?: RATING_UNKNOWN,
url = href,
isNsfw = false,
tags = emptySet(),
state = null,
publicUrl = href.toAbsoluteUrl(getDomain()),
source = source,
)
}
}
override suspend fun getDetails(manga: Manga): Manga {
val doc = context.httpGet(manga.publicUrl).parseHtml()
val info = doc.body().selectFirstOrThrow("div.single_detail")
val table = doc.body().selectFirstOrThrow("div.single-grid-right")
val dateFormat = SimpleDateFormat("MM/dd/yyyy", Locale.US)
val trRegex = "window\\['tr'] = '([^']*)';".toRegex(RegexOption.IGNORE_CASE)
val trElement = doc.getElementsByTag("script").find { trRegex.find(it.data()) != null }
?: doc.parseFailed("Oops, tr not found")
val tr = trRegex.find(trElement.data())!!.groups[1]!!.value
val s = context.encodeBase64(getDomain().toByteArray())
var isNsfw = manga.isNsfw
val parsedTags = info.select("div.col-xs-12.col-md-8.single-right-grid-right > p > a[href*=genres]")
.mapNotNullToSet {
val a = it.selectFirst("a") ?: return@mapNotNullToSet null
val name = a.text()
if (!isNsfw && isNsfwGenre(name)) {
isNsfw = true
}
MangaTag(
title = name.toTitleCase(),
key = a.attr("href"),
source = source,
)
}
return manga.copy(
description = info.selectFirst(".description")?.html(),
largeCoverUrl = info.select("img").first()?.let { img ->
if (img.hasAttr("data-src")) img.attr("abs:data-src") else img.attr("abs:src")
},
isNsfw = isNsfw,
author = info.selectFirst("p.fexi_header_para a.author_link")?.text(),
state = parseStatus(info.select("p.fexi_header_para:contains(status)").first()?.ownText()),
tags = manga.tags + parsedTags,
chapters = table.select("div.table.table-chapter-list").select("li.list-group-item.chapter_list")
.asReversed().mapChapters { i, li ->
val a = li.select("a")
val href = a.attr("data-href").ifEmpty {
li.parseFailed("Link is missing")
}
MangaChapter(
id = generateUid(href),
name = a.select("label").text(),
number = i + 1,
url = "$href?tr=$tr&s=$s",
scanlator = null,
branch = null,
uploadDate = dateFormat.tryParse(li.selectFirst("small:last-of-type")?.text()),
source = MangaSource.MANGAOWL,
)
},
)
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(getDomain())
val doc = context.httpGet(fullUrl).parseHtml()
val root = doc.body().selectOrThrow("div.item img.owl-lazy")
return root.map { div ->
val url = div?.attrAsRelativeUrlOrNull("data-src") ?: doc.parseFailed("Page image not found")
MangaPage(
id = generateUid(url),
url = url,
preview = null,
referer = url,
source = MangaSource.MANGAOWL,
)
}
}
private fun parseStatus(status: String?) = when {
status == null -> null
status.contains("Ongoing") -> MangaState.ONGOING
status.contains("Completed") -> MangaState.FINISHED
else -> null
}
override suspend fun getTags(): Set<MangaTag> {
val doc = context.httpGet("https://${getDomain()}/").parseHtml()
val root = doc.body().select("ul.dropdown-menu.multi-column.columns-3").select("li")
return root.mapToSet { p ->
val a = p.selectFirstOrThrow("a")
MangaTag(
title = a.text().toTitleCase(),
key = a.attr("href"),
source = source,
)
}
}
private fun getSortKey(sortOrder: SortOrder) =
when (sortOrder) {
SortOrder.POPULARITY -> "popular"
SortOrder.NEWEST -> "new_release"
SortOrder.UPDATED -> "lastest"
else -> "lastest"
}
private fun getAlternativeSortKey(sortOrder: SortOrder) =
when (sortOrder) {
SortOrder.POPULARITY -> "0"
SortOrder.NEWEST -> "2"
SortOrder.UPDATED -> "3"
else -> "3"
}
private fun isNsfwGenre(name: String): Boolean = regexNsfw.containsMatchIn(name)
}

384
src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaTownParser.kt
Unescape Escape View File

@ -11,207 +11,207 @@ import java.text.SimpleDateFormat
import java.util.*
@MangaSourceParser("MANGATOWN", "MangaTown", "en")
internal class MangaTownParser(override val context: MangaLoaderContext) : MangaParser(MangaSource.MANGATOWN) {
internal class MangaTownParser(context: MangaLoaderContext) : MangaParser(context, MangaSource.MANGATOWN) {
override val configKeyDomain = ConfigKey.Domain("www.mangatown.com", null)
override val configKeyDomain = ConfigKey.Domain("www.mangatown.com", null)
override val sortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.ALPHABETICAL,
SortOrder.RATING,
SortOrder.POPULARITY,
SortOrder.UPDATED,
)
override val sortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.ALPHABETICAL,
SortOrder.RATING,
SortOrder.POPULARITY,
SortOrder.UPDATED,
)
private val regexTag = Regex("[^\\-]+-[^\\-]+-[^\\-]+-[^\\-]+-[^\\-]+-[^\\-]+")
private val regexTag = Regex("[^\\-]+-[^\\-]+-[^\\-]+-[^\\-]+-[^\\-]+-[^\\-]+")
override suspend fun getList(
offset: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val sortKey = when (sortOrder) {
SortOrder.ALPHABETICAL -> "?name.az"
SortOrder.RATING -> "?rating.za"
SortOrder.UPDATED -> "?last_chapter_time.za"
else -> ""
}
val page = (offset / 30) + 1
val url = when {
!query.isNullOrEmpty() -> {
if (offset != 0) {
return emptyList()
}
"/search?name=${query.urlEncoded()}".toAbsoluteUrl(getDomain())
}
override suspend fun getList(
offset: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val sortKey = when (sortOrder) {
SortOrder.ALPHABETICAL -> "?name.az"
SortOrder.RATING -> "?rating.za"
SortOrder.UPDATED -> "?last_chapter_time.za"
else -> ""
}
val page = (offset / 30) + 1
val url = when {
!query.isNullOrEmpty() -> {
if (offset != 0) {
return emptyList()
}
"/search?name=${query.urlEncoded()}".toAbsoluteUrl(domain)
}
tags.isNullOrEmpty() -> "/directory/$page.htm$sortKey".toAbsoluteUrl(getDomain())
tags.size == 1 -> "/directory/${tags.first().key}/$page.htm$sortKey".toAbsoluteUrl(getDomain())
else -> tags.joinToString(
prefix = "/search?page=$page".toAbsoluteUrl(getDomain()),
) { tag ->
"&genres[${tag.key}]=1"
}
}
val doc = context.httpGet(url).parseHtml()
val root = doc.body().selectFirstOrThrow("ul.manga_pic_list")
return root.select("li").mapNotNull { li ->
val a = li.selectFirst("a.manga_cover")
val href = a?.attrAsRelativeUrlOrNull("href")
?: return@mapNotNull null
val views = li.select("p.view")
val status = views.firstNotNullOfOrNull { it.ownText().takeIf { x -> x.startsWith("Status:") } }
?.substringAfter(':')?.trim()?.lowercase(Locale.ROOT)
Manga(
id = generateUid(href),
title = a.attr("title"),
coverUrl = a.selectFirst("img")?.absUrl("src").orEmpty(),
source = MangaSource.MANGATOWN,
altTitle = null,
rating = li.selectFirst("p.score")?.selectFirst("b")
?.ownText()?.toFloatOrNull()?.div(5f) ?: RATING_UNKNOWN,
author = views.firstNotNullOfOrNull { it.text().takeIf { x -> x.startsWith("Author:") } }
?.substringAfter(':')
?.trim(),
state = when (status) {
"ongoing" -> MangaState.ONGOING
"completed" -> MangaState.FINISHED
else -> null
},
tags = li.selectFirst("p.keyWord")?.select("a")?.mapNotNullToSet tags@{ x ->
MangaTag(
title = x.attr("title").toTitleCase(),
key = x.attr("href").parseTagKey() ?: return@tags null,
source = MangaSource.MANGATOWN,
)
}.orEmpty(),
url = href,
isNsfw = false,
publicUrl = href.toAbsoluteUrl(a.host ?: getDomain()),
)
}
}
tags.isNullOrEmpty() -> "/directory/$page.htm$sortKey".toAbsoluteUrl(domain)
tags.size == 1 -> "/directory/${tags.first().key}/$page.htm$sortKey".toAbsoluteUrl(domain)
else -> tags.joinToString(
prefix = "/search?page=$page".toAbsoluteUrl(domain),
) { tag ->
"&genres[${tag.key}]=1"
}
}
val doc = webClient.httpGet(url).parseHtml()
val root = doc.body().selectFirstOrThrow("ul.manga_pic_list")
return root.select("li").mapNotNull { li ->
val a = li.selectFirst("a.manga_cover")
val href = a?.attrAsRelativeUrlOrNull("href")
?: return@mapNotNull null
val views = li.select("p.view")
val status = views.firstNotNullOfOrNull { it.ownText().takeIf { x -> x.startsWith("Status:") } }
?.substringAfter(':')?.trim()?.lowercase(Locale.ROOT)
Manga(
id = generateUid(href),
title = a.attr("title"),
coverUrl = a.selectFirst("img")?.absUrl("src").orEmpty(),
source = MangaSource.MANGATOWN,
altTitle = null,
rating = li.selectFirst("p.score")?.selectFirst("b")
?.ownText()?.toFloatOrNull()?.div(5f) ?: RATING_UNKNOWN,
author = views.firstNotNullOfOrNull { it.text().takeIf { x -> x.startsWith("Author:") } }
?.substringAfter(':')
?.trim(),
state = when (status) {
"ongoing" -> MangaState.ONGOING
"completed" -> MangaState.FINISHED
else -> null
},
tags = li.selectFirst("p.keyWord")?.select("a")?.mapNotNullToSet tags@{ x ->
MangaTag(
title = x.attr("title").toTitleCase(),
key = x.attr("href").parseTagKey() ?: return@tags null,
source = MangaSource.MANGATOWN,
)
}.orEmpty(),
url = href,
isNsfw = false,
publicUrl = href.toAbsoluteUrl(a.host ?: domain),
)
}
}
override suspend fun getDetails(manga: Manga): Manga {
val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml()
val root = doc.body().selectFirstOrThrow("section.main")
.selectFirstOrThrow("div.article_content")
val info = root.selectFirst("div.detail_info")?.selectFirst("ul")
val chaptersList = root.selectFirst("div.chapter_content")
?.selectFirst("ul.chapter_list")?.select("li")?.asReversed()
val dateFormat = SimpleDateFormat("MMM dd,yyyy", Locale.US)
return manga.copy(
tags = manga.tags + info?.select("li")?.find { x ->
x.selectFirst("b")?.ownText() == "Genre(s):"
}?.select("a")?.mapNotNull { a ->
MangaTag(
title = a.attr("title").toTitleCase(),
key = a.attr("href").parseTagKey() ?: return@mapNotNull null,
source = MangaSource.MANGATOWN,
)
}.orEmpty(),
description = info?.getElementById("show")?.ownText(),
chapters = chaptersList?.mapChapters { i, li ->
val href = li.selectFirst("a")?.attrAsRelativeUrlOrNull("href")
?: return@mapChapters null
val name = li.select("span")
.filter { x -> x.className().isEmpty() }
.joinToString(" - ") { it.text() }.trim()
MangaChapter(
id = generateUid(href),
url = href,
source = MangaSource.MANGATOWN,
number = i + 1,
uploadDate = parseChapterDate(
dateFormat,
li.selectFirst("span.time")?.text(),
),
name = name.ifEmpty { "${manga.title} - ${i + 1}" },
scanlator = null,
branch = null,
)
} ?: bypassLicensedChapters(manga),
)
}
override suspend fun getDetails(manga: Manga): Manga {
val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml()
val root = doc.body().selectFirstOrThrow("section.main")
.selectFirstOrThrow("div.article_content")
val info = root.selectFirst("div.detail_info")?.selectFirst("ul")
val chaptersList = root.selectFirst("div.chapter_content")
?.selectFirst("ul.chapter_list")?.select("li")?.asReversed()
val dateFormat = SimpleDateFormat("MMM dd,yyyy", Locale.US)
return manga.copy(
tags = manga.tags + info?.select("li")?.find { x ->
x.selectFirst("b")?.ownText() == "Genre(s):"
}?.select("a")?.mapNotNull { a ->
MangaTag(
title = a.attr("title").toTitleCase(),
key = a.attr("href").parseTagKey() ?: return@mapNotNull null,
source = MangaSource.MANGATOWN,
)
}.orEmpty(),
description = info?.getElementById("show")?.ownText(),
chapters = chaptersList?.mapChapters { i, li ->
val href = li.selectFirst("a")?.attrAsRelativeUrlOrNull("href")
?: return@mapChapters null
val name = li.select("span")
.filter { x -> x.className().isEmpty() }
.joinToString(" - ") { it.text() }.trim()
MangaChapter(
id = generateUid(href),
url = href,
source = MangaSource.MANGATOWN,
number = i + 1,
uploadDate = parseChapterDate(
dateFormat,
li.selectFirst("span.time")?.text(),
),
name = name.ifEmpty { "${manga.title} - ${i + 1}" },
scanlator = null,
branch = null,
)
} ?: bypassLicensedChapters(manga),
)
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(getDomain())
val doc = context.httpGet(fullUrl).parseHtml()
val root = doc.body().selectFirstOrThrow("div.page_select")
return root.selectFirstOrThrow("select").selectOrThrow("option").mapNotNull {
val href = it.attrAsRelativeUrlOrNull("value")
if (href == null || href.endsWith("featured.html")) {
return@mapNotNull null
}
MangaPage(
id = generateUid(href),
url = href,
preview = null,
referer = fullUrl,
source = MangaSource.MANGATOWN,
)
}
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(domain)
val doc = webClient.httpGet(fullUrl).parseHtml()
val root = doc.body().selectFirstOrThrow("div.page_select")
return root.selectFirstOrThrow("select").selectOrThrow("option").mapNotNull {
val href = it.attrAsRelativeUrlOrNull("value")
if (href == null || href.endsWith("featured.html")) {
return@mapNotNull null
}
MangaPage(
id = generateUid(href),
url = href,
preview = null,
referer = fullUrl,
source = MangaSource.MANGATOWN,
)
}
}
override suspend fun getPageUrl(page: MangaPage): String {
val doc = context.httpGet(page.url.toAbsoluteUrl(getDomain())).parseHtml()
return doc.requireElementById("image").attrAsAbsoluteUrl("src")
}
override suspend fun getPageUrl(page: MangaPage): String {
val doc = webClient.httpGet(page.url.toAbsoluteUrl(domain)).parseHtml()
return doc.requireElementById("image").attrAsAbsoluteUrl("src")
}
override suspend fun getTags(): Set<MangaTag> {
val doc = context.httpGet("/directory/".toAbsoluteUrl(getDomain())).parseHtml()
val root = doc.body().selectFirst("aside.right")
?.getElementsContainingOwnText("Genres")
?.first()
?.nextElementSibling() ?: doc.parseFailed("Root not found")
return root.select("li").mapNotNullToSet { li ->
val a = li.selectFirst("a") ?: return@mapNotNullToSet null
val key = a.attr("href").parseTagKey()
if (key.isNullOrEmpty()) {
return@mapNotNullToSet null
}
MangaTag(
source = MangaSource.MANGATOWN,
key = key,
title = a.text().toTitleCase(),
)
}
}
override suspend fun getTags(): Set<MangaTag> {
val doc = webClient.httpGet("/directory/".toAbsoluteUrl(domain)).parseHtml()
val root = doc.body().selectFirst("aside.right")
?.getElementsContainingOwnText("Genres")
?.first()
?.nextElementSibling() ?: doc.parseFailed("Root not found")
return root.select("li").mapNotNullToSet { li ->
val a = li.selectFirst("a") ?: return@mapNotNullToSet null
val key = a.attr("href").parseTagKey()
if (key.isNullOrEmpty()) {
return@mapNotNullToSet null
}
MangaTag(
source = MangaSource.MANGATOWN,
key = key,
title = a.text().toTitleCase(),
)
}
}
private fun parseChapterDate(dateFormat: DateFormat, date: String?): Long {
return when {
date.isNullOrEmpty() -> 0L
date.contains("Today") -> Calendar.getInstance().timeInMillis
date.contains("Yesterday") -> Calendar.getInstance().apply { add(Calendar.DAY_OF_MONTH, -1) }.timeInMillis
else -> dateFormat.tryParse(date)
}
}
private fun parseChapterDate(dateFormat: DateFormat, date: String?): Long {
return when {
date.isNullOrEmpty() -> 0L
date.contains("Today") -> Calendar.getInstance().timeInMillis
date.contains("Yesterday") -> Calendar.getInstance().apply { add(Calendar.DAY_OF_MONTH, -1) }.timeInMillis
else -> dateFormat.tryParse(date)
}
}
private suspend fun bypassLicensedChapters(manga: Manga): List<MangaChapter> {
val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain("m"))).parseHtml()
val list = doc.body().selectFirst("ul.detail-ch-list") ?: return emptyList()
val dateFormat = SimpleDateFormat("MMM dd,yyyy", Locale.US)
return list.select("li").asReversed().mapIndexedNotNull { i, li ->
val a = li.selectFirst("a") ?: return@mapIndexedNotNull null
val href = a.attrAsRelativeUrl("href")
val name = a.selectFirst("span.vol")?.text().orEmpty().ifEmpty {
a.ownText()
}
MangaChapter(
id = generateUid(href),
url = href,
source = MangaSource.MANGATOWN,
number = i + 1,
uploadDate = parseChapterDate(
dateFormat,
li.selectFirst("span.time")?.text(),
),
name = name.ifEmpty { "${manga.title} - ${i + 1}" },
scanlator = null,
branch = null,
)
}
}
private suspend fun bypassLicensedChapters(manga: Manga): List<MangaChapter> {
val doc = webClient.httpGet(manga.url.toAbsoluteUrl(getDomain("m"))).parseHtml()
val list = doc.body().selectFirst("ul.detail-ch-list") ?: return emptyList()
val dateFormat = SimpleDateFormat("MMM dd,yyyy", Locale.US)
return list.select("li").asReversed().mapIndexedNotNull { i, li ->
val a = li.selectFirst("a") ?: return@mapIndexedNotNull null
val href = a.attrAsRelativeUrl("href")
val name = a.selectFirst("span.vol")?.text().orEmpty().ifEmpty {
a.ownText()
}
MangaChapter(
id = generateUid(href),
url = href,
source = MangaSource.MANGATOWN,
number = i + 1,
uploadDate = parseChapterDate(
dateFormat,
li.selectFirst("span.time")?.text(),
),
name = name.ifEmpty { "${manga.title} - ${i + 1}" },
scanlator = null,
branch = null,
)
}
}
private fun String.parseTagKey() = split('/').findLast { regexTag matches it }
}
private fun String.parseTagKey() = split('/').findLast { regexTag matches it }
}

318
src/main/kotlin/org/koitharu/kotatsu/parsers/site/Manhwa18Parser.kt
Unescape Escape View File

@ -9,173 +9,177 @@ import org.koitharu.kotatsu.parsers.util.*
import java.util.*
@MangaSourceParser("MANHWA18", "Manhwa18", "en")
class Manhwa18Parser(override val context: MangaLoaderContext) : PagedMangaParser(MangaSource.MANHWA18, pageSize = 20, searchPageSize = 20) {
class Manhwa18Parser(context: MangaLoaderContext) :
PagedMangaParser(context, MangaSource.MANHWA18, pageSize = 20, searchPageSize = 20) {
override val configKeyDomain: ConfigKey.Domain
get() = ConfigKey.Domain("manhwa18.net", null)
override val configKeyDomain: ConfigKey.Domain
get() = ConfigKey.Domain("manhwa18.net", null)
override val sortOrders: Set<SortOrder>
get() = EnumSet.of(SortOrder.UPDATED, SortOrder.POPULARITY, SortOrder.ALPHABETICAL)
override val sortOrders: Set<SortOrder>
get() = EnumSet.of(SortOrder.UPDATED, SortOrder.POPULARITY, SortOrder.ALPHABETICAL)
override suspend fun getFavicons(): Favicons {
return Favicons(listOf(
Favicon("https://${getDomain()}/uploads/logos/logo-mini.png", 92, null)
), getDomain())
}
override suspend fun getFavicons(): Favicons {
return Favicons(
listOf(
Favicon("https://${domain}/uploads/logos/logo-mini.png", 92, null),
),
domain,
)
}
override suspend fun getDetails(manga: Manga): Manga {
val docs = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml()
val cardInfoElement = docs.selectFirst(".card .manga-info")
val author = cardInfoElement?.selectFirst("b:contains(Author(s))")?.parent()
?.select("a.btn")
?.joinToString(", ") { it.text() }
val tags = cardInfoElement?.selectFirst("b:contains(Genre(s))")?.parent()
?.select("a.btn")
?.mapToSet { MangaTag(it.text(), it.text().lowercase(), MangaSource.MANHWA18) }
val state = cardInfoElement?.selectFirst("b:contains(Status)")?.parent()
?.selectFirst("a.btn")
?.let {
when (it.text()) {
"On going" -> MangaState.ONGOING
"Completed" -> MangaState.FINISHED
else -> null
}
}
override suspend fun getDetails(manga: Manga): Manga {
val docs = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml()
val cardInfoElement = docs.selectFirst(".card .manga-info")
val author = cardInfoElement?.selectFirst("b:contains(Author(s))")?.parent()
?.select("a.btn")
?.joinToString(", ") { it.text() }
val tags = cardInfoElement?.selectFirst("b:contains(Genre(s))")?.parent()
?.select("a.btn")
?.mapToSet { MangaTag(it.text(), it.text().lowercase(), MangaSource.MANHWA18) }
val state = cardInfoElement?.selectFirst("b:contains(Status)")?.parent()
?.selectFirst("a.btn")
?.let {
when (it.text()) {
"On going" -> MangaState.ONGOING
"Completed" -> MangaState.FINISHED
else -> null
}
}
return manga.copy(
altTitle = cardInfoElement?.selectFirst("b:contains(Other names)")?.parent()?.ownText()?.removePrefix(": "),
author = author,
description = docs.selectFirst(".series-summary .summary-content")?.html(),
tags = tags.orEmpty(),
state = state,
chapters = docs.select(".card-body > .list-chapters > a").asReversed().mapChapters { index, element ->
// attrAsRelativeUrl only return page url without the '/'
val chapterUrl = element.attrAsAbsoluteUrlOrNull("href")?.toRelativeUrl(getDomain())
?: return@mapChapters null
val uploadDate = parseUploadDate(element.selectFirst(".chapter-time")?.text())
MangaChapter(
id = generateUid(chapterUrl),
name = element.selectFirst(".chapter-name")?.text().orEmpty(),
number = index + 1,
url = chapterUrl,
scanlator = null,
uploadDate = uploadDate,
branch = null,
source = MangaSource.MANHWA18,
)
}
)
}
return manga.copy(
altTitle = cardInfoElement?.selectFirst("b:contains(Other names)")?.parent()?.ownText()?.removePrefix(": "),
author = author,
description = docs.selectFirst(".series-summary .summary-content")?.html(),
tags = tags.orEmpty(),
state = state,
chapters = docs.select(".card-body > .list-chapters > a").asReversed().mapChapters { index, element ->
// attrAsRelativeUrl only return page url without the '/'
val chapterUrl = element.attrAsAbsoluteUrlOrNull("href")?.toRelativeUrl(domain)
?: return@mapChapters null
val uploadDate = parseUploadDate(element.selectFirst(".chapter-time")?.text())
MangaChapter(
id = generateUid(chapterUrl),
name = element.selectFirst(".chapter-name")?.text().orEmpty(),
number = index + 1,
url = chapterUrl,
scanlator = null,
uploadDate = uploadDate,
branch = null,
source = MangaSource.MANHWA18,
)
},
)
}
// 7 minutes ago
// 5 hours ago
// 2 days ago
// 2 weeks ago
// 4 years ago
private fun parseUploadDate(timeStr: String?): Long {
timeStr ?: return 0
// 7 minutes ago
// 5 hours ago
// 2 days ago
// 2 weeks ago
// 4 years ago
private fun parseUploadDate(timeStr: String?): Long {
timeStr ?: return 0
val timeWords = timeStr.split(' ')
if (timeWords.size != 3) return 0
val timeWord = timeWords[1]
val timeAmount = timeWords[0].toIntOrNull() ?: return 0
val timeUnit = when (timeWord) {
"minute", "minutes" -> Calendar.MINUTE
"hour", "hours" -> Calendar.HOUR
"day", "days" -> Calendar.DAY_OF_YEAR
"week", "weeks" -> Calendar.WEEK_OF_YEAR
"month", "months" -> Calendar.MONTH
"year", "years" -> Calendar.YEAR
else -> return 0
}
val cal = Calendar.getInstance()
cal.add(timeUnit, -timeAmount)
return cal.time.time
}
val timeWords = timeStr.split(' ')
if (timeWords.size != 3) return 0
val timeWord = timeWords[1]
val timeAmount = timeWords[0].toIntOrNull() ?: return 0
val timeUnit = when (timeWord) {
"minute", "minutes" -> Calendar.MINUTE
"hour", "hours" -> Calendar.HOUR
"day", "days" -> Calendar.DAY_OF_YEAR
"week", "weeks" -> Calendar.WEEK_OF_YEAR
"month", "months" -> Calendar.MONTH
"year", "years" -> Calendar.YEAR
else -> return 0
}
val cal = Calendar.getInstance()
cal.add(timeUnit, -timeAmount)
return cal.time.time
}
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val sortQuery = when(sortOrder) {
SortOrder.ALPHABETICAL -> "name"
SortOrder.POPULARITY -> "views"
SortOrder.UPDATED -> "last_update"
else -> ""
}
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val sortQuery = when (sortOrder) {
SortOrder.ALPHABETICAL -> "name"
SortOrder.POPULARITY -> "views"
SortOrder.UPDATED -> "last_update"
else -> ""
}
val sortType = if (sortOrder == SortOrder.ALPHABETICAL) "ASC" else "DESC"
val tagQuery = tags?.joinToString(",") { it.key }.orEmpty()
val url = buildString {
append("https://")
append(getDomain())
append("/manga-list.html?listType=pagination&page=")
append(page)
append("&artist=&author=&group=&m_status=&name=")
append(query?.urlEncoded().orEmpty())
append("&genre=$tagQuery")
append("&ungenre=")
append("&sort=")
append(sortQuery)
append("&sort_type=")
append(sortType)
}
val sortType = if (sortOrder == SortOrder.ALPHABETICAL) "ASC" else "DESC"
val tagQuery = tags?.joinToString(",") { it.key }.orEmpty()
val url = buildString {
append("https://")
append(domain)
append("/manga-list.html?listType=pagination&page=")
append(page)
append("&artist=&author=&group=&m_status=&name=")
append(query?.urlEncoded().orEmpty())
append("&genre=$tagQuery")
append("&ungenre=")
append("&sort=")
append(sortQuery)
append("&sort_type=")
append(sortType)
}
val docs = context.httpGet(url).parseHtml()
val actualPage = docs.selectFirst("ul.pagination a.active")?.text()?.toIntOrNull()
if (actualPage != page) {
return emptyList()
}
val docs = webClient.httpGet(url).parseHtml()
val actualPage = docs.selectFirst("ul.pagination a.active")?.text()?.toIntOrNull()
if (actualPage != page) {
return emptyList()
}
return docs.select(".card-body .thumb-item-flow")
.map {
val titleElement = it.selectFirstOrThrow(".thumb_attr.series-title > a")
val absUrl = titleElement.attrAsAbsoluteUrl("href")
Manga(
id = generateUid(absUrl.toRelativeUrl(getDomain())),
title = titleElement.text(),
altTitle = null,
url = absUrl.toRelativeUrl(getDomain()),
publicUrl = absUrl,
rating = RATING_UNKNOWN,
isNsfw = true,
coverUrl = it.selectFirst("div.img-in-ratio")?.attrAsAbsoluteUrl("data-bg").orEmpty(),
tags = emptySet(),
state = null,
author = null,
largeCoverUrl = null,
description = null,
source = MangaSource.MANHWA18,
)
}
}
return docs.select(".card-body .thumb-item-flow")
.map {
val titleElement = it.selectFirstOrThrow(".thumb_attr.series-title > a")
val absUrl = titleElement.attrAsAbsoluteUrl("href")
Manga(
id = generateUid(absUrl.toRelativeUrl(domain)),
title = titleElement.text(),
altTitle = null,
url = absUrl.toRelativeUrl(domain),
publicUrl = absUrl,
rating = RATING_UNKNOWN,
isNsfw = true,
coverUrl = it.selectFirst("div.img-in-ratio")?.attrAsAbsoluteUrl("data-bg").orEmpty(),
tags = emptySet(),
state = null,
author = null,
largeCoverUrl = null,
description = null,
source = MangaSource.MANHWA18,
)
}
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val chapterUrl = chapter.url.toAbsoluteUrl(getDomain())
return context.httpGet(chapterUrl).parseHtml()
.select("div.chapter-content > img").map {
val url = it.attrAsRelativeUrlOrNull("src").orEmpty()
MangaPage(
id = generateUid(url),
url = url,
referer = chapterUrl,
preview = null,
source = MangaSource.MANHWA18,
)
}
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val chapterUrl = chapter.url.toAbsoluteUrl(domain)
return webClient.httpGet(chapterUrl).parseHtml()
.select("div.chapter-content > img").map {
val url = it.attrAsRelativeUrlOrNull("src").orEmpty()
MangaPage(
id = generateUid(url),
url = url,
referer = chapterUrl,
preview = null,
source = MangaSource.MANHWA18,
)
}
}
override suspend fun getTags(): Set<MangaTag> {
return context.httpGet("https://${getDomain()}/").parseHtml().selectFirstOrThrow(".genres-menu")
.select("a.genres-item").orEmpty()
.mapToSet {
MangaTag(
title = it.text(),
key = it.text().lowercase(),
source = MangaSource.MANHWA18,
)
}
}
}
override suspend fun getTags(): Set<MangaTag> {
return webClient.httpGet("https://${domain}/").parseHtml().selectFirstOrThrow(".genres-menu")
.select("a.genres-item").orEmpty()
.mapToSet {
MangaTag(
title = it.text(),
key = it.text().lowercase(),
source = MangaSource.MANHWA18,
)
}
}
}

322
src/main/kotlin/org/koitharu/kotatsu/parsers/site/NHentaiParser.kt
Unescape Escape View File

@ -15,174 +15,174 @@ import java.text.SimpleDateFormat
import java.util.*
@MangaSourceParser("NHENTAI", "N-Hentai")
class NHentaiParser(override val context: MangaLoaderContext) : PagedMangaParser(MangaSource.NHENTAI, pageSize = 25) {
class NHentaiParser(context: MangaLoaderContext) : PagedMangaParser(context, MangaSource.NHENTAI, pageSize = 25) {
override val configKeyDomain: ConfigKey.Domain
get() = ConfigKey.Domain("nhentai.net", null)
override val configKeyDomain: ConfigKey.Domain
get() = ConfigKey.Domain("nhentai.net", null)
override val sortOrders: Set<SortOrder>
get() = EnumSet.of(SortOrder.NEWEST, SortOrder.POPULARITY)
override val sortOrders: Set<SortOrder>
get() = EnumSet.of(SortOrder.NEWEST, SortOrder.POPULARITY)
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
if (query.isNullOrEmpty() && tags != null && tags.size > 1) {
return getListPage(page, buildQuery(tags), emptySet(), sortOrder)
}
val domain = getDomain()
val url = buildString {
append("https://")
append(domain)
if (!query.isNullOrEmpty()) {
append("/search/?q=")
append(query.urlEncoded())
append("&page=")
append(page)
if (sortOrder == SortOrder.POPULARITY) {
append("&sort=popular")
}
} else {
append('/')
if (!tags.isNullOrEmpty()) {
val tag = tags.single()
append("tag/")
append(tag.key)
append('/')
if (sortOrder == SortOrder.POPULARITY) {
append("popular")
}
append("?page=")
append(page)
} else {
if (sortOrder == SortOrder.POPULARITY) {
append("?sort=popular&page=")
} else {
append("?page=")
}
append(page)
}
}
}
val root = context.httpGet(url).parseHtml().body().requireElementById("content")
.selectLastOrThrow("div.index-container")
val regexBrackets = Regex("\\[[^]]+]|\\([^)]+\\)")
val regexSpaces = Regex("\\s+")
return root.select(".gallery").map { div ->
val a = div.selectFirstOrThrow("a.cover")
val href = a.attrAsRelativeUrl("href")
val img = div.selectFirstOrThrow("img")
val title = div.selectFirstOrThrow(".caption").text()
Manga(
id = generateUid(href),
title = title.replace(regexBrackets, "")
.replace(regexSpaces, " ")
.trim(),
altTitle = null,
url = href,
publicUrl = href.toAbsoluteUrl(domain),
rating = RATING_UNKNOWN,
isNsfw = true,
coverUrl = img.attrAsAbsoluteUrlOrNull("data-src")
?: img.attrAsAbsoluteUrl("src"),
tags = setOf(),
state = null,
author = null,
largeCoverUrl = null,
description = null,
chapters = listOf(),
source = source,
)
}
}
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
if (query.isNullOrEmpty() && tags != null && tags.size > 1) {
return getListPage(page, buildQuery(tags), emptySet(), sortOrder)
}
val domain = domain
val url = buildString {
append("https://")
append(domain)
if (!query.isNullOrEmpty()) {
append("/search/?q=")
append(query.urlEncoded())
append("&page=")
append(page)
if (sortOrder == SortOrder.POPULARITY) {
append("&sort=popular")
}
} else {
append('/')
if (!tags.isNullOrEmpty()) {
val tag = tags.single()
append("tag/")
append(tag.key)
append('/')
if (sortOrder == SortOrder.POPULARITY) {
append("popular")
}
append("?page=")
append(page)
} else {
if (sortOrder == SortOrder.POPULARITY) {
append("?sort=popular&page=")
} else {
append("?page=")
}
append(page)
}
}
}
val root = webClient.httpGet(url).parseHtml().body().requireElementById("content")
.selectLastOrThrow("div.index-container")
val regexBrackets = Regex("\\[[^]]+]|\\([^)]+\\)")
val regexSpaces = Regex("\\s+")
return root.select(".gallery").map { div ->
val a = div.selectFirstOrThrow("a.cover")
val href = a.attrAsRelativeUrl("href")
val img = div.selectFirstOrThrow("img")
val title = div.selectFirstOrThrow(".caption").text()
Manga(
id = generateUid(href),
title = title.replace(regexBrackets, "")
.replace(regexSpaces, " ")
.trim(),
altTitle = null,
url = href,
publicUrl = href.toAbsoluteUrl(domain),
rating = RATING_UNKNOWN,
isNsfw = true,
coverUrl = img.attrAsAbsoluteUrlOrNull("data-src")
?: img.attrAsAbsoluteUrl("src"),
tags = setOf(),
state = null,
author = null,
largeCoverUrl = null,
description = null,
chapters = listOf(),
source = source,
)
}
}
override suspend fun getDetails(manga: Manga): Manga {
val root = context.httpGet(
url = manga.url.toAbsoluteUrl(getDomain()),
).parseHtml().body().requireElementById("bigcontainer")
val img = root.requireElementById("cover").selectFirstOrThrow("img")
val tagContainers = root.requireElementById("tags").select(".tag-container")
val dateFormat = SimpleDateFormat(
"yyyy-MM-dd'T'HH:mm:ss.SSSSSS'+00:00'",
Locale.ROOT,
)
return manga.copy(
tags = tagContainers.find { x -> x.ownText() == "Tags:" }?.parseTags() ?: manga.tags,
author = tagContainers.find { x -> x.ownText() == "Artists:" }
?.selectFirst("span.name")?.text()?.toCamelCase(),
largeCoverUrl = img.attrAsAbsoluteUrlOrNull("data-src")
?: img.attrAsAbsoluteUrl("src"),
description = null,
chapters = listOf(
MangaChapter(
id = manga.id,
name = manga.title,
number = 1,
url = manga.url,
scanlator = null,
uploadDate = dateFormat.tryParse(
tagContainers.find { x -> x.ownText() == "Uploaded:" }
?.selectFirst("time")
?.attr("datetime"),
),
branch = null,
source = source,
),
),
)
}
override suspend fun getDetails(manga: Manga): Manga {
val root = webClient.httpGet(
url = manga.url.toAbsoluteUrl(domain),
).parseHtml().body().requireElementById("bigcontainer")
val img = root.requireElementById("cover").selectFirstOrThrow("img")
val tagContainers = root.requireElementById("tags").select(".tag-container")
val dateFormat = SimpleDateFormat(
"yyyy-MM-dd'T'HH:mm:ss.SSSSSS'+00:00'",
Locale.ROOT,
)
return manga.copy(
tags = tagContainers.find { x -> x.ownText() == "Tags:" }?.parseTags() ?: manga.tags,
author = tagContainers.find { x -> x.ownText() == "Artists:" }
?.selectFirst("span.name")?.text()?.toCamelCase(),
largeCoverUrl = img.attrAsAbsoluteUrlOrNull("data-src")
?: img.attrAsAbsoluteUrl("src"),
description = null,
chapters = listOf(
MangaChapter(
id = manga.id,
name = manga.title,
number = 1,
url = manga.url,
scanlator = null,
uploadDate = dateFormat.tryParse(
tagContainers.find { x -> x.ownText() == "Uploaded:" }
?.selectFirst("time")
?.attr("datetime"),
),
branch = null,
source = source,
),
),
)
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val url = chapter.url.toAbsoluteUrl(getDomain())
val root = context.httpGet(url).parseHtml().requireElementById("thumbnail-container")
return root.select(".thumb-container").map { div ->
val a = div.selectFirstOrThrow("a")
val img = div.selectFirstOrThrow("img")
val href = a.attrAsRelativeUrl("href")
MangaPage(
id = generateUid(href),
url = href,
referer = url,
preview = img.attrAsAbsoluteUrlOrNull("data-src")
?: img.attrAsAbsoluteUrl("src"),
source = source,
)
}
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val url = chapter.url.toAbsoluteUrl(domain)
val root = webClient.httpGet(url).parseHtml().requireElementById("thumbnail-container")
return root.select(".thumb-container").map { div ->
val a = div.selectFirstOrThrow("a")
val img = div.selectFirstOrThrow("img")
val href = a.attrAsRelativeUrl("href")
MangaPage(
id = generateUid(href),
url = href,
referer = url,
preview = img.attrAsAbsoluteUrlOrNull("data-src")
?: img.attrAsAbsoluteUrl("src"),
source = source,
)
}
}
override suspend fun getPageUrl(page: MangaPage): String {
val root = context.httpGet(page.url.toAbsoluteUrl(getDomain())).parseHtml().body()
.requireElementById("image-container")
return root.selectFirstOrThrow("img").attrAsAbsoluteUrl("src")
}
override suspend fun getPageUrl(page: MangaPage): String {
val root = webClient.httpGet(page.url.toAbsoluteUrl(domain)).parseHtml().body()
.requireElementById("image-container")
return root.selectFirstOrThrow("img").attrAsAbsoluteUrl("src")
}
override suspend fun getTags(): Set<MangaTag> {
return coroutineScope {
// parse first 3 pages of tags
(1..3).map { page ->
async { getTags(page) }
}
}.awaitAll().flattenTo(ArraySet(360))
}
override suspend fun getTags(): Set<MangaTag> {
return coroutineScope {
// parse first 3 pages of tags
(1..3).map { page ->
async { getTags(page) }
}
}.awaitAll().flattenTo(ArraySet(360))
}
private suspend fun getTags(page: Int): Set<MangaTag> {
val root = context.httpGet("https://${getDomain()}/tags/popular?page=$page").parseHtml().body()
.getElementById("tag-container")
return root?.parseTags().orEmpty()
}
private suspend fun getTags(page: Int): Set<MangaTag> {
val root = webClient.httpGet("https://${domain}/tags/popular?page=$page").parseHtml().body()
.getElementById("tag-container")
return root?.parseTags().orEmpty()
}
private fun Element.parseTags() = select("a.tag").mapToSet { a ->
val href = a.attr("href").removeSuffix('/')
MangaTag(
title = a.selectFirstOrThrow(".name").text().toTitleCase(),
key = href.substringAfterLast('/'),
source = source,
)
}
private fun Element.parseTags() = select("a.tag").mapToSet { a ->
val href = a.attr("href").removeSuffix('/')
MangaTag(
title = a.selectFirstOrThrow(".name").text().toTitleCase(),
key = href.substringAfterLast('/'),
source = source,
)
}
private fun buildQuery(tags: Collection<MangaTag>) = tags.joinToString(separator = " ") { tag ->
"tag:\"${tag.key}\""
}
}
private fun buildQuery(tags: Collection<MangaTag>) = tags.joinToString(separator = " ") { tag ->
"tag:\"${tag.key}\""
}
}

412
src/main/kotlin/org/koitharu/kotatsu/parsers/site/NetTruyenParser.kt
Unescape Escape View File

@ -15,210 +15,210 @@ import java.text.SimpleDateFormat
import java.util.*
@MangaSourceParser("NETTRUYEN", "NetTruyen", "vi")
class NetTruyenParser(override val context: MangaLoaderContext) :
PagedMangaParser(MangaSource.NETTRUYEN, pageSize = 36) {
override val configKeyDomain: ConfigKey.Domain
get() = ConfigKey.Domain("nettruyenin.com", null)
override val sortOrders: Set<SortOrder>
get() = EnumSet.of(SortOrder.UPDATED, SortOrder.POPULARITY, SortOrder.NEWEST, SortOrder.RATING)
private val mutex = Mutex()
private val dateFormat = SimpleDateFormat("dd/MM/yy", Locale.US)
private var tagCache: ArrayMap<String, MangaTag>? = null
override suspend fun getDetails(manga: Manga): Manga {
val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml()
val rating = doc.selectFirst("span[itemprop=ratingValue]")
?.ownText()
?.toFloatOrNull() ?: 0f
val chapterElements = doc.getElementById("nt_listchapter")?.select("ul > li") ?: doc.parseFailed()
val chapters = chapterElements.asReversed().mapChapters { index, element ->
val a = element.selectFirst("div.chapter > a") ?: return@mapChapters null
val relativeUrl = a.attrAsRelativeUrlOrNull("href") ?: return@mapChapters null
val timeText = element.selectFirst("div.col-xs-4.text-center.no-wrap.small")?.text()
MangaChapter(
id = generateUid(relativeUrl),
name = a.text(),
number = index + 1,
url = relativeUrl,
scanlator = null,
uploadDate = parseChapterTime(timeText),
branch = null,
source = source,
)
}
return manga.copy(
rating = rating / 5,
chapters = chapters,
description = doc.selectFirst("div.detail-content > p")?.html(),
isNsfw = doc.selectFirst("div.alert.alert-danger > strong:contains(Cảnh báo độ tuổi)") != null,
)
}
// 20 giây trước
// 52 phút trước
// 6 giờ trước
// 2 ngày trước
// 19:09 30/07
// 23/12/21
private fun parseChapterTime(timeText: String?): Long {
if (timeText.isNullOrEmpty()) {
return 0L
}
val timeWords = arrayOf("giây", "phút", "giờ", "ngày")
val calendar = Calendar.getInstance()
val timeArr = timeText.split(' ')
if (WordSet(*timeWords).anyWordIn(timeText)) {
val timeSuffix = timeArr.getOrNull(1)
val timeDiff = timeArr.getOrNull(0)?.toIntOrNull() ?: return 0L
when (timeSuffix) {
timeWords[0] -> calendar.add(Calendar.SECOND, -timeDiff)
timeWords[1] -> calendar.add(Calendar.MINUTE, -timeDiff)
timeWords[2] -> calendar.add(Calendar.HOUR, -timeDiff)
timeWords[3] -> calendar.add(Calendar.DATE, -timeDiff)
else -> return 0L
}
} else {
val relativeDate = timeArr.lastOrNull() ?: return 0L
val dateString = when (relativeDate.split('/').size) {
2 -> {
val currentYear = calendar.get(Calendar.YEAR).toString().takeLast(2)
"$relativeDate/$currentYear"
}
3 -> relativeDate
else -> return 0L
}
calendar.timeInMillis = dateFormat.tryParse(dateString)
}
return calendar.time.time
}
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val isSearching = !query.isNullOrEmpty()
val url = buildString {
append("https://")
append(getDomain())
if (isSearching) {
append("/tim-truyen?keyword=")
append(query!!.urlEncoded())
append("&page=")
append(page)
} else {
val tagQuery = tags.orEmpty().joinToString(",") { it.key }
append("/tim-truyen-nang-cao?genres=$tagQuery")
append("&notgenres=&gender=-1&status=-1&minchapter=1&sort=${getSortOrderKey(sortOrder)}")
append("&page=$page")
}
}
val response = if (isSearching) {
val result = runCatchingCancellable { context.httpGet(url) }
val exception = result.exceptionOrNull()
if (exception is NotFoundException) {
return emptyList()
}
result.getOrThrow()
} else {
context.httpGet(url)
}
val itemsElements = response.parseHtml()
.select("div.ModuleContent > div.items")
.select("div.item")
return itemsElements.mapNotNull { item ->
val tooltipElement = item.selectFirst("div.box_tootip") ?: return@mapNotNull null
val absUrl = item.selectFirst("div.image > a")?.attrAsAbsoluteUrlOrNull("href") ?: return@mapNotNull null
val slug = absUrl.substringAfterLast('/')
val mangaState = when (tooltipElement.selectFirst("div.message_main > p:contains(Tình trạng)")?.ownText()) {
"Đang tiến hành" -> MangaState.ONGOING
"Hoàn thành" -> MangaState.FINISHED
else -> null
}
val tagMap = getOrCreateTagMap()
val tagsElement = tooltipElement.selectFirst("div.message_main > p:contains(Thể loại)")?.ownText().orEmpty()
val mangaTags = tagsElement.split(',').mapNotNullToSet { tagMap[it.trim()] }
Manga(
id = generateUid(slug),
title = tooltipElement.selectFirst("div.title")?.text().orEmpty(),
altTitle = null,
url = absUrl.toRelativeUrl(getDomain()),
publicUrl = absUrl,
rating = RATING_UNKNOWN,
isNsfw = false,
coverUrl = item.selectFirst("div.image a img")?.absUrl("data-original").orEmpty(),
largeCoverUrl = null,
tags = mangaTags,
state = mangaState,
author = tooltipElement.selectFirst("div.message_main > p:contains(Tác giả)")?.ownText(),
description = tooltipElement.selectFirst("div.box_text")?.text(),
chapters = null,
source = source,
)
}
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val pageElements = context.httpGet(chapter.url.toAbsoluteUrl(getDomain())).parseHtml()
.select("div.reading-detail.box_doc > div img")
return pageElements.map { element ->
val url = element.attrAsAbsoluteUrl("data-original")
MangaPage(
id = generateUid(url),
url = url,
referer = getDomain(),
preview = null,
source = source,
)
}
}
override suspend fun getTags(): Set<MangaTag> {
val map = getOrCreateTagMap()
val tagSet = ArraySet<MangaTag>(map.size)
for (entry in map) {
tagSet.add(entry.value)
}
return tagSet
}
private suspend fun getOrCreateTagMap(): ArrayMap<String, MangaTag> = mutex.withLock {
tagCache?.let { return@withLock it }
val doc = context.httpGet("/tim-truyen-nang-cao".toAbsoluteUrl(getDomain())).parseHtml()
val tagItems = doc.select("div.genre-item")
val result = ArrayMap<String, MangaTag>(tagItems.size)
for (item in tagItems) {
val title = item.text().trim()
val key = item.select("span[data-id]").attr("data-id")
result[title] = MangaTag(title = title, key = key, source = source)
}
tagCache = result
result
}
private fun getSortOrderKey(sortOrder: SortOrder) = when (sortOrder) {
SortOrder.UPDATED -> 0
SortOrder.POPULARITY -> 10
SortOrder.NEWEST -> 15
SortOrder.RATING -> 20
else -> throw IllegalArgumentException("Sort order ${sortOrder.name} not supported")
}
class NetTruyenParser(context: MangaLoaderContext) :
PagedMangaParser(context, MangaSource.NETTRUYEN, pageSize = 36) {
override val configKeyDomain: ConfigKey.Domain
get() = ConfigKey.Domain("nettruyenin.com", null)
override val sortOrders: Set<SortOrder>
get() = EnumSet.of(SortOrder.UPDATED, SortOrder.POPULARITY, SortOrder.NEWEST, SortOrder.RATING)
private val mutex = Mutex()
private val dateFormat = SimpleDateFormat("dd/MM/yy", Locale.US)
private var tagCache: ArrayMap<String, MangaTag>? = null
override suspend fun getDetails(manga: Manga): Manga {
val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml()
val rating = doc.selectFirst("span[itemprop=ratingValue]")
?.ownText()
?.toFloatOrNull() ?: 0f
val chapterElements = doc.getElementById("nt_listchapter")?.select("ul > li") ?: doc.parseFailed()
val chapters = chapterElements.asReversed().mapChapters { index, element ->
val a = element.selectFirst("div.chapter > a") ?: return@mapChapters null
val relativeUrl = a.attrAsRelativeUrlOrNull("href") ?: return@mapChapters null
val timeText = element.selectFirst("div.col-xs-4.text-center.no-wrap.small")?.text()
MangaChapter(
id = generateUid(relativeUrl),
name = a.text(),
number = index + 1,
url = relativeUrl,
scanlator = null,
uploadDate = parseChapterTime(timeText),
branch = null,
source = source,
)
}
return manga.copy(
rating = rating / 5,
chapters = chapters,
description = doc.selectFirst("div.detail-content > p")?.html(),
isNsfw = doc.selectFirst("div.alert.alert-danger > strong:contains(Cảnh báo độ tuổi)") != null,
)
}
// 20 giây trước
// 52 phút trước
// 6 giờ trước
// 2 ngày trước
// 19:09 30/07
// 23/12/21
private fun parseChapterTime(timeText: String?): Long {
if (timeText.isNullOrEmpty()) {
return 0L
}
val timeWords = arrayOf("giây", "phút", "giờ", "ngày")
val calendar = Calendar.getInstance()
val timeArr = timeText.split(' ')
if (WordSet(*timeWords).anyWordIn(timeText)) {
val timeSuffix = timeArr.getOrNull(1)
val timeDiff = timeArr.getOrNull(0)?.toIntOrNull() ?: return 0L
when (timeSuffix) {
timeWords[0] -> calendar.add(Calendar.SECOND, -timeDiff)
timeWords[1] -> calendar.add(Calendar.MINUTE, -timeDiff)
timeWords[2] -> calendar.add(Calendar.HOUR, -timeDiff)
timeWords[3] -> calendar.add(Calendar.DATE, -timeDiff)
else -> return 0L
}
} else {
val relativeDate = timeArr.lastOrNull() ?: return 0L
val dateString = when (relativeDate.split('/').size) {
2 -> {
val currentYear = calendar.get(Calendar.YEAR).toString().takeLast(2)
"$relativeDate/$currentYear"
}
3 -> relativeDate
else -> return 0L
}
calendar.timeInMillis = dateFormat.tryParse(dateString)
}
return calendar.time.time
}
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val isSearching = !query.isNullOrEmpty()
val url = buildString {
append("https://")
append(domain)
if (isSearching) {
append("/tim-truyen?keyword=")
append(query!!.urlEncoded())
append("&page=")
append(page)
} else {
val tagQuery = tags.orEmpty().joinToString(",") { it.key }
append("/tim-truyen-nang-cao?genres=$tagQuery")
append("&notgenres=&gender=-1&status=-1&minchapter=1&sort=${getSortOrderKey(sortOrder)}")
append("&page=$page")
}
}
val response = if (isSearching) {
val result = runCatchingCancellable { webClient.httpGet(url) }
val exception = result.exceptionOrNull()
if (exception is NotFoundException) {
return emptyList()
}
result.getOrThrow()
} else {
webClient.httpGet(url)
}
val itemsElements = response.parseHtml()
.select("div.ModuleContent > div.items")
.select("div.item")
return itemsElements.mapNotNull { item ->
val tooltipElement = item.selectFirst("div.box_tootip") ?: return@mapNotNull null
val absUrl = item.selectFirst("div.image > a")?.attrAsAbsoluteUrlOrNull("href") ?: return@mapNotNull null
val slug = absUrl.substringAfterLast('/')
val mangaState = when (tooltipElement.selectFirst("div.message_main > p:contains(Tình trạng)")?.ownText()) {
"Đang tiến hành" -> MangaState.ONGOING
"Hoàn thành" -> MangaState.FINISHED
else -> null
}
val tagMap = getOrCreateTagMap()
val tagsElement = tooltipElement.selectFirst("div.message_main > p:contains(Thể loại)")?.ownText().orEmpty()
val mangaTags = tagsElement.split(',').mapNotNullToSet { tagMap[it.trim()] }
Manga(
id = generateUid(slug),
title = tooltipElement.selectFirst("div.title")?.text().orEmpty(),
altTitle = null,
url = absUrl.toRelativeUrl(domain),
publicUrl = absUrl,
rating = RATING_UNKNOWN,
isNsfw = false,
coverUrl = item.selectFirst("div.image a img")?.absUrl("data-original").orEmpty(),
largeCoverUrl = null,
tags = mangaTags,
state = mangaState,
author = tooltipElement.selectFirst("div.message_main > p:contains(Tác giả)")?.ownText(),
description = tooltipElement.selectFirst("div.box_text")?.text(),
chapters = null,
source = source,
)
}
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val pageElements = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml()
.select("div.reading-detail.box_doc > div img")
return pageElements.map { element ->
val url = element.attrAsAbsoluteUrl("data-original")
MangaPage(
id = generateUid(url),
url = url,
referer = domain,
preview = null,
source = source,
)
}
}
override suspend fun getTags(): Set<MangaTag> {
val map = getOrCreateTagMap()
val tagSet = ArraySet<MangaTag>(map.size)
for (entry in map) {
tagSet.add(entry.value)
}
return tagSet
}
private suspend fun getOrCreateTagMap(): ArrayMap<String, MangaTag> = mutex.withLock {
tagCache?.let { return@withLock it }
val doc = webClient.httpGet("/tim-truyen-nang-cao".toAbsoluteUrl(domain)).parseHtml()
val tagItems = doc.select("div.genre-item")
val result = ArrayMap<String, MangaTag>(tagItems.size)
for (item in tagItems) {
val title = item.text().trim()
val key = item.select("span[data-id]").attr("data-id")
result[title] = MangaTag(title = title, key = key, source = source)
}
tagCache = result
result
}
private fun getSortOrderKey(sortOrder: SortOrder) = when (sortOrder) {
SortOrder.UPDATED -> 0
SortOrder.POPULARITY -> 10
SortOrder.NEWEST -> 15
SortOrder.RATING -> 20
else -> throw IllegalArgumentException("Sort order ${sortOrder.name} not supported")
}
}

320
src/main/kotlin/org/koitharu/kotatsu/parsers/site/NicovideoSeigaParser.kt
Unescape Escape View File

@ -11,175 +11,175 @@ private const val STATUS_ONGOING = "連載"
private const val STATUS_FINISHED = "完結"
@MangaSourceParser("NICOVIDEO_SEIGA", "Nicovideo Seiga", "ja")
class NicovideoSeigaParser(override val context: MangaLoaderContext) :
MangaParser(MangaSource.NICOVIDEO_SEIGA),
MangaParserAuthProvider {
class NicovideoSeigaParser(context: MangaLoaderContext) :
MangaParser(context, MangaSource.NICOVIDEO_SEIGA),
MangaParserAuthProvider {
override val authUrl: String
get() = "https://${getDomain("account")}/login?site=seiga"
override val authUrl: String
get() = "https://${getDomain("account")}/login?site=seiga"
override val isAuthorized: Boolean
get() = context.cookieJar.getCookies(getDomain("seiga")).any {
it.name == "user_session"
}
override val isAuthorized: Boolean
get() = context.cookieJar.getCookies(getDomain("seiga")).any {
it.name == "user_session"
}
override suspend fun getUsername(): String {
val body = context.httpGet("https://${getDomain("app")}/my/apps").parseHtml().body()
return body.selectFirst("#userinfo > div > div > strong")?.text() ?: throw AuthRequiredException(source)
}
override suspend fun getUsername(): String {
val body = webClient.httpGet("https://${getDomain("app")}/my/apps").parseHtml().body()
return body.selectFirst("#userinfo > div > div > strong")?.text() ?: throw AuthRequiredException(source)
}
override val sortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.UPDATED,
SortOrder.POPULARITY,
)
override val sortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.UPDATED,
SortOrder.POPULARITY,
)
override val configKeyDomain: ConfigKey.Domain = ConfigKey.Domain("nicovideo.jp", null)
override val configKeyDomain: ConfigKey.Domain = ConfigKey.Domain("nicovideo.jp", null)
@InternalParsersApi
override suspend fun getList(
offset: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val page = (offset / 20f).toIntUp().inc()
val domain = getDomain("seiga")
val url = when {
!query.isNullOrEmpty() -> return if (offset == 0) getSearchList(query, page) else emptyList()
tags.isNullOrEmpty() -> "https://$domain/manga/list?page=$page&sort=${getSortKey(sortOrder)}"
tags.size == 1 -> "https://$domain/manga/list?category=${tags.first().key}&page=$page" +
"&sort=${getSortKey(sortOrder)}"
@InternalParsersApi
override suspend fun getList(
offset: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val page = (offset / 20f).toIntUp().inc()
val domain = getDomain("seiga")
val url = when {
!query.isNullOrEmpty() -> return if (offset == 0) getSearchList(query, page) else emptyList()
tags.isNullOrEmpty() -> "https://$domain/manga/list?page=$page&sort=${getSortKey(sortOrder)}"
tags.size == 1 -> "https://$domain/manga/list?category=${tags.first().key}&page=$page" +
"&sort=${getSortKey(sortOrder)}"
tags.size > 1 -> throw IllegalArgumentException("This source supports only 1 category")
else -> "https://$domain/manga/list?page=$page&sort=${getSortKey(sortOrder)}"
}
val doc = context.httpGet(url).parseHtml()
val comicList = doc.body().select("#comic_list > ul > li") ?: doc.parseFailed("Container not found")
val items = comicList.select("div > .description > div > div")
return items.mapNotNull { item ->
val href =
item.selectFirst(".comic_icon > div > a")?.attrAsRelativeUrlOrNull("href") ?: return@mapNotNull null
val statusText = item.selectFirst(".mg_description_header > .mg_icon > .content_status > span")?.text()
Manga(
id = generateUid(href),
title = item.selectFirst(".mg_body > .title > a")?.text() ?: return@mapNotNull null,
coverUrl = item.selectFirst(".comic_icon > div > a > img")?.attrAsAbsoluteUrl("src").orEmpty(),
altTitle = null,
author = item.selectFirst(".mg_description_header > .mg_author > a")?.text(),
rating = RATING_UNKNOWN,
url = href,
isNsfw = false,
tags = item.getElementsByAttributeValueContaining("href", "?category=").mapToSet { a ->
MangaTag(
key = a.attr("href").substringAfterLast('='),
title = a.ownText().trim(),
source = source,
)
},
state = when (statusText) {
STATUS_ONGOING -> MangaState.ONGOING
STATUS_FINISHED -> MangaState.FINISHED
else -> null
},
publicUrl = href.toAbsoluteUrl(item.host ?: getDomain("seiga")),
source = source,
)
}
}
tags.size > 1 -> throw IllegalArgumentException("This source supports only 1 category")
else -> "https://$domain/manga/list?page=$page&sort=${getSortKey(sortOrder)}"
}
val doc = webClient.httpGet(url).parseHtml()
val comicList = doc.body().select("#comic_list > ul > li") ?: doc.parseFailed("Container not found")
val items = comicList.select("div > .description > div > div")
return items.mapNotNull { item ->
val href =
item.selectFirst(".comic_icon > div > a")?.attrAsRelativeUrlOrNull("href") ?: return@mapNotNull null
val statusText = item.selectFirst(".mg_description_header > .mg_icon > .content_status > span")?.text()
Manga(
id = generateUid(href),
title = item.selectFirst(".mg_body > .title > a")?.text() ?: return@mapNotNull null,
coverUrl = item.selectFirst(".comic_icon > div > a > img")?.attrAsAbsoluteUrl("src").orEmpty(),
altTitle = null,
author = item.selectFirst(".mg_description_header > .mg_author > a")?.text(),
rating = RATING_UNKNOWN,
url = href,
isNsfw = false,
tags = item.getElementsByAttributeValueContaining("href", "?category=").mapToSet { a ->
MangaTag(
key = a.attr("href").substringAfterLast('='),
title = a.ownText().trim(),
source = source,
)
},
state = when (statusText) {
STATUS_ONGOING -> MangaState.ONGOING
STATUS_FINISHED -> MangaState.FINISHED
else -> null
},
publicUrl = href.toAbsoluteUrl(item.host ?: getDomain("seiga")),
source = source,
)
}
}
override suspend fun getDetails(manga: Manga): Manga {
val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain("seiga"))).parseHtml()
val contents = doc.body().selectFirstOrThrow("#contents")
val statusText = contents
.select("div.mg_work_detail > div > div:nth-child(2) > div.tip.content_status.status_series > span")
.text()
return manga.copy(
description = contents.selectFirst("div.mg_work_detail > div > div.row > div.description_text")?.html(),
largeCoverUrl = contents.selectFirst("div.primaries > div.main_visual > a > img")
?.attrAsAbsoluteUrlOrNull("src"),
state = when (statusText) {
STATUS_ONGOING -> MangaState.ONGOING
STATUS_FINISHED -> MangaState.FINISHED
else -> null
},
isNsfw = contents.select(".icon_adult").isNotEmpty(),
chapters = contents.select("#episode_list > ul > li").mapChapters { i, li ->
val href = li.selectFirst("div > div.description > div.title > a")
?.attrAsRelativeUrl("href") ?: li.parseFailed()
MangaChapter(
id = generateUid(href),
name = li.select("div > div.description > div.title > a").text(),
number = i + 1,
url = href,
scanlator = null,
branch = null,
uploadDate = 0,
source = source,
)
},
)
}
override suspend fun getDetails(manga: Manga): Manga {
val doc = webClient.httpGet(manga.url.toAbsoluteUrl(getDomain("seiga"))).parseHtml()
val contents = doc.body().selectFirstOrThrow("#contents")
val statusText = contents
.select("div.mg_work_detail > div > div:nth-child(2) > div.tip.content_status.status_series > span")
.text()
return manga.copy(
description = contents.selectFirst("div.mg_work_detail > div > div.row > div.description_text")?.html(),
largeCoverUrl = contents.selectFirst("div.primaries > div.main_visual > a > img")
?.attrAsAbsoluteUrlOrNull("src"),
state = when (statusText) {
STATUS_ONGOING -> MangaState.ONGOING
STATUS_FINISHED -> MangaState.FINISHED
else -> null
},
isNsfw = contents.select(".icon_adult").isNotEmpty(),
chapters = contents.select("#episode_list > ul > li").mapChapters { i, li ->
val href = li.selectFirst("div > div.description > div.title > a")
?.attrAsRelativeUrl("href") ?: li.parseFailed()
MangaChapter(
id = generateUid(href),
name = li.select("div > div.description > div.title > a").text(),
number = i + 1,
url = href,
scanlator = null,
branch = null,
uploadDate = 0,
source = source,
)
},
)
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(getDomain("seiga"))
val doc = context.httpGet(fullUrl).parseHtml()
if (!doc.select("#login_manga").isEmpty())
throw AuthRequiredException(source)
val root = doc.body().select("#page_contents > li")
return root.map { li ->
val url = li.select("div > img").attr("data-original")
MangaPage(
id = generateUid(url),
url = url,
preview = null,
referer = fullUrl,
source = source,
)
}
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(getDomain("seiga"))
val doc = webClient.httpGet(fullUrl).parseHtml()
if (!doc.select("#login_manga").isEmpty())
throw AuthRequiredException(source)
val root = doc.body().select("#page_contents > li")
return root.map { li ->
val url = li.select("div > img").attr("data-original")
MangaPage(
id = generateUid(url),
url = url,
preview = null,
referer = fullUrl,
source = source,
)
}
}
override suspend fun getTags(): Set<MangaTag> {
val doc = context.httpGet("https://${getDomain("seiga")}/manga/list").parseHtml()
val root = doc.body().selectOrThrow("#mg_category_list > ul > li")
return root.mapToSet { li ->
val a = li.selectFirstOrThrow("a")
MangaTag(
title = a.text(),
key = a.attrAsRelativeUrlOrNull("href").orEmpty(),
source = source,
)
}
}
override suspend fun getTags(): Set<MangaTag> {
val doc = webClient.httpGet("https://${getDomain("seiga")}/manga/list").parseHtml()
val root = doc.body().selectOrThrow("#mg_category_list > ul > li")
return root.mapToSet { li ->
val a = li.selectFirstOrThrow("a")
MangaTag(
title = a.text(),
key = a.attrAsRelativeUrlOrNull("href").orEmpty(),
source = source,
)
}
}
private suspend fun getSearchList(query: String, page: Int): List<Manga> {
val domain = getDomain("seiga")
val doc = context.httpGet("https://$domain/manga/search/?q=$query&page=$page&sort=score").parseHtml()
val root = doc.body().select(".search_result__item")
return root.mapNotNull { item ->
val href = item.selectFirst(".search_result__item__thumbnail > a")
?.attrAsRelativeUrl("href") ?: doc.parseFailed()
Manga(
id = generateUid(href),
url = href,
publicUrl = href.toAbsoluteUrl(item.host ?: domain),
title = item.selectFirst(".search_result__item__info > .search_result__item__info--title > a")
?.text()?.trim() ?: return@mapNotNull null,
altTitle = null,
author = null,
tags = emptySet(),
rating = RATING_UNKNOWN,
state = null,
isNsfw = false,
source = source,
coverUrl = item.selectFirst(".search_result__item__thumbnail > a > img")
?.attrAsAbsoluteUrl("data-original").orEmpty(),
)
}
}
private suspend fun getSearchList(query: String, page: Int): List<Manga> {
val domain = getDomain("seiga")
val doc = webClient.httpGet("https://$domain/manga/search/?q=$query&page=$page&sort=score").parseHtml()
val root = doc.body().select(".search_result__item")
return root.mapNotNull { item ->
val href = item.selectFirst(".search_result__item__thumbnail > a")
?.attrAsRelativeUrl("href") ?: doc.parseFailed()
Manga(
id = generateUid(href),
url = href,
publicUrl = href.toAbsoluteUrl(item.host ?: domain),
title = item.selectFirst(".search_result__item__info > .search_result__item__info--title > a")
?.text()?.trim() ?: return@mapNotNull null,
altTitle = null,
author = null,
tags = emptySet(),
rating = RATING_UNKNOWN,
state = null,
isNsfw = false,
source = source,
coverUrl = item.selectFirst(".search_result__item__thumbnail > a > img")
?.attrAsAbsoluteUrl("data-original").orEmpty(),
)
}
}
private fun getSortKey(sortOrder: SortOrder) = when (sortOrder) {
SortOrder.POPULARITY -> "manga_view"
SortOrder.UPDATED -> "manga_updated"
else -> "manga_view"
}
}
private fun getSortKey(sortOrder: SortOrder) = when (sortOrder) {
SortOrder.POPULARITY -> "manga_view"
SortOrder.UPDATED -> "manga_updated"
else -> "manga_view"
}
}

437
src/main/kotlin/org/koitharu/kotatsu/parsers/site/NineMangaParser.kt
Unescape Escape View File

@ -12,249 +12,248 @@ import java.text.SimpleDateFormat
import java.util.*
internal abstract class NineMangaParser(
final override val context: MangaLoaderContext,
source: MangaSource,
defaultDomain: String,
) : PagedMangaParser(source, pageSize = 26) {
context: MangaLoaderContext,
source: MangaSource,
defaultDomain: String,
) : PagedMangaParser(context, source, pageSize = 26) {
override val configKeyDomain = ConfigKey.Domain(defaultDomain, null)
override val configKeyDomain = ConfigKey.Domain(defaultDomain, null)
init {
context.cookieJar.insertCookies(getDomain(), "ninemanga_template_desk=yes")
}
init {
context.cookieJar.insertCookies(domain, "ninemanga_template_desk=yes")
}
override val headers = Headers.Builder()
.add("Accept-Language", "en-US;q=0.7,en;q=0.3")
.build()
override val headers = Headers.Builder()
.add("Accept-Language", "en-US;q=0.7,en;q=0.3")
.build()
override val sortOrders: Set<SortOrder> = Collections.singleton(
SortOrder.POPULARITY,
)
override val sortOrders: Set<SortOrder> = Collections.singleton(
SortOrder.POPULARITY,
)
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val url = buildString {
append("https://")
append(getDomain())
when {
!query.isNullOrEmpty() -> {
append("/search/?name_sel=&wd=")
append(query.urlEncoded())
append("&page=")
}
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val url = buildString {
append("https://")
append(domain)
when {
!query.isNullOrEmpty() -> {
append("/search/?name_sel=&wd=")
append(query.urlEncoded())
append("&page=")
}
!tags.isNullOrEmpty() -> {
append("/search/?category_id=")
for (tag in tags) {
append(tag.key)
append(',')
}
append("&page=")
}
!tags.isNullOrEmpty() -> {
append("/search/?category_id=")
for (tag in tags) {
append(tag.key)
append(',')
}
append("&page=")
}
else -> {
append("/category/index_")
}
}
append(page)
append(".html")
}
val doc = context.httpGet(url, headers).parseHtml()
val root = doc.body().selectFirst("ul.direlist")
?: doc.parseFailed("Cannot find root")
val baseHost = root.baseUri().toHttpUrl().host
return root.select("li").map { node ->
val href = node.selectFirst("a")?.absUrl("href")
?: node.parseFailed("Link not found")
val relUrl = href.toRelativeUrl(baseHost)
val dd = node.selectFirst("dd")
Manga(
id = generateUid(relUrl),
url = relUrl,
publicUrl = href,
title = dd?.selectFirst("a.bookname")?.text()?.toCamelCase().orEmpty(),
altTitle = null,
coverUrl = node.selectFirst("img")?.absUrl("src").orEmpty(),
rating = RATING_UNKNOWN,
author = null,
isNsfw = false,
tags = emptySet(),
state = null,
source = source,
description = dd?.selectFirst("p")?.html(),
)
}
}
else -> {
append("/category/index_")
}
}
append(page)
append(".html")
}
val doc = webClient.httpGet(url).parseHtml()
val root = doc.body().selectFirst("ul.direlist")
?: doc.parseFailed("Cannot find root")
val baseHost = root.baseUri().toHttpUrl().host
return root.select("li").map { node ->
val href = node.selectFirst("a")?.absUrl("href")
?: node.parseFailed("Link not found")
val relUrl = href.toRelativeUrl(baseHost)
val dd = node.selectFirst("dd")
Manga(
id = generateUid(relUrl),
url = relUrl,
publicUrl = href,
title = dd?.selectFirst("a.bookname")?.text()?.toCamelCase().orEmpty(),
altTitle = null,
coverUrl = node.selectFirst("img")?.absUrl("src").orEmpty(),
rating = RATING_UNKNOWN,
author = null,
isNsfw = false,
tags = emptySet(),
state = null,
source = source,
description = dd?.selectFirst("p")?.html(),
)
}
}
override suspend fun getDetails(manga: Manga): Manga {
val doc = context.httpGet(
manga.url.toAbsoluteUrl(getDomain()) + "?waring=1",
headers,
).parseHtml()
val root = doc.body().selectFirstOrThrow("div.manga")
val infoRoot = root.selectFirstOrThrow("div.bookintro")
return manga.copy(
tags = infoRoot.getElementsByAttributeValue("itemprop", "genre").first()
?.select("a")?.mapToSet { a ->
MangaTag(
title = a.text().toTitleCase(),
key = a.attr("href").substringBetween("/", "."),
source = source,
)
}.orEmpty(),
author = infoRoot.getElementsByAttributeValue("itemprop", "author").first()?.text(),
state = parseStatus(infoRoot.select("li a.red").text()),
description = infoRoot.getElementsByAttributeValue("itemprop", "description").first()
?.html()?.substringAfter("</b>"),
chapters = root.selectFirst("div.chapterbox")?.select("ul.sub_vol_ul > li")
?.asReversed()?.mapChapters { i, li ->
val a = li.selectFirst("a.chapter_list_a")
val href = a?.attrAsRelativeUrlOrNull("href")
?.replace("%20", " ") ?: li.parseFailed("Link not found")
MangaChapter(
id = generateUid(href),
name = a.text(),
number = i + 1,
url = href,
uploadDate = parseChapterDateByLang(li.selectFirst("span")?.text().orEmpty()),
source = source,
scanlator = null,
branch = null,
)
},
)
}
override suspend fun getDetails(manga: Manga): Manga {
val doc = webClient.httpGet(
manga.url.toAbsoluteUrl(domain) + "?waring=1",
).parseHtml()
val root = doc.body().selectFirstOrThrow("div.manga")
val infoRoot = root.selectFirstOrThrow("div.bookintro")
return manga.copy(
tags = infoRoot.getElementsByAttributeValue("itemprop", "genre").first()
?.select("a")?.mapToSet { a ->
MangaTag(
title = a.text().toTitleCase(),
key = a.attr("href").substringBetween("/", "."),
source = source,
)
}.orEmpty(),
author = infoRoot.getElementsByAttributeValue("itemprop", "author").first()?.text(),
state = parseStatus(infoRoot.select("li a.red").text()),
description = infoRoot.getElementsByAttributeValue("itemprop", "description").first()
?.html()?.substringAfter("</b>"),
chapters = root.selectFirst("div.chapterbox")?.select("ul.sub_vol_ul > li")
?.asReversed()?.mapChapters { i, li ->
val a = li.selectFirst("a.chapter_list_a")
val href = a?.attrAsRelativeUrlOrNull("href")
?.replace("%20", " ") ?: li.parseFailed("Link not found")
MangaChapter(
id = generateUid(href),
name = a.text(),
number = i + 1,
url = href,
uploadDate = parseChapterDateByLang(li.selectFirst("span")?.text().orEmpty()),
source = source,
scanlator = null,
branch = null,
)
},
)
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val doc = context.httpGet(chapter.url.toAbsoluteUrl(getDomain()), headers).parseHtml()
return doc.body().getElementById("page")?.select("option")?.map { option ->
val url = option.attr("value")
MangaPage(
id = generateUid(url),
url = url,
referer = chapter.url.toAbsoluteUrl(getDomain()),
preview = null,
source = source,
)
} ?: doc.parseFailed("Pages list not found")
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml()
return doc.body().getElementById("page")?.select("option")?.map { option ->
val url = option.attr("value")
MangaPage(
id = generateUid(url),
url = url,
referer = chapter.url.toAbsoluteUrl(domain),
preview = null,
source = source,
)
} ?: doc.parseFailed("Pages list not found")
}
override suspend fun getPageUrl(page: MangaPage): String {
val doc = context.httpGet(page.url.toAbsoluteUrl(getDomain()), headers).parseHtml()
val root = doc.body()
return root.selectFirst("a.pic_download")?.absUrl("href")
?: doc.parseFailed("Page image not found")
}
override suspend fun getPageUrl(page: MangaPage): String {
val doc = webClient.httpGet(page.url.toAbsoluteUrl(domain)).parseHtml()
val root = doc.body()
return root.selectFirst("a.pic_download")?.absUrl("href")
?: doc.parseFailed("Page image not found")
}
override suspend fun getTags(): Set<MangaTag> {
val doc = context.httpGet("https://${getDomain()}/search/?type=high", headers)
.parseHtml()
val root = doc.body().getElementById("search_form")
return root?.select("li.cate_list")?.mapNotNullToSet { li ->
val cateId = li.attr("cate_id") ?: return@mapNotNullToSet null
val a = li.selectFirst("a") ?: return@mapNotNullToSet null
MangaTag(
title = a.text().toTitleCase(),
key = cateId,
source = source,
)
} ?: doc.parseFailed("Root not found")
}
override suspend fun getTags(): Set<MangaTag> {
val doc = webClient.httpGet("https://${domain}/search/?type=high")
.parseHtml()
val root = doc.body().getElementById("search_form")
return root?.select("li.cate_list")?.mapNotNullToSet { li ->
val cateId = li.attr("cate_id") ?: return@mapNotNullToSet null
val a = li.selectFirst("a") ?: return@mapNotNullToSet null
MangaTag(
title = a.text().toTitleCase(),
key = cateId,
source = source,
)
} ?: doc.parseFailed("Root not found")
}
private fun parseStatus(status: String) = when {
status.contains("Ongoing") -> MangaState.ONGOING
status.contains("Completed") -> MangaState.FINISHED
else -> null
}
private fun parseStatus(status: String) = when {
status.contains("Ongoing") -> MangaState.ONGOING
status.contains("Completed") -> MangaState.FINISHED
else -> null
}
private fun parseChapterDateByLang(date: String): Long {
val dateWords = date.split(" ")
private fun parseChapterDateByLang(date: String): Long {
val dateWords = date.split(" ")
if (dateWords.size == 3) {
if (dateWords[1].contains(",")) {
SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH).tryParse(date)
} else {
val timeAgo = Integer.parseInt(dateWords[0])
return Calendar.getInstance().apply {
when (dateWords[1]) {
"minutes" -> Calendar.MINUTE // EN-FR
"hours" -> Calendar.HOUR // EN
if (dateWords.size == 3) {
if (dateWords[1].contains(",")) {
SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH).tryParse(date)
} else {
val timeAgo = Integer.parseInt(dateWords[0])
return Calendar.getInstance().apply {
when (dateWords[1]) {
"minutes" -> Calendar.MINUTE // EN-FR
"hours" -> Calendar.HOUR // EN
"minutos" -> Calendar.MINUTE // ES
"horas" -> Calendar.HOUR
"minutos" -> Calendar.MINUTE // ES
"horas" -> Calendar.HOUR
// "minutos" -> Calendar.MINUTE // BR
"hora" -> Calendar.HOUR
// "minutos" -> Calendar.MINUTE // BR
"hora" -> Calendar.HOUR
"минут" -> Calendar.MINUTE // RU
"часа" -> Calendar.HOUR
"минут" -> Calendar.MINUTE // RU
"часа" -> Calendar.HOUR
"Stunden" -> Calendar.HOUR // DE
"Stunden" -> Calendar.HOUR // DE
"minuti" -> Calendar.MINUTE // IT
"ore" -> Calendar.HOUR
"minuti" -> Calendar.MINUTE // IT
"ore" -> Calendar.HOUR
"heures" -> Calendar.HOUR // FR ("minutes" also French word)
else -> null
}?.let {
add(it, -timeAgo)
}
}.timeInMillis
}
}
return 0L
}
"heures" -> Calendar.HOUR // FR ("minutes" also French word)
else -> null
}?.let {
add(it, -timeAgo)
}
}.timeInMillis
}
}
return 0L
}
@MangaSourceParser("NINEMANGA_EN", "NineManga English", "en")
class English(context: MangaLoaderContext) : NineMangaParser(
context,
MangaSource.NINEMANGA_EN,
"www.ninemanga.com",
)
@MangaSourceParser("NINEMANGA_EN", "NineManga English", "en")
class English(context: MangaLoaderContext) : NineMangaParser(
context,
MangaSource.NINEMANGA_EN,
"www.ninemanga.com",
)
@MangaSourceParser("NINEMANGA_ES", "NineManga Español", "es")
class Spanish(context: MangaLoaderContext) : NineMangaParser(
context,
MangaSource.NINEMANGA_ES,
"es.ninemanga.com",
)
@MangaSourceParser("NINEMANGA_ES", "NineManga Español", "es")
class Spanish(context: MangaLoaderContext) : NineMangaParser(
context,
MangaSource.NINEMANGA_ES,
"es.ninemanga.com",
)
@MangaSourceParser("NINEMANGA_RU", "NineManga Русский", "ru")
class Russian(context: MangaLoaderContext) : NineMangaParser(
context,
MangaSource.NINEMANGA_RU,
"ru.ninemanga.com",
)
@MangaSourceParser("NINEMANGA_RU", "NineManga Русский", "ru")
class Russian(context: MangaLoaderContext) : NineMangaParser(
context,
MangaSource.NINEMANGA_RU,
"ru.ninemanga.com",
)
@MangaSourceParser("NINEMANGA_DE", "NineManga Deutsch", "de")
class Deutsch(context: MangaLoaderContext) : NineMangaParser(
context,
MangaSource.NINEMANGA_DE,
"de.ninemanga.com",
)
@MangaSourceParser("NINEMANGA_DE", "NineManga Deutsch", "de")
class Deutsch(context: MangaLoaderContext) : NineMangaParser(
context,
MangaSource.NINEMANGA_DE,
"de.ninemanga.com",
)
@MangaSourceParser("NINEMANGA_BR", "NineManga Brasil", "pt")
class Brazil(context: MangaLoaderContext) : NineMangaParser(
context,
MangaSource.NINEMANGA_BR,
"br.ninemanga.com",
)
@MangaSourceParser("NINEMANGA_BR", "NineManga Brasil", "pt")
class Brazil(context: MangaLoaderContext) : NineMangaParser(
context,
MangaSource.NINEMANGA_BR,
"br.ninemanga.com",
)
@MangaSourceParser("NINEMANGA_IT", "NineManga Italiano", "it")
class Italiano(context: MangaLoaderContext) : NineMangaParser(
context,
MangaSource.NINEMANGA_IT,
"it.ninemanga.com",
)
@MangaSourceParser("NINEMANGA_IT", "NineManga Italiano", "it")
class Italiano(context: MangaLoaderContext) : NineMangaParser(
context,
MangaSource.NINEMANGA_IT,
"it.ninemanga.com",
)
@MangaSourceParser("NINEMANGA_FR", "NineManga Français", "fr")
class Francais(context: MangaLoaderContext) : NineMangaParser(
context,
MangaSource.NINEMANGA_FR,
"fr.ninemanga.com",
)
}
@MangaSourceParser("NINEMANGA_FR", "NineManga Français", "fr")
class Francais(context: MangaLoaderContext) : NineMangaParser(
context,
MangaSource.NINEMANGA_FR,
"fr.ninemanga.com",
)
}

424
src/main/kotlin/org/koitharu/kotatsu/parsers/site/NudeMoonParser.kt
Unescape Escape View File

@ -15,217 +15,213 @@ private const val MAX_THUMB_INDEX = 19
@MangaSourceParser("NUDEMOON", "Nude-Moon", "ru")
internal class NudeMoonParser(
override val context: MangaLoaderContext,
) : MangaParser(MangaSource.NUDEMOON), MangaParserAuthProvider {
override val configKeyDomain = ConfigKey.Domain(
defaultValue = "nude-moon.org",
presetValues = arrayOf("nude-moon.org", "nude-moon.net"),
)
override val authUrl: String
get() = "https://${getDomain()}/index.php"
override val isAuthorized: Boolean
get() {
return context.cookieJar.getCookies(getDomain()).any {
it.name == "fusion_user"
}
}
override val sortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.NEWEST,
SortOrder.POPULARITY,
SortOrder.RATING,
)
init {
context.cookieJar.insertCookies(
getDomain(),
"NMfYa=1;",
"nm_mobile=0;",
)
}
override suspend fun getList(
offset: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val domain = getDomain()
val url = when {
!query.isNullOrEmpty() -> "https://$domain/search?stext=${query.urlEncoded()}&rowstart=$offset"
!tags.isNullOrEmpty() -> tags.joinToString(
separator = "_",
prefix = "https://$domain/tags/",
postfix = "&rowstart=$offset",
transform = { it.key.urlEncoded() },
)
else -> "https://$domain/all_manga?${getSortKey(sortOrder)}&rowstart=$offset"
}
val doc = context.httpGet(url).parseHtml()
val root = doc.body().run {
selectFirst("td.main-bg") ?: selectFirst("td.main-body")
} ?: doc.parseFailed("Cannot find root")
return root.select("table.news_pic2").mapNotNull { row ->
val a = row.selectFirst("td.bg_style1")?.selectFirst("a")
?: return@mapNotNull null
val href = a.attrAsRelativeUrl("href")
val title = a.selectFirst("h2")?.text().orEmpty()
val info = row.selectFirst("td[width=100%]") ?: return@mapNotNull null
Manga(
id = generateUid(href),
url = href,
title = title.substringAfter(" / "),
altTitle = title.substringBefore(" / ", "")
.takeUnless { it.isBlank() },
author = info.getElementsContainingOwnText("Автор:").firstOrNull()
?.nextElementSibling()?.ownText(),
coverUrl = row.selectFirst("img.news_pic2")?.absUrl("data-src")
.orEmpty(),
tags = row.selectFirst("span.tag-links")?.select("a")
?.mapToSet {
MangaTag(
title = it.text().toTitleCase(),
key = it.attr("href").substringAfterLast('/'),
source = source,
)
}.orEmpty(),
source = source,
publicUrl = a.absUrl("href"),
rating = RATING_UNKNOWN,
isNsfw = true,
description = row.selectFirst("div.description")?.html(),
state = null,
)
}
}
override suspend fun getDetails(manga: Manga): Manga {
val body = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml().body()
val root = body.selectFirst("table.shoutbox")
?: body.parseFailed("Cannot find root")
val info = root.select("div.tbl2")
val lastInfo = info.last()
return manga.copy(
largeCoverUrl = body.selectFirst("img.news_pic2")?.absUrl("src"),
description = info.select("div.blockquote").lastOrNull()?.html() ?: manga.description,
tags = info.select("span.tag-links").firstOrNull()?.select("a")?.mapToSet {
MangaTag(
title = it.text().toTitleCase(),
key = it.attr("href").substringAfterLast('/'),
source = source,
)
}?.plus(manga.tags) ?: manga.tags,
author = lastInfo?.getElementsByAttributeValueContaining("href", "mangaka/")?.text()
?: manga.author,
chapters = listOf(
MangaChapter(
id = manga.id,
url = getReadLink(manga.url),
source = source,
number = 1,
name = manga.title,
scanlator = lastInfo?.getElementsByAttributeValueContaining("href", "perevod/")?.text(),
uploadDate = lastInfo?.getElementsContainingOwnText("Дата:")
?.firstOrNull()
?.html()
?.parseDate() ?: 0L,
branch = null,
),
),
)
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(getDomain())
val doc = context.httpGet(fullUrl).parseHtml()
val mangaId = chapter.url.substringAfterLast('/').substringBefore('-').toIntOrNull()
val script = doc.select("script").firstNotNullOfOrNull {
it.html().takeIf { x -> x.contains(" images = new ") }
} ?: if (isAuthorized) {
doc.parseFailed("Cannot find pages list")
} else {
throw AuthRequiredException(source)
}
val pagesRegex = Regex("images\\[(\\d+)].src\\s*=\\s*'([^']+)'", RegexOption.MULTILINE)
return pagesRegex.findAll(script).map { match ->
val i = match.groupValues[1].toInt()
val url = match.groupValues[2]
MangaPage(
id = generateUid(url),
url = url,
referer = fullUrl,
preview = if (i <= MAX_THUMB_INDEX && mangaId != null) {
val part2 = url.substringBeforeLast('/')
val part3 = url.substringAfterLast('/')
val part1 = part2.substringBeforeLast('/')
"$part1/thumb/$mangaId/thumb_$part3"
} else {
null
},
source = source,
)
}.toList()
}
override suspend fun getTags(): Set<MangaTag> {
val domain = getDomain()
val doc = context.httpGet("https://$domain/all_manga").parseHtml()
val root = doc.body().getElementsContainingOwnText("Поиск манги по тегам")
.firstOrNull()?.parents()?.find { it.tag().normalName() == "tbody" }
?.selectFirst("td.textbox")?.selectFirst("td.small")
?: doc.parseFailed("Tags root not found")
return root.select("a").mapToSet {
MangaTag(
title = it.text().toTitleCase(),
key = it.attr("href").substringAfterLast('/')
.removeSuffix("+"),
source = source,
)
}
}
override suspend fun getUsername(): String {
val body = context.httpGet("https://${getDomain()}/").parseHtml()
.body()
return body
.getElementsContainingOwnText("Профиль")
.firstOrNull()
?.attr("href")
?.substringAfterLast('/')
?: run {
throw if (body.selectFirst("form[name=\"loginform\"]") != null) {
AuthRequiredException(source)
} else {
body.parseFailed("Cannot find username")
}
}
}
override fun getFaviconUrl(): String {
return "https://${getDomain()}/favicon.jpg"
}
private fun getSortKey(sortOrder: SortOrder) =
when (sortOrder) {
SortOrder.POPULARITY -> "views"
SortOrder.NEWEST -> "date"
SortOrder.RATING -> "like"
else -> "like"
}
private fun String.parseDate(): Long {
val dateString = substringBetweenFirst("Дата:", "<")?.trim() ?: return 0
val dateFormat = SimpleDateFormat("d MMMM yyyy", Locale("ru"))
return dateFormat.tryParse(dateString)
}
private fun getReadLink(url: String): String {
val prefix = url.substringBefore('-', "")
val suffix = url.substringAfter('-').trimStart('-')
return "$prefix-online-$suffix"
}
}
context: MangaLoaderContext,
) : MangaParser(context, MangaSource.NUDEMOON), MangaParserAuthProvider {
override val configKeyDomain = ConfigKey.Domain(
defaultValue = "nude-moon.org",
presetValues = arrayOf("nude-moon.org", "nude-moon.net"),
)
override val authUrl: String
get() = "https://${domain}/index.php"
override val isAuthorized: Boolean
get() {
return context.cookieJar.getCookies(domain).any {
it.name == "fusion_user"
}
}
override val sortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.NEWEST,
SortOrder.POPULARITY,
SortOrder.RATING,
)
init {
context.cookieJar.insertCookies(
domain,
"NMfYa=1;",
"nm_mobile=0;",
)
}
override suspend fun getList(
offset: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val domain = domain
val url = when {
!query.isNullOrEmpty() -> "https://$domain/search?stext=${query.urlEncoded()}&rowstart=$offset"
!tags.isNullOrEmpty() -> tags.joinToString(
separator = "_",
prefix = "https://$domain/tags/",
postfix = "&rowstart=$offset",
transform = { it.key.urlEncoded() },
)
else -> "https://$domain/all_manga?${getSortKey(sortOrder)}&rowstart=$offset"
}
val doc = webClient.httpGet(url).parseHtml()
val root = doc.body().run {
selectFirst("td.main-bg") ?: selectFirst("td.main-body")
} ?: doc.parseFailed("Cannot find root")
return root.select("table.news_pic2").mapNotNull { row ->
val a = row.selectFirst("td.bg_style1")?.selectFirst("a")
?: return@mapNotNull null
val href = a.attrAsRelativeUrl("href")
val title = a.selectFirst("h2")?.text().orEmpty()
val info = row.selectFirst("td[width=100%]") ?: return@mapNotNull null
Manga(
id = generateUid(href),
url = href,
title = title.substringAfter(" / "),
altTitle = title.substringBefore(" / ", "")
.takeUnless { it.isBlank() },
author = info.getElementsContainingOwnText("Автор:").firstOrNull()
?.nextElementSibling()?.ownText(),
coverUrl = row.selectFirst("img.news_pic2")?.absUrl("data-src")
.orEmpty(),
tags = row.selectFirst("span.tag-links")?.select("a")
?.mapToSet {
MangaTag(
title = it.text().toTitleCase(),
key = it.attr("href").substringAfterLast('/'),
source = source,
)
}.orEmpty(),
source = source,
publicUrl = a.absUrl("href"),
rating = RATING_UNKNOWN,
isNsfw = true,
description = row.selectFirst("div.description")?.html(),
state = null,
)
}
}
override suspend fun getDetails(manga: Manga): Manga {
val body = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml().body()
val root = body.selectFirst("table.shoutbox")
?: body.parseFailed("Cannot find root")
val info = root.select("div.tbl2")
val lastInfo = info.last()
return manga.copy(
largeCoverUrl = body.selectFirst("img.news_pic2")?.absUrl("src"),
description = info.select("div.blockquote").lastOrNull()?.html() ?: manga.description,
tags = info.select("span.tag-links").firstOrNull()?.select("a")?.mapToSet {
MangaTag(
title = it.text().toTitleCase(),
key = it.attr("href").substringAfterLast('/'),
source = source,
)
}?.plus(manga.tags) ?: manga.tags,
author = lastInfo?.getElementsByAttributeValueContaining("href", "mangaka/")?.text()
?: manga.author,
chapters = listOf(
MangaChapter(
id = manga.id,
url = getReadLink(manga.url),
source = source,
number = 1,
name = manga.title,
scanlator = lastInfo?.getElementsByAttributeValueContaining("href", "perevod/")?.text(),
uploadDate = lastInfo?.getElementsContainingOwnText("Дата:")
?.firstOrNull()
?.html()
?.parseDate() ?: 0L,
branch = null,
),
),
)
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(domain)
val doc = webClient.httpGet(fullUrl).parseHtml()
val mangaId = chapter.url.substringAfterLast('/').substringBefore('-').toIntOrNull()
val script = doc.select("script").firstNotNullOfOrNull {
it.html().takeIf { x -> x.contains(" images = new ") }
} ?: if (isAuthorized) {
doc.parseFailed("Cannot find pages list")
} else {
throw AuthRequiredException(source)
}
val pagesRegex = Regex("images\\[(\\d+)].src\\s*=\\s*'([^']+)'", RegexOption.MULTILINE)
return pagesRegex.findAll(script).map { match ->
val i = match.groupValues[1].toInt()
val url = match.groupValues[2]
MangaPage(
id = generateUid(url),
url = url,
referer = fullUrl,
preview = if (i <= MAX_THUMB_INDEX && mangaId != null) {
val part2 = url.substringBeforeLast('/')
val part3 = url.substringAfterLast('/')
val part1 = part2.substringBeforeLast('/')
"$part1/thumb/$mangaId/thumb_$part3"
} else {
null
},
source = source,
)
}.toList()
}
override suspend fun getTags(): Set<MangaTag> {
val domain = domain
val doc = webClient.httpGet("https://$domain/all_manga").parseHtml()
val root = doc.body().getElementsContainingOwnText("Поиск манги по тегам")
.firstOrNull()?.parents()?.find { it.tag().normalName() == "tbody" }
?.selectFirst("td.textbox")?.selectFirst("td.small")
?: doc.parseFailed("Tags root not found")
return root.select("a").mapToSet {
MangaTag(
title = it.text().toTitleCase(),
key = it.attr("href").substringAfterLast('/')
.removeSuffix("+"),
source = source,
)
}
}
override suspend fun getUsername(): String {
val body = webClient.httpGet("https://${domain}/").parseHtml()
.body()
return body
.getElementsContainingOwnText("Профиль")
.firstOrNull()
?.attr("href")
?.substringAfterLast('/')
?: run {
throw if (body.selectFirst("form[name=\"loginform\"]") != null) {
AuthRequiredException(source)
} else {
body.parseFailed("Cannot find username")
}
}
}
private fun getSortKey(sortOrder: SortOrder) =
when (sortOrder) {
SortOrder.POPULARITY -> "views"
SortOrder.NEWEST -> "date"
SortOrder.RATING -> "like"
else -> "like"
}
private fun String.parseDate(): Long {
val dateString = substringBetweenFirst("Дата:", "<")?.trim() ?: return 0
val dateFormat = SimpleDateFormat("d MMMM yyyy", Locale("ru"))
return dateFormat.tryParse(dateString)
}
private fun getReadLink(url: String): String {
val prefix = url.substringBefore('-', "")
val suffix = url.substringAfter('-').trimStart('-')
return "$prefix-online-$suffix"
}
}

480
src/main/kotlin/org/koitharu/kotatsu/parsers/site/RemangaParser.kt
Unescape Escape View File

@ -28,259 +28,259 @@ private const val STATUS_FINISHED = 0
@MangaSourceParser("REMANGA", "Remanga", "ru")
internal class RemangaParser(
override val context: MangaLoaderContext,
) : PagedMangaParser(MangaSource.REMANGA, PAGE_SIZE), MangaParserAuthProvider {
context: MangaLoaderContext,
) : PagedMangaParser(context, MangaSource.REMANGA, PAGE_SIZE), MangaParserAuthProvider {
override val headers = Headers.Builder()
.add("User-Agent", "Mozilla/5.0 (Android 13; Mobile; rv:68.0) Gecko/68.0 Firefox/109.0")
.build()
private val baseHeaders = Headers.Builder()
.add("User-Agent", "Mozilla/5.0 (Android 13; Mobile; rv:68.0) Gecko/68.0 Firefox/109.0")
.build()
override val configKeyDomain = ConfigKey.Domain("remanga.org", null)
override val authUrl: String
get() = "https://${getDomain()}/user/login"
override val headers
get() = getApiHeaders()
override val sortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.UPDATED,
SortOrder.POPULARITY,
SortOrder.RATING,
SortOrder.NEWEST,
)
override val configKeyDomain = ConfigKey.Domain("remanga.org", null)
override val authUrl: String
get() = "https://${domain}/user/login"
override val isAuthorized: Boolean
get() {
return context.cookieJar.getCookies(getDomain()).any {
it.name == "user"
}
}
override val sortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.UPDATED,
SortOrder.POPULARITY,
SortOrder.RATING,
SortOrder.NEWEST,
)
private val regexLastUrlPath = Regex("/[^/]+/?$")
override val isAuthorized: Boolean
get() {
return context.cookieJar.getCookies(domain).any {
it.name == "user"
}
}
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
copyCookies()
val domain = getDomain()
val urlBuilder = StringBuilder()
.append("https://api.")
.append(domain)
if (query != null) {
urlBuilder.append("/api/search/?query=")
.append(query.urlEncoded())
} else {
urlBuilder.append("/api/search/catalog/?ordering=")
.append(getSortKey(sortOrder))
tags?.forEach { tag ->
urlBuilder.append("&genres=")
urlBuilder.append(tag.key)
}
}
urlBuilder
.append("&page=")
.append(page)
.append("&count=")
.append(PAGE_SIZE)
val content = context.httpGet(urlBuilder.toString(), getApiHeaders()).parseJson()
.getJSONArray("content")
return content.mapJSON { jo ->
val url = "/manga/${jo.getString("dir")}"
val img = jo.getJSONObject("img")
Manga(
id = generateUid(url),
url = url,
publicUrl = "https://$domain$url",
title = jo.getString("rus_name"),
altTitle = jo.getString("en_name"),
rating = jo.getString("avg_rating").toFloatOrNull()?.div(10f) ?: RATING_UNKNOWN,
coverUrl = "https://api.$domain${img.getString("mid")}",
largeCoverUrl = "https://api.$domain${img.getString("high")}",
author = null,
isNsfw = false,
state = null,
tags = jo.optJSONArray("genres")?.mapJSONToSet { g ->
MangaTag(
title = g.getString("name").toTitleCase(),
key = g.getInt("id").toString(),
source = MangaSource.REMANGA,
)
}.orEmpty(),
source = MangaSource.REMANGA,
)
}
}
private val regexLastUrlPath = Regex("/[^/]+/?$")
override suspend fun getDetails(manga: Manga): Manga {
copyCookies()
val domain = getDomain()
val slug = manga.url.find(regexLastUrlPath)
?: throw ParseException("Cannot obtain slug from ${manga.url}", manga.publicUrl)
val data = context.httpGet(
url = "https://api.$domain/api/titles$slug/",
headers = getApiHeaders(),
).handle401().parseJson()
val content = try {
data.getJSONObject("content")
} catch (e: JSONException) {
throw ParseException(data.optString("msg"), manga.publicUrl, e)
}
val branchId = content.getJSONArray("branches").optJSONObject(0)
?.getLong("id") ?: throw ParseException("No branches found", manga.publicUrl)
val chapters = grabChapters(domain, branchId)
val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.US)
return manga.copy(
description = content.getString("description"),
state = when (content.optJSONObject("status")?.getInt("id")) {
STATUS_ONGOING -> MangaState.ONGOING
STATUS_FINISHED -> MangaState.FINISHED
else -> null
},
tags = content.getJSONArray("genres").mapJSONToSet { g ->
MangaTag(
title = g.getString("name").toTitleCase(),
key = g.getInt("id").toString(),
source = MangaSource.REMANGA,
)
},
chapters = chapters.mapChapters { i, jo ->
if (
jo.getBooleanOrDefault("is_paid", false) &&
!jo.getBooleanOrDefault("is_bought", false)
) {
return@mapChapters null
}
val id = jo.getLong("id")
val name = jo.getString("name").toTitleCase(Locale.ROOT)
val publishers = jo.optJSONArray("publishers")
MangaChapter(
id = generateUid(id),
url = "/api/titles/chapters/$id/",
number = chapters.size - i,
name = buildString {
append("Том ")
append(jo.optString("tome", "0"))
append(". ")
append("Глава ")
append(jo.optString("chapter", "0"))
if (name.isNotEmpty()) {
append(" - ")
append(name)
}
},
uploadDate = dateFormat.tryParse(jo.getString("upload_date")),
scanlator = publishers?.optJSONObject(0)?.getStringOrNull("name"),
source = MangaSource.REMANGA,
branch = null,
)
}.asReversed(),
)
}
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
copyCookies()
val domain = domain
val urlBuilder = StringBuilder()
.append("https://api.")
.append(domain)
if (query != null) {
urlBuilder.append("/api/search/?query=")
.append(query.urlEncoded())
} else {
urlBuilder.append("/api/search/catalog/?ordering=")
.append(getSortKey(sortOrder))
tags?.forEach { tag ->
urlBuilder.append("&genres=")
urlBuilder.append(tag.key)
}
}
urlBuilder
.append("&page=")
.append(page)
.append("&count=")
.append(PAGE_SIZE)
val content = webClient.httpGet(urlBuilder.toString()).parseJson()
.getJSONArray("content")
return content.mapJSON { jo ->
val url = "/manga/${jo.getString("dir")}"
val img = jo.getJSONObject("img")
Manga(
id = generateUid(url),
url = url,
publicUrl = "https://$domain$url",
title = jo.getString("rus_name"),
altTitle = jo.getString("en_name"),
rating = jo.getString("avg_rating").toFloatOrNull()?.div(10f) ?: RATING_UNKNOWN,
coverUrl = "https://api.$domain${img.getString("mid")}",
largeCoverUrl = "https://api.$domain${img.getString("high")}",
author = null,
isNsfw = false,
state = null,
tags = jo.optJSONArray("genres")?.mapJSONToSet { g ->
MangaTag(
title = g.getString("name").toTitleCase(),
key = g.getInt("id").toString(),
source = MangaSource.REMANGA,
)
}.orEmpty(),
source = MangaSource.REMANGA,
)
}
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val referer = "https://${getDomain()}/"
val content = context.httpGet(chapter.url.toAbsoluteUrl(getDomain("api")), getApiHeaders())
.handle401()
.parseJson()
.getJSONObject("content")
val pages = content.optJSONArray("pages")
if (pages == null) {
val pubDate = content.getStringOrNull("pub_date")?.let {
SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.US).tryParse(it)
}
if (pubDate != null && pubDate > System.currentTimeMillis()) {
val at = SimpleDateFormat.getDateInstance(DateFormat.LONG).format(Date(pubDate))
throw ContentUnavailableException("Глава станет доступной $at")
} else {
throw ContentUnavailableException("Глава недоступна")
}
}
val result = ArrayList<MangaPage>(pages.length())
for (i in 0 until pages.length()) {
when (val item = pages.get(i)) {
is JSONObject -> result += parsePage(item, referer)
is JSONArray -> item.mapJSONTo(result) { parsePage(it, referer) }
else -> throw ParseException("Unknown json item $item", chapter.url)
}
}
return result
}
override suspend fun getDetails(manga: Manga): Manga {
copyCookies()
val domain = domain
val slug = manga.url.find(regexLastUrlPath)
?: throw ParseException("Cannot obtain slug from ${manga.url}", manga.publicUrl)
val data = webClient.httpGet(
url = "https://api.$domain/api/titles$slug/",
).handle401().parseJson()
val content = try {
data.getJSONObject("content")
} catch (e: JSONException) {
throw ParseException(data.optString("msg"), manga.publicUrl, e)
}
val branchId = content.getJSONArray("branches").optJSONObject(0)
?.getLong("id") ?: throw ParseException("No branches found", manga.publicUrl)
val chapters = grabChapters(domain, branchId)
val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.US)
return manga.copy(
description = content.getString("description"),
state = when (content.optJSONObject("status")?.getInt("id")) {
STATUS_ONGOING -> MangaState.ONGOING
STATUS_FINISHED -> MangaState.FINISHED
else -> null
},
tags = content.getJSONArray("genres").mapJSONToSet { g ->
MangaTag(
title = g.getString("name").toTitleCase(),
key = g.getInt("id").toString(),
source = MangaSource.REMANGA,
)
},
chapters = chapters.mapChapters { i, jo ->
if (
jo.getBooleanOrDefault("is_paid", false) &&
!jo.getBooleanOrDefault("is_bought", false)
) {
return@mapChapters null
}
val id = jo.getLong("id")
val name = jo.getString("name").toTitleCase(Locale.ROOT)
val publishers = jo.optJSONArray("publishers")
MangaChapter(
id = generateUid(id),
url = "/api/titles/chapters/$id/",
number = chapters.size - i,
name = buildString {
append("Том ")
append(jo.optString("tome", "0"))
append(". ")
append("Глава ")
append(jo.optString("chapter", "0"))
if (name.isNotEmpty()) {
append(" - ")
append(name)
}
},
uploadDate = dateFormat.tryParse(jo.getString("upload_date")),
scanlator = publishers?.optJSONObject(0)?.getStringOrNull("name"),
source = MangaSource.REMANGA,
branch = null,
)
}.asReversed(),
)
}
override suspend fun getTags(): Set<MangaTag> {
val domain = getDomain()
val content = context.httpGet("https://api.$domain/api/forms/titles/?get=genres", getApiHeaders())
.parseJson().getJSONObject("content").getJSONArray("genres")
return content.mapJSONToSet { jo ->
MangaTag(
title = jo.getString("name").toTitleCase(),
key = jo.getInt("id").toString(),
source = source,
)
}
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val referer = "https://${domain}/"
val content = webClient.httpGet(chapter.url.toAbsoluteUrl(getDomain("api")))
.handle401()
.parseJson()
.getJSONObject("content")
val pages = content.optJSONArray("pages")
if (pages == null) {
val pubDate = content.getStringOrNull("pub_date")?.let {
SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.US).tryParse(it)
}
if (pubDate != null && pubDate > System.currentTimeMillis()) {
val at = SimpleDateFormat.getDateInstance(DateFormat.LONG).format(Date(pubDate))
throw ContentUnavailableException("Глава станет доступной $at")
} else {
throw ContentUnavailableException("Глава недоступна")
}
}
val result = ArrayList<MangaPage>(pages.length())
for (i in 0 until pages.length()) {
when (val item = pages.get(i)) {
is JSONObject -> result += parsePage(item, referer)
is JSONArray -> item.mapJSONTo(result) { parsePage(it, referer) }
else -> throw ParseException("Unknown json item $item", chapter.url)
}
}
return result
}
override suspend fun getUsername(): String {
val jo = context.httpGet(
url = "https://api.${getDomain()}/api/users/current/",
headers = getApiHeaders(),
).handle401().parseJson()
return jo.getJSONObject("content").getString("username")
}
override suspend fun getTags(): Set<MangaTag> {
val domain = domain
val content = webClient.httpGet("https://api.$domain/api/forms/titles/?get=genres")
.parseJson().getJSONObject("content").getJSONArray("genres")
return content.mapJSONToSet { jo ->
MangaTag(
title = jo.getString("name").toTitleCase(),
key = jo.getInt("id").toString(),
source = source,
)
}
}
private fun getApiHeaders(): Headers {
val userCookie = context.cookieJar.getCookies(getDomain()).find {
it.name == "user"
} ?: return headers
val jo = JSONObject(URLDecoder.decode(userCookie.value, Charsets.UTF_8.name()))
val accessToken = jo.getStringOrNull("access_token") ?: return headers
return headers.newBuilder().add("authorization", "bearer $accessToken").build()
}
override suspend fun getUsername(): String {
val jo = webClient.httpGet(
url = "https://api.${domain}/api/users/current/",
).handle401().parseJson()
return jo.getJSONObject("content").getString("username")
}
private fun copyCookies() {
val domain = getDomain()
context.cookieJar.copyCookies(domain, "api.$domain")
}
private fun getApiHeaders(): Headers {
val userCookie = context.cookieJar.getCookies(domain).find {
it.name == "user"
} ?: return baseHeaders
val jo = JSONObject(URLDecoder.decode(userCookie.value, Charsets.UTF_8.name()))
val accessToken = jo.getStringOrNull("access_token") ?: return baseHeaders
return baseHeaders.newBuilder().add("authorization", "bearer $accessToken").build()
}
private fun getSortKey(order: SortOrder?) = when (order) {
SortOrder.UPDATED -> "-chapter_date"
SortOrder.POPULARITY -> "-rating"
SortOrder.RATING -> "-votes"
SortOrder.NEWEST -> "-id"
else -> "-chapter_date"
}
private fun copyCookies() {
val domain = domain
context.cookieJar.copyCookies(domain, "api.$domain")
}
private fun parsePage(jo: JSONObject, referer: String) = MangaPage(
id = generateUid(jo.getLong("id")),
url = jo.getString("link"),
preview = null,
referer = referer,
source = source,
)
private fun getSortKey(order: SortOrder?) = when (order) {
SortOrder.UPDATED -> "-chapter_date"
SortOrder.POPULARITY -> "-rating"
SortOrder.RATING -> "-votes"
SortOrder.NEWEST -> "-id"
else -> "-chapter_date"
}
private suspend fun grabChapters(domain: String, branchId: Long): List<JSONObject> {
val result = ArrayList<JSONObject>(100)
var page = 1
while (true) {
val content = context.httpGet(
url = "https://api.$domain/api/titles/chapters/?branch_id=$branchId&page=$page&count=100",
headers = getApiHeaders(),
).handle401().parseJson().getJSONArray("content")
val len = content.length()
if (len == 0) {
break
}
result.ensureCapacity(result.size + len)
for (i in 0 until len) {
result.add(content.getJSONObject(i))
}
page++
}
return result
}
private fun parsePage(jo: JSONObject, referer: String) = MangaPage(
id = generateUid(jo.getLong("id")),
url = jo.getString("link"),
preview = null,
referer = referer,
source = source,
)
private fun Response.handle401() = apply {
if (code == HttpURLConnection.HTTP_UNAUTHORIZED) {
throw AuthRequiredException(source)
}
}
}
private suspend fun grabChapters(domain: String, branchId: Long): List<JSONObject> {
val result = ArrayList<JSONObject>(100)
var page = 1
while (true) {
val content = webClient.httpGet(
url = "https://api.$domain/api/titles/chapters/?branch_id=$branchId&page=$page&count=100",
).handle401().parseJson().getJSONArray("content")
val len = content.length()
if (len == 0) {
break
}
result.ensureCapacity(result.size + len)
for (i in 0 until len) {
result.add(content.getJSONObject(i))
}
page++
}
return result
}
private fun Response.handle401() = apply {
if (code == HttpURLConnection.HTTP_UNAUTHORIZED) {
throw AuthRequiredException(source)
}
}
}

306
src/main/kotlin/org/koitharu/kotatsu/parsers/site/UnionMangasParser.kt
Unescape Escape View File

@ -13,165 +13,165 @@ import java.text.SimpleDateFormat
import java.util.*
@MangaSourceParser("UNION_MANGAS", "Union Mangás", "pt")
class UnionMangasParser(override val context: MangaLoaderContext) : PagedMangaParser(MangaSource.UNION_MANGAS, 40) {
class UnionMangasParser(context: MangaLoaderContext) : PagedMangaParser(context, MangaSource.UNION_MANGAS, 40) {
override val sortOrders = EnumSet.of(
SortOrder.ALPHABETICAL,
SortOrder.POPULARITY,
)
override val sortOrders = EnumSet.of(
SortOrder.ALPHABETICAL,
SortOrder.POPULARITY,
)
override val configKeyDomain = ConfigKey.Domain("unionleitor.top", emptyArray())
override val configKeyDomain = ConfigKey.Domain("unionleitor.top", emptyArray())
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
if (!query.isNullOrEmpty()) {
return if (page == searchPaginator.firstPage) {
search(query)
} else {
emptyList()
}
}
val tag = tags.oneOrThrowIfMany()
val url = urlBuilder()
.addPathSegment("lista-mangas")
.addPathSegment(
when {
tag != null -> tag.key
sortOrder == SortOrder.ALPHABETICAL -> "a-z"
else -> "visualizacoes"
},
).addPathSegment(page.toString())
val doc = context.httpGet(url.build()).parseHtml()
val root = doc.selectFirstOrThrow("div.tamanho-bloco-perfil")
return root.select(".lista-mangas-novos").map { div ->
val a = div.selectFirstOrThrow("a")
val img = div.selectFirstOrThrow("img")
val href = a.attrAsRelativeUrl("href")
Manga(
id = generateUid(href),
url = href,
publicUrl = a.attrAsAbsoluteUrl("href"),
title = div.selectLastOrThrow("a").text(),
coverUrl = img.attrAsAbsoluteUrl("src"),
altTitle = null,
rating = RATING_UNKNOWN,
tags = emptySet(),
description = div.selectLast("div")?.ownText(),
state = null,
author = null,
isNsfw = false,
source = source,
)
}
}
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
if (!query.isNullOrEmpty()) {
return if (page == searchPaginator.firstPage) {
search(query)
} else {
emptyList()
}
}
val tag = tags.oneOrThrowIfMany()
val url = urlBuilder()
.addPathSegment("lista-mangas")
.addPathSegment(
when {
tag != null -> tag.key
sortOrder == SortOrder.ALPHABETICAL -> "a-z"
else -> "visualizacoes"
},
).addPathSegment(page.toString())
val doc = webClient.httpGet(url.build()).parseHtml()
val root = doc.selectFirstOrThrow("div.tamanho-bloco-perfil")
return root.select(".lista-mangas-novos").map { div ->
val a = div.selectFirstOrThrow("a")
val img = div.selectFirstOrThrow("img")
val href = a.attrAsRelativeUrl("href")
Manga(
id = generateUid(href),
url = href,
publicUrl = a.attrAsAbsoluteUrl("href"),
title = div.selectLastOrThrow("a").text(),
coverUrl = img.attrAsAbsoluteUrl("src"),
altTitle = null,
rating = RATING_UNKNOWN,
tags = emptySet(),
description = div.selectLast("div")?.ownText(),
state = null,
author = null,
isNsfw = false,
source = source,
)
}
}
override suspend fun getDetails(manga: Manga): Manga {
val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml()
val root = doc.selectFirstOrThrow(".perfil-manga")
val dateFormat = SimpleDateFormat("dd/MM/yyyy", Locale.ROOT)
return manga.copy(
rating = root.select("h2")
.find { it.ownText().startsWith('#') }
?.ownText()?.drop(1)?.toFloatOrNull()?.div(10f) ?: manga.rating,
largeCoverUrl = root.selectFirst("img.img-thumbnail")?.attrAsAbsoluteUrlOrNull("src"),
description = root.selectFirst(".panel-default")?.selectFirst(".panel-body")?.html(),
author = root.tableValue("Autor")?.ownText(),
altTitle = root.tableValue("Título(s) Alternativo(s)")?.ownText(),
state = when (root.tableValue("Status")?.selectLast(".label")?.text()) {
"Completo" -> MangaState.FINISHED
"Ativo" -> MangaState.ONGOING
else -> null
},
tags = root.tableValue("Gênero(s)")?.select("a")?.mapToSet {
it.toMangaTag()
} ?: manga.tags,
isNsfw = root.selectFirst(".alert-danger")?.html()?.contains("18 anos") == true,
chapters = root.select("div.row.capitulos").asReversed().mapChapters { i, div ->
val a = div.selectFirstOrThrow("a")
val href = a.attrAsRelativeUrl("href")
val title = a.text()
MangaChapter(
id = generateUid(href),
name = title,
number = i + 1,
url = href,
scanlator = div.selectLast("a")?.text()?.takeUnless { it == title },
uploadDate = dateFormat.tryParse(
a.nextElementSibling()?.text()?.removeSurrounding("(", ")"),
),
branch = null,
source = source,
)
},
)
}
override suspend fun getDetails(manga: Manga): Manga {
val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml()
val root = doc.selectFirstOrThrow(".perfil-manga")
val dateFormat = SimpleDateFormat("dd/MM/yyyy", Locale.ROOT)
return manga.copy(
rating = root.select("h2")
.find { it.ownText().startsWith('#') }
?.ownText()?.drop(1)?.toFloatOrNull()?.div(10f) ?: manga.rating,
largeCoverUrl = root.selectFirst("img.img-thumbnail")?.attrAsAbsoluteUrlOrNull("src"),
description = root.selectFirst(".panel-default")?.selectFirst(".panel-body")?.html(),
author = root.tableValue("Autor")?.ownText(),
altTitle = root.tableValue("Título(s) Alternativo(s)")?.ownText(),
state = when (root.tableValue("Status")?.selectLast(".label")?.text()) {
"Completo" -> MangaState.FINISHED
"Ativo" -> MangaState.ONGOING
else -> null
},
tags = root.tableValue("Gênero(s)")?.select("a")?.mapToSet {
it.toMangaTag()
} ?: manga.tags,
isNsfw = root.selectFirst(".alert-danger")?.html()?.contains("18 anos") == true,
chapters = root.select("div.row.capitulos").asReversed().mapChapters { i, div ->
val a = div.selectFirstOrThrow("a")
val href = a.attrAsRelativeUrl("href")
val title = a.text()
MangaChapter(
id = generateUid(href),
name = title,
number = i + 1,
url = href,
scanlator = div.selectLast("a")?.text()?.takeUnless { it == title },
uploadDate = dateFormat.tryParse(
a.nextElementSibling()?.text()?.removeSurrounding("(", ")"),
),
branch = null,
source = source,
)
},
)
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(getDomain())
val doc = context.httpGet(fullUrl).parseHtml()
val root = doc.body().selectFirstOrThrow("article")
return root.selectOrThrow("img[pag]").mapNotNull { img ->
val href = img.attrAsRelativeUrl("src")
if (href.startsWith("/images/banner")) {
return@mapNotNull null
}
MangaPage(
id = generateUid(href),
url = href,
referer = fullUrl,
preview = null,
source = source,
)
}
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(domain)
val doc = webClient.httpGet(fullUrl).parseHtml()
val root = doc.body().selectFirstOrThrow("article")
return root.selectOrThrow("img[pag]").mapNotNull { img ->
val href = img.attrAsRelativeUrl("src")
if (href.startsWith("/images/banner")) {
return@mapNotNull null
}
MangaPage(
id = generateUid(href),
url = href,
referer = fullUrl,
preview = null,
source = source,
)
}
}
override suspend fun getTags(): Set<MangaTag> {
val doc = context.httpGet(urlBuilder().addPathSegment("lista-mangas").build()).parseHtml()
val ul = doc.body().selectFirstOrThrow(".nav-tabs").selectFirstOrThrow("ul.dropdown-menu")
return ul.select("li").mapToSet { li ->
li.selectFirstOrThrow("a").toMangaTag()
}
}
override suspend fun getTags(): Set<MangaTag> {
val doc = webClient.httpGet(urlBuilder().addPathSegment("lista-mangas").build()).parseHtml()
val ul = doc.body().selectFirstOrThrow(".nav-tabs").selectFirstOrThrow("ul.dropdown-menu")
return ul.select("li").mapToSet { li ->
li.selectFirstOrThrow("a").toMangaTag()
}
}
private suspend fun search(query: String): List<Manga> {
val domain = getDomain()
val json = context.httpGet(
urlBuilder()
.addPathSegments("assets/busca.php")
.addQueryParameter("nomeManga", query)
.build(),
).parseJson()
return json.getJSONArray("items").mapJSON { jo ->
val href = "/pagina-manga/" + jo.getString("url")
Manga(
id = generateUid(href),
url = href,
publicUrl = href.toAbsoluteUrl(domain),
title = jo.getString("titulo"),
rating = RATING_UNKNOWN,
tags = emptySet(),
author = jo.getStringOrNull("autor"),
coverUrl = jo.getString("imagem"),
state = null,
isNsfw = false,
altTitle = null,
source = source,
)
}
}
private suspend fun search(query: String): List<Manga> {
val domain = domain
val json = webClient.httpGet(
urlBuilder()
.addPathSegments("assets/busca.php")
.addQueryParameter("nomeManga", query)
.build(),
).parseJson()
return json.getJSONArray("items").mapJSON { jo ->
val href = "/pagina-manga/" + jo.getString("url")
Manga(
id = generateUid(href),
url = href,
publicUrl = href.toAbsoluteUrl(domain),
title = jo.getString("titulo"),
rating = RATING_UNKNOWN,
tags = emptySet(),
author = jo.getStringOrNull("autor"),
coverUrl = jo.getString("imagem"),
state = null,
isNsfw = false,
altTitle = null,
source = source,
)
}
}
private fun Element.tableValue(title: String): Element? {
return select("h4.media-heading")
.find { it.selectFirst("label.subtit-manga")?.text()?.contains(title, ignoreCase = true) == true }
}
private fun Element.tableValue(title: String): Element? {
return select("h4.media-heading")
.find { it.selectFirst("label.subtit-manga")?.text()?.contains(title, ignoreCase = true) == true }
}
private fun Element.toMangaTag() = MangaTag(
title = text().toTitleCase(sourceLocale ?: Locale.ROOT),
key = attr("href").removeSuffix('/').substringAfterLast('/'),
source = source,
)
}
private fun Element.toMangaTag() = MangaTag(
title = text().toTitleCase(sourceLocale ?: Locale.ROOT),
key = attr("href").removeSuffix('/').substringAfterLast('/'),
source = source,
)
}

558
src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt
Unescape Escape View File

@ -4,6 +4,7 @@ import okhttp3.Headers
import okhttp3.HttpUrl.Companion.toHttpUrl
import okhttp3.Response
import org.json.JSONArray
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.MangaParserAuthProvider
import org.koitharu.kotatsu.parsers.exception.AuthRequiredException
@ -20,301 +21,298 @@ private const val NSFW_ALERT = "сексуальные сцены"
private const val NOTHING_FOUND = "Ничего не найдено"
internal abstract class GroupleParser(
source: MangaSource,
userAgent: String,
private val siteId: Int,
) : MangaParser(source), MangaParserAuthProvider {
context: MangaLoaderContext,
source: MangaSource,
userAgent: String,
private val siteId: Int,
) : MangaParser(context, source), MangaParserAuthProvider {
override val headers = Headers.Builder()
.add("User-Agent", userAgent)
.build()
override val headers = Headers.Builder()
.add("User-Agent", userAgent)
.build()
override val sortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.UPDATED,
SortOrder.POPULARITY,
SortOrder.NEWEST,
SortOrder.RATING,
)
override val sortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.UPDATED,
SortOrder.POPULARITY,
SortOrder.NEWEST,
SortOrder.RATING,
)
override val authUrl: String
get() {
val targetUri = "https://${getDomain()}/".urlEncoded()
return "https://grouple.co/internal/auth/sso?siteId=$siteId&=targetUri=$targetUri"
}
override val authUrl: String
get() {
val targetUri = "https://${domain}/".urlEncoded()
return "https://grouple.co/internal/auth/sso?siteId=$siteId&=targetUri=$targetUri"
}
override val isAuthorized: Boolean
get() = context.cookieJar.getCookies(getDomain()).any { it.name == "gwt" }
override val isAuthorized: Boolean
get() = context.cookieJar.getCookies(domain).any { it.name == "gwt" }
override suspend fun getList(
offset: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val domain = getDomain()
val doc = when {
!query.isNullOrEmpty() -> context.httpPost(
"https://$domain/search",
mapOf(
"q" to query.urlEncoded(),
"offset" to (offset upBy PAGE_SIZE_SEARCH).toString(),
),
headers,
)
override suspend fun getList(
offset: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val domain = domain
val doc = when {
!query.isNullOrEmpty() -> webClient.httpPost(
"https://$domain/search",
mapOf(
"q" to query.urlEncoded(),
"offset" to (offset upBy PAGE_SIZE_SEARCH).toString(),
),
)
tags.isNullOrEmpty() -> context.httpGet(
"https://$domain/list?sortType=${
getSortKey(sortOrder)
}&offset=${offset upBy PAGE_SIZE}",
headers,
)
tags.isNullOrEmpty() -> webClient.httpGet(
"https://$domain/list?sortType=${
getSortKey(sortOrder)
}&offset=${offset upBy PAGE_SIZE}",
)
tags.size == 1 -> context.httpGet(
"https://$domain/list/genre/${tags.first().key}?sortType=${
getSortKey(sortOrder)
}&offset=${offset upBy PAGE_SIZE}",
headers,
)
tags.size == 1 -> webClient.httpGet(
"https://$domain/list/genre/${tags.first().key}?sortType=${
getSortKey(sortOrder)
}&offset=${offset upBy PAGE_SIZE}",
)
offset > 0 -> return emptyList()
else -> advancedSearch(domain, tags)
}.parseHtml().body()
val root = (doc.getElementById("mangaBox") ?: doc.getElementById("mangaResults"))
?: doc.parseFailed("Cannot find root")
val tiles = root.selectFirst("div.tiles.row") ?: if (
root.select(".alert").any { it.ownText() == NOTHING_FOUND }
) {
return emptyList()
} else {
doc.parseFailed("No tiles found")
}
val baseHost = root.baseUri().toHttpUrl().host
return tiles.select("div.tile").mapNotNull { node ->
val imgDiv = node.selectFirst("div.img") ?: return@mapNotNull null
val descDiv = node.selectFirst("div.desc") ?: return@mapNotNull null
if (descDiv.selectFirst("i.fa-user") != null) {
return@mapNotNull null // skip author
}
val href = imgDiv.selectFirst("a")?.attrAsAbsoluteUrlOrNull("href")
if (href == null || href.toHttpUrl().host != baseHost) {
return@mapNotNull null // skip external links
}
val title = descDiv.selectFirst("h3")?.selectFirst("a")?.text()
?: return@mapNotNull null
val tileInfo = descDiv.selectFirst("div.tile-info")
val relUrl = href.toRelativeUrl(baseHost)
Manga(
id = generateUid(relUrl),
url = relUrl,
publicUrl = href,
title = title,
altTitle = descDiv.selectFirst("h4")?.text(),
coverUrl = imgDiv.selectFirst("img.lazy")?.attr("data-original")?.replace("_p.", ".").orEmpty(),
rating = runCatching {
node.selectFirst(".compact-rate")
?.attr("title")
?.toFloatOrNull()
?.div(5f)
}.getOrNull() ?: RATING_UNKNOWN,
author = tileInfo?.selectFirst("a.person-link")?.text(),
isNsfw = false,
tags = runCatching {
tileInfo?.select("a.element-link")
?.mapToSet {
MangaTag(
title = it.text().toTitleCase(),
key = it.attr("href").substringAfterLast('/'),
source = source,
)
}
}.getOrNull().orEmpty(),
state = when {
node.selectFirst("div.tags")
?.selectFirst("span.mangaCompleted") != null -> MangaState.FINISHED
offset > 0 -> return emptyList()
else -> advancedSearch(domain, tags)
}.parseHtml().body()
val root = (doc.getElementById("mangaBox") ?: doc.getElementById("mangaResults"))
?: doc.parseFailed("Cannot find root")
val tiles = root.selectFirst("div.tiles.row") ?: if (
root.select(".alert").any { it.ownText() == NOTHING_FOUND }
) {
return emptyList()
} else {
doc.parseFailed("No tiles found")
}
val baseHost = root.baseUri().toHttpUrl().host
return tiles.select("div.tile").mapNotNull { node ->
val imgDiv = node.selectFirst("div.img") ?: return@mapNotNull null
val descDiv = node.selectFirst("div.desc") ?: return@mapNotNull null
if (descDiv.selectFirst("i.fa-user") != null) {
return@mapNotNull null // skip author
}
val href = imgDiv.selectFirst("a")?.attrAsAbsoluteUrlOrNull("href")
if (href == null || href.toHttpUrl().host != baseHost) {
return@mapNotNull null // skip external links
}
val title = descDiv.selectFirst("h3")?.selectFirst("a")?.text()
?: return@mapNotNull null
val tileInfo = descDiv.selectFirst("div.tile-info")
val relUrl = href.toRelativeUrl(baseHost)
Manga(
id = generateUid(relUrl),
url = relUrl,
publicUrl = href,
title = title,
altTitle = descDiv.selectFirst("h4")?.text(),
coverUrl = imgDiv.selectFirst("img.lazy")?.attr("data-original")?.replace("_p.", ".").orEmpty(),
rating = runCatching {
node.selectFirst(".compact-rate")
?.attr("title")
?.toFloatOrNull()
?.div(5f)
}.getOrNull() ?: RATING_UNKNOWN,
author = tileInfo?.selectFirst("a.person-link")?.text(),
isNsfw = false,
tags = runCatching {
tileInfo?.select("a.element-link")
?.mapToSet {
MangaTag(
title = it.text().toTitleCase(),
key = it.attr("href").substringAfterLast('/'),
source = source,
)
}
}.getOrNull().orEmpty(),
state = when {
node.selectFirst("div.tags")
?.selectFirst("span.mangaCompleted") != null -> MangaState.FINISHED
else -> null
},
source = source,
)
}
}
else -> null
},
source = source,
)
}
}
override suspend fun getDetails(manga: Manga): Manga {
val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain()), headers).checkAuthRequired().parseHtml()
val root = doc.body().getElementById("mangaBox")?.selectFirst("div.leftContent")
?: doc.parseFailed("Cannot find root")
val dateFormat = SimpleDateFormat("dd.MM.yy", Locale.US)
val coverImg = root.selectFirst("div.subject-cover")?.selectFirst("img")
return manga.copy(
description = root.selectFirst("div.manga-description")?.html(),
largeCoverUrl = coverImg?.attr("data-full"),
coverUrl = coverImg?.attr("data-thumb") ?: manga.coverUrl,
tags = manga.tags + root.select("div.subject-meta").select("span.elem_genre ")
.mapNotNull {
val a = it.selectFirst("a.element-link") ?: return@mapNotNull null
MangaTag(
title = a.text().toTitleCase(),
key = a.attr("href").substringAfterLast('/'),
source = source,
)
},
author = root.selectFirst("a.person-link")?.text() ?: manga.author,
isNsfw = root.select(".alert-warning").any { it.ownText().contains(NSFW_ALERT) },
chapters = root.selectFirst("div.chapters-link")?.selectFirst("table")
?.select("tr:has(td > a)")?.asReversed()?.mapChapters { i, tr ->
val a = tr.selectFirst("a.chapter-link") ?: return@mapChapters null
val href = a.attrAsRelativeUrl("href")
var translators = ""
val translatorElement = a.attr("title")
if (!translatorElement.isNullOrBlank()) {
translators = translatorElement
.replace("(Переводчик),", "&")
.removeSuffix(" (Переводчик)")
}
MangaChapter(
id = generateUid(href),
name = tr.selectFirst("a")?.text().orEmpty().removePrefix(manga.title).trim(),
number = i + 1,
url = href,
uploadDate = dateFormat.tryParse(tr.selectFirst("td.date")?.text()),
scanlator = translators,
source = source,
branch = null,
)
},
)
}
override suspend fun getDetails(manga: Manga): Manga {
val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).checkAuthRequired().parseHtml()
val root = doc.body().getElementById("mangaBox")?.selectFirst("div.leftContent")
?: doc.parseFailed("Cannot find root")
val dateFormat = SimpleDateFormat("dd.MM.yy", Locale.US)
val coverImg = root.selectFirst("div.subject-cover")?.selectFirst("img")
return manga.copy(
description = root.selectFirst("div.manga-description")?.html(),
largeCoverUrl = coverImg?.attr("data-full"),
coverUrl = coverImg?.attr("data-thumb") ?: manga.coverUrl,
tags = manga.tags + root.select("div.subject-meta").select("span.elem_genre ")
.mapNotNull {
val a = it.selectFirst("a.element-link") ?: return@mapNotNull null
MangaTag(
title = a.text().toTitleCase(),
key = a.attr("href").substringAfterLast('/'),
source = source,
)
},
author = root.selectFirst("a.person-link")?.text() ?: manga.author,
isNsfw = root.select(".alert-warning").any { it.ownText().contains(NSFW_ALERT) },
chapters = root.selectFirst("div.chapters-link")?.selectFirst("table")
?.select("tr:has(td > a)")?.asReversed()?.mapChapters { i, tr ->
val a = tr.selectFirst("a.chapter-link") ?: return@mapChapters null
val href = a.attrAsRelativeUrl("href")
var translators = ""
val translatorElement = a.attr("title")
if (!translatorElement.isNullOrBlank()) {
translators = translatorElement
.replace("(Переводчик),", "&")
.removeSuffix(" (Переводчик)")
}
MangaChapter(
id = generateUid(href),
name = tr.selectFirst("a")?.text().orEmpty().removePrefix(manga.title).trim(),
number = i + 1,
url = href,
uploadDate = dateFormat.tryParse(tr.selectFirst("td.date")?.text()),
scanlator = translators,
source = source,
branch = null,
)
},
)
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val doc = context.httpGet(chapter.url.toAbsoluteUrl(getDomain()) + "?mtr=1", headers)
.checkAuthRequired()
.parseHtml()
val scripts = doc.select("script")
for (script in scripts) {
val data = script.html()
val pos = data.indexOf("rm_h.readerInit( 0,")
if (pos == -1) {
continue
}
val json = data.substring(pos)
.substringAfter('(')
.substringBefore('\n')
.substringBeforeLast(')')
if (json.isEmpty()) {
continue
}
val ja = JSONArray("[$json]")
val pages = ja.getJSONArray(1)
val servers = ja.getJSONArray(3).mapJSON { it.getString("path") }
val serversStr = servers.joinToString("|")
return (0 until pages.length()).map { i ->
val page = pages.getJSONArray(i)
val primaryServer = page.getString(0)
val url = page.getString(2)
MangaPage(
id = generateUid(url),
url = "$primaryServer|$serversStr|$url",
preview = null,
referer = chapter.url,
source = source,
)
}
}
doc.parseFailed("Pages list not found at ${chapter.url}")
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain) + "?mtr=1")
.checkAuthRequired()
.parseHtml()
val scripts = doc.select("script")
for (script in scripts) {
val data = script.html()
val pos = data.indexOf("rm_h.readerInit( 0,")
if (pos == -1) {
continue
}
val json = data.substring(pos)
.substringAfter('(')
.substringBefore('\n')
.substringBeforeLast(')')
if (json.isEmpty()) {
continue
}
val ja = JSONArray("[$json]")
val pages = ja.getJSONArray(1)
val servers = ja.getJSONArray(3).mapJSON { it.getString("path") }
val serversStr = servers.joinToString("|")
return (0 until pages.length()).map { i ->
val page = pages.getJSONArray(i)
val primaryServer = page.getString(0)
val url = page.getString(2)
MangaPage(
id = generateUid(url),
url = "$primaryServer|$serversStr|$url",
preview = null,
referer = chapter.url,
source = source,
)
}
}
doc.parseFailed("Pages list not found at ${chapter.url}")
}
override suspend fun getPageUrl(page: MangaPage): String {
val parts = page.url.split('|')
val path = parts.last()
val servers = parts.dropLast(1).toSet()
val headers = Headers.headersOf("Referer", page.referer)
for (server in servers) {
val url = server + path
if (tryHead(url, headers)) {
return url
}
}
val fallbackServer = servers.firstOrNull() ?: throw ParseException("Cannot find any page url", page.url)
return fallbackServer + path
}
override suspend fun getPageUrl(page: MangaPage): String {
val parts = page.url.split('|')
val path = parts.last()
val servers = parts.dropLast(1).toSet()
for (server in servers) {
val url = server + path
if (tryHead(url)) {
return url
}
}
val fallbackServer = servers.firstOrNull() ?: throw ParseException("Cannot find any page url", page.url)
return fallbackServer + path
}
override suspend fun getTags(): Set<MangaTag> {
val doc = context.httpGet("https://${getDomain()}/list/genres/sort_name", headers).parseHtml()
val root = doc.body().getElementById("mangaBox")?.selectFirst("div.leftContent")
?.selectFirst("table.table") ?: doc.parseFailed("Cannot find root")
return root.select("a.element-link").mapToSet { a ->
MangaTag(
title = a.text().toTitleCase(),
key = a.attr("href").substringAfterLast('/'),
source = source,
)
}
}
override suspend fun getTags(): Set<MangaTag> {
val doc = webClient.httpGet("https://${domain}/list/genres/sort_name").parseHtml()
val root = doc.body().getElementById("mangaBox")?.selectFirst("div.leftContent")
?.selectFirst("table.table") ?: doc.parseFailed("Cannot find root")
return root.select("a.element-link").mapToSet { a ->
MangaTag(
title = a.text().toTitleCase(),
key = a.attr("href").substringAfterLast('/'),
source = source,
)
}
}
override suspend fun getUsername(): String {
val root = context.httpGet("https://grouple.co/").parseHtml().body()
val element = root.selectFirst("img.user-avatar") ?: throw AuthRequiredException(source)
val res = element.parent()?.text()
return if (res.isNullOrEmpty()) {
root.parseFailed("Cannot find username")
} else res
}
override suspend fun getUsername(): String {
val root = webClient.httpGet("https://grouple.co/").parseHtml().body()
val element = root.selectFirst("img.user-avatar") ?: throw AuthRequiredException(source)
val res = element.parent()?.text()
return if (res.isNullOrEmpty()) {
root.parseFailed("Cannot find username")
} else res
}
private fun getSortKey(sortOrder: SortOrder) =
when (sortOrder) {
SortOrder.ALPHABETICAL -> "name"
SortOrder.POPULARITY -> "rate"
SortOrder.UPDATED -> "updated"
SortOrder.NEWEST -> "created"
SortOrder.RATING -> "votes"
}
private fun getSortKey(sortOrder: SortOrder) =
when (sortOrder) {
SortOrder.ALPHABETICAL -> "name"
SortOrder.POPULARITY -> "rate"
SortOrder.UPDATED -> "updated"
SortOrder.NEWEST -> "created"
SortOrder.RATING -> "votes"
}
private suspend fun advancedSearch(domain: String, tags: Set<MangaTag>): Response {
val url = "https://$domain/search/advanced"
// Step 1: map catalog genres names to advanced-search genres ids
val tagsIndex = context.httpGet(url, headers).parseHtml()
.body().selectFirst("form.search-form")
?.select("div.form-group")
?.get(1) ?: throw ParseException("Genres filter element not found", url)
val tagNames = tags.map { it.title.lowercase() }
val payload = HashMap<String, String>()
var foundGenres = 0
tagsIndex.select("li.property").forEach { li ->
val name = li.text().trim().lowercase()
val id = li.selectFirst("input")?.id()
?: li.parseFailed("Id for tag $name not found")
payload[id] = if (name in tagNames) {
foundGenres++
"in"
} else ""
}
if (foundGenres != tags.size) {
tagsIndex.parseFailed("Some genres are not found")
}
// Step 2: advanced search
payload["q"] = ""
payload["s_high_rate"] = ""
payload["s_single"] = ""
payload["s_mature"] = ""
payload["s_completed"] = ""
payload["s_translated"] = ""
payload["s_many_chapters"] = ""
payload["s_wait_upload"] = ""
payload["s_sale"] = ""
payload["years"] = "1900,2099"
payload["+"] = "Искать".urlEncoded()
return context.httpPost(url, payload, headers)
}
private suspend fun advancedSearch(domain: String, tags: Set<MangaTag>): Response {
val url = "https://$domain/search/advanced"
// Step 1: map catalog genres names to advanced-search genres ids
val tagsIndex = webClient.httpGet(url).parseHtml()
.body().selectFirst("form.search-form")
?.select("div.form-group")
?.get(1) ?: throw ParseException("Genres filter element not found", url)
val tagNames = tags.map { it.title.lowercase() }
val payload = HashMap<String, String>()
var foundGenres = 0
tagsIndex.select("li.property").forEach { li ->
val name = li.text().trim().lowercase()
val id = li.selectFirst("input")?.id()
?: li.parseFailed("Id for tag $name not found")
payload[id] = if (name in tagNames) {
foundGenres++
"in"
} else ""
}
if (foundGenres != tags.size) {
tagsIndex.parseFailed("Some genres are not found")
}
// Step 2: advanced search
payload["q"] = ""
payload["s_high_rate"] = ""
payload["s_single"] = ""
payload["s_mature"] = ""
payload["s_completed"] = ""
payload["s_translated"] = ""
payload["s_many_chapters"] = ""
payload["s_wait_upload"] = ""
payload["s_sale"] = ""
payload["years"] = "1900,2099"
payload["+"] = "Искать".urlEncoded()
return webClient.httpPost(url, payload)
}
private suspend fun tryHead(url: String, headers: Headers): Boolean = runCatchingCancellable {
context.httpHead(url, headers).isSuccessful
}.getOrDefault(false)
private suspend fun tryHead(url: String): Boolean = runCatchingCancellable {
webClient.httpHead(url).isSuccessful
}.getOrDefault(false)
private fun Response.checkAuthRequired(): Response {
val lastPathSegment = request.url.pathSegments.lastOrNull() ?: return this
if (lastPathSegment == "login") {
throw AuthRequiredException(source)
}
return this
}
}
private fun Response.checkAuthRequired(): Response {
val lastPathSegment = request.url.pathSegments.lastOrNull() ?: return this
if (lastPathSegment == "login") {
throw AuthRequiredException(source)
}
return this
}
}

17
src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/MintMangaParser.kt
Unescape Escape View File

@ -7,15 +7,12 @@ import org.koitharu.kotatsu.parsers.model.MangaSource
@MangaSourceParser("MINTMANGA", "MintManga", "ru")
internal class MintMangaParser(
override val context: MangaLoaderContext,
) : GroupleParser(MangaSource.MINTMANGA, "mintmangafun", 2) {
context: MangaLoaderContext,
) : GroupleParser(context, MangaSource.MINTMANGA, "mintmangafun", 2) {
override val configKeyDomain = ConfigKey.Domain(
"mintmanga.live",
arrayOf("mintmanga.live", "mintmanga.com"),
)
override val configKeyDomain = ConfigKey.Domain(
"mintmanga.live",
arrayOf("mintmanga.live", "mintmanga.com"),
)
override fun getFaviconUrl(): String {
return "https://resmm.rmr.rocks/static/apple-touch-icon-8fff291039c140493adb0c7ba81065ad.png"
}
}
}

17
src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/ReadmangaParser.kt
Unescape Escape View File

@ -7,15 +7,12 @@ import org.koitharu.kotatsu.parsers.model.MangaSource
@MangaSourceParser("READMANGA_RU", "ReadManga", "ru")
internal class ReadmangaParser(
override val context: MangaLoaderContext,
) : GroupleParser(MangaSource.READMANGA_RU, "readmangafun", 1) {
context: MangaLoaderContext,
) : GroupleParser(context, MangaSource.READMANGA_RU, "readmangafun", 1) {
override val configKeyDomain = ConfigKey.Domain(
"readmanga.live",
arrayOf("readmanga.io", "readmanga.live", "readmanga.me"),
)
override val configKeyDomain = ConfigKey.Domain(
"readmanga.live",
arrayOf("readmanga.io", "readmanga.live", "readmanga.me"),
)
override fun getFaviconUrl(): String {
return "https://resrm.rmr.rocks/static/apple-touch-icon-3162037c9df9f28dca0f9a4092cb0f65.png"
}
}
}

11
src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/SelfMangaParser.kt
Unescape Escape View File

@ -7,12 +7,9 @@ import org.koitharu.kotatsu.parsers.model.MangaSource
@MangaSourceParser("SELFMANGA", "SelfManga", "ru")
internal class SelfMangaParser(
override val context: MangaLoaderContext,
) : GroupleParser(MangaSource.SELFMANGA, "selfmangafun", 3) {
context: MangaLoaderContext,
) : GroupleParser(context, MangaSource.SELFMANGA, "selfmangafun", 3) {
override val configKeyDomain = ConfigKey.Domain("selfmanga.live", null)
override val configKeyDomain = ConfigKey.Domain("selfmanga.live", null)
override fun getFaviconUrl(): String {
return "https://ressm.rmr.rocks/static/apple-touch-icon-a769ea533d811b73ac3eedde658bb1d3.png"
}
}
}

62
src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/IsekaiScanEu.kt
Unescape Escape View File

@ -12,36 +12,36 @@ import java.util.*
@MangaSourceParser("ISEKAISCAN_EU", "IsekaiScan (eu)", "en")
internal class IsekaiScanEuParser(context: MangaLoaderContext) :
MadaraParser(context, MangaSource.ISEKAISCAN_EU, "isekaiscan.eu") {
MadaraParser(context, MangaSource.ISEKAISCAN_EU, "isekaiscan.eu") {
override val datePattern = "MM/dd/yyyy"
override val datePattern = "MM/dd/yyyy"
override suspend fun getChapters(manga: Manga, doc: Document): List<MangaChapter> {
val mangaId = doc.body().requireElementById("manga-chapters-holder").attr("data-id")
val ul = context.httpPost(
"https://${getDomain()}/wp-admin/admin-ajax.php",
mapOf(
"action" to "manga_get_chapters",
"manga" to mangaId,
),
).parseHtml().body().selectFirstOrThrow("ul")
val dateFormat = SimpleDateFormat(datePattern, Locale.US)
return ul.select("li").asReversed().mapChapters { i, li ->
val a = li.selectFirst("a")
val href = a?.attrAsRelativeUrlOrNull("href") ?: li.parseFailed("Link is missing")
MangaChapter(
id = generateUid(href),
name = a.ownText(),
number = i + 1,
url = href,
uploadDate = parseChapterDate(
dateFormat,
li.selectFirst("span.chapter-release-date i")?.text(),
),
source = source,
scanlator = null,
branch = null,
)
}
}
}
override suspend fun getChapters(manga: Manga, doc: Document): List<MangaChapter> {
val mangaId = doc.body().requireElementById("manga-chapters-holder").attr("data-id")
val ul = webClient.httpPost(
"https://${domain}/wp-admin/admin-ajax.php",
mapOf(
"action" to "manga_get_chapters",
"manga" to mangaId,
),
).parseHtml().body().selectFirstOrThrow("ul")
val dateFormat = SimpleDateFormat(datePattern, Locale.US)
return ul.select("li").asReversed().mapChapters { i, li ->
val a = li.selectFirst("a")
val href = a?.attrAsRelativeUrlOrNull("href") ?: li.parseFailed("Link is missing")
MangaChapter(
id = generateUid(href),
name = a.ownText(),
number = i + 1,
url = href,
uploadDate = parseChapterDate(
dateFormat,
li.selectFirst("span.chapter-release-date i")?.text(),
),
source = source,
scanlator = null,
branch = null,
)
}
}
}

341
src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Madara5Parser.kt
Unescape Escape View File

@ -13,175 +13,174 @@ import java.text.SimpleDateFormat
import java.util.*
abstract class Madara5Parser @InternalParsersApi constructor(
override val context: MangaLoaderContext,
source: MangaSource,
domain: String,
) : PagedMangaParser(source, pageSize = 22) {
protected open val datePattern = "MMMM dd, HH:mm"
protected open val tagPrefix = "/mangas/"
protected open val nsfwTags = arraySetOf("yaoi", "yuri", "mature")
override val sortOrders: Set<SortOrder> = EnumSet.of(SortOrder.UPDATED)
override val configKeyDomain = ConfigKey.Domain(domain, null)
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val domain = getDomain()
val url = buildString {
append("https://")
append(domain)
append("/search?s=")
if (!query.isNullOrEmpty()) {
append(query.urlEncoded())
}
append("&post_type=wp-manga")
if (!tags.isNullOrEmpty()) {
for (tag in tags) {
append("&genre%5B%5D=")
append(tag.key)
}
}
append("&op=1&author=&artist=&page=")
append(page)
}
val root = context.httpGet(url).parseHtml().body().selectFirstOrThrow(".search-wrap")
return root.select(".c-tabs-item__content").map { div ->
val a = div.selectFirstOrThrow("a")
val img = div.selectLastOrThrow("img")
val href = a.attrAsRelativeUrl("href")
val postContent = root.selectFirstOrThrow(".post-content")
val tagSet = postContent.getElementsContainingOwnText("Genre")
.firstOrNull()?.tableValue()
?.getElementsByAttributeValueContaining("href", tagPrefix)
?.mapToSet { it.asMangaTag() }.orEmpty()
Manga(
id = generateUid(href),
title = a.attr("title"),
altTitle = postContent.getElementsContainingOwnText("Alternative")
.firstOrNull()?.tableValue()?.text()?.trim(),
url = href,
publicUrl = a.attrAsAbsoluteUrl("href"),
coverUrl = img.src().orEmpty(),
author = postContent.getElementsContainingOwnText("Author")
.firstOrNull()?.tableValue()?.text()?.trim(),
state = postContent.getElementsContainingOwnText("Status")
.firstOrNull()?.tableValue()?.text()?.asMangaState(),
isNsfw = isNsfw(tagSet),
rating = div.selectFirstOrThrow(".score").text()
.toFloatOrNull()?.div(5f) ?: RATING_UNKNOWN,
tags = tagSet,
source = source,
)
}
}
override suspend fun getDetails(manga: Manga): Manga {
val root = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml().body()
.selectFirstOrThrow(".site-content")
val postContent = root.selectFirstOrThrow(".post-content")
val tags = postContent.getElementsContainingOwnText("Genre")
.firstOrNull()?.tableValue()
?.getElementsByAttributeValueContaining("href", tagPrefix)
?.mapToSet { a -> a.asMangaTag() } ?: manga.tags
val mangaId = root.getElementById("manga-chapters-holder")?.attr("data-id")?.toLongOrNull()
?: root.parseFailed("Cannot find mangaId")
return manga.copy(
description = (root.selectFirst(".detail-content")
?: root.selectFirstOrThrow(".description-summary")).html(),
author = postContent.getElementsContainingOwnText("Author")
.firstOrNull()?.tableValue()?.text()?.trim(),
state = postContent.getElementsContainingOwnText("Status")
.firstOrNull()?.tableValue()?.text()?.asMangaState(),
tags = tags,
isNsfw = isNsfw(tags),
chapters = loadChapters(mangaId),
)
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(getDomain())
val doc = context.httpGet(fullUrl).parseHtml()
val arrayData = doc.getElementById("arraydata") ?: doc.parseFailed("#arraydata not found")
return arrayData.html().split(',').map { url ->
MangaPage(
id = generateUid(url),
url = url,
referer = fullUrl,
preview = null,
source = source,
)
}
}
override suspend fun getTags(): Set<MangaTag> {
val doc = context.httpGet("http://${getDomain()}/").parseHtml().body()
return doc.getElementsByAttributeValueContaining("href", tagPrefix)
.mapToSet { it.asMangaTag() }
}
private suspend fun loadChapters(mangaId: Long): List<MangaChapter> {
val dateFormat = SimpleDateFormat(datePattern, sourceLocale ?: Locale.US)
val doc = context.httpGet("https://${getDomain()}/ajax-list-chapter?mangaID=$mangaId").parseHtml()
return doc.select("li.wp-manga-chapter").asReversed().mapChapters { i, li ->
val a = li.selectFirstOrThrow("a")
val href = a.attrAsRelativeUrl("href")
MangaChapter(
id = generateUid(href),
url = href,
name = a.text(),
number = i + 1,
branch = null,
uploadDate = dateFormat.tryParse(
li.selectFirst(".chapter-release-date")?.text()?.trim(),
),
scanlator = null,
source = source,
)
}
}
protected fun isNsfw(tags: Set<MangaTag>): Boolean {
return tags.any { it.key in nsfwTags }
}
private fun Element.src(): String? {
return absUrl("data-src").ifEmpty {
absUrl("src")
}.takeUnless { it.isEmpty() }
}
private fun Element.tableValue(): Element {
for (p in parents()) {
val children = p.children()
if (children.size == 2) {
return children[1]
}
}
parseFailed("Cannot find tableValue for node ${text()}")
}
private fun String.asMangaState() = when (trim().lowercase(sourceLocale ?: Locale.US)) {
"ongoing" -> MangaState.ONGOING
"completed" -> MangaState.FINISHED
else -> null
}
private fun Element.asMangaTag() = MangaTag(
title = ownText(),
key = attr("href").removeSuffix('/').substringAfterLast('/')
.replace('-', '+'),
source = source,
)
@MangaSourceParser("MANGAOWLS", "BeautyManga", "en")
class BeautyManga(context: MangaLoaderContext) : Madara5Parser(context, MangaSource.MANGAOWLS, "beautymanga.com") {
override fun getFaviconUrl() = "http://${getDomain()}/frontend/images/favico.png"
}
context: MangaLoaderContext,
source: MangaSource,
domain: String,
) : PagedMangaParser(context, source, pageSize = 22) {
protected open val datePattern = "MMMM dd, HH:mm"
protected open val tagPrefix = "/mangas/"
protected open val nsfwTags = arraySetOf("yaoi", "yuri", "mature")
override val sortOrders: Set<SortOrder> = EnumSet.of(SortOrder.UPDATED)
override val configKeyDomain = ConfigKey.Domain(domain, null)
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val domain = domain
val url = buildString {
append("https://")
append(domain)
append("/search?s=")
if (!query.isNullOrEmpty()) {
append(query.urlEncoded())
}
append("&post_type=wp-manga")
if (!tags.isNullOrEmpty()) {
for (tag in tags) {
append("&genre%5B%5D=")
append(tag.key)
}
}
append("&op=1&author=&artist=&page=")
append(page)
}
val root = webClient.httpGet(url).parseHtml().body().selectFirstOrThrow(".search-wrap")
return root.select(".c-tabs-item__content").map { div ->
val a = div.selectFirstOrThrow("a")
val img = div.selectLastOrThrow("img")
val href = a.attrAsRelativeUrl("href")
val postContent = root.selectFirstOrThrow(".post-content")
val tagSet = postContent.getElementsContainingOwnText("Genre")
.firstOrNull()?.tableValue()
?.getElementsByAttributeValueContaining("href", tagPrefix)
?.mapToSet { it.asMangaTag() }.orEmpty()
Manga(
id = generateUid(href),
title = a.attr("title"),
altTitle = postContent.getElementsContainingOwnText("Alternative")
.firstOrNull()?.tableValue()?.text()?.trim(),
url = href,
publicUrl = a.attrAsAbsoluteUrl("href"),
coverUrl = img.src().orEmpty(),
author = postContent.getElementsContainingOwnText("Author")
.firstOrNull()?.tableValue()?.text()?.trim(),
state = postContent.getElementsContainingOwnText("Status")
.firstOrNull()?.tableValue()?.text()?.asMangaState(),
isNsfw = isNsfw(tagSet),
rating = div.selectFirstOrThrow(".score").text()
.toFloatOrNull()?.div(5f) ?: RATING_UNKNOWN,
tags = tagSet,
source = source,
)
}
}
override suspend fun getDetails(manga: Manga): Manga {
val root = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml().body()
.selectFirstOrThrow(".site-content")
val postContent = root.selectFirstOrThrow(".post-content")
val tags = postContent.getElementsContainingOwnText("Genre")
.firstOrNull()?.tableValue()
?.getElementsByAttributeValueContaining("href", tagPrefix)
?.mapToSet { a -> a.asMangaTag() } ?: manga.tags
val mangaId = root.getElementById("manga-chapters-holder")?.attr("data-id")?.toLongOrNull()
?: root.parseFailed("Cannot find mangaId")
return manga.copy(
description = (root.selectFirst(".detail-content")
?: root.selectFirstOrThrow(".description-summary")).html(),
author = postContent.getElementsContainingOwnText("Author")
.firstOrNull()?.tableValue()?.text()?.trim(),
state = postContent.getElementsContainingOwnText("Status")
.firstOrNull()?.tableValue()?.text()?.asMangaState(),
tags = tags,
isNsfw = isNsfw(tags),
chapters = loadChapters(mangaId),
)
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(domain)
val doc = webClient.httpGet(fullUrl).parseHtml()
val arrayData = doc.getElementById("arraydata") ?: doc.parseFailed("#arraydata not found")
return arrayData.html().split(',').map { url ->
MangaPage(
id = generateUid(url),
url = url,
referer = fullUrl,
preview = null,
source = source,
)
}
}
override suspend fun getTags(): Set<MangaTag> {
val doc = webClient.httpGet("http://${domain}/").parseHtml().body()
return doc.getElementsByAttributeValueContaining("href", tagPrefix)
.mapToSet { it.asMangaTag() }
}
private suspend fun loadChapters(mangaId: Long): List<MangaChapter> {
val dateFormat = SimpleDateFormat(datePattern, sourceLocale ?: Locale.US)
val doc = webClient.httpGet("https://${domain}/ajax-list-chapter?mangaID=$mangaId").parseHtml()
return doc.select("li.wp-manga-chapter").asReversed().mapChapters { i, li ->
val a = li.selectFirstOrThrow("a")
val href = a.attrAsRelativeUrl("href")
MangaChapter(
id = generateUid(href),
url = href,
name = a.text(),
number = i + 1,
branch = null,
uploadDate = dateFormat.tryParse(
li.selectFirst(".chapter-release-date")?.text()?.trim(),
),
scanlator = null,
source = source,
)
}
}
protected fun isNsfw(tags: Set<MangaTag>): Boolean {
return tags.any { it.key in nsfwTags }
}
private fun Element.src(): String? {
return absUrl("data-src").ifEmpty {
absUrl("src")
}.takeUnless { it.isEmpty() }
}
private fun Element.tableValue(): Element {
for (p in parents()) {
val children = p.children()
if (children.size == 2) {
return children[1]
}
}
parseFailed("Cannot find tableValue for node ${text()}")
}
private fun String.asMangaState() = when (trim().lowercase(sourceLocale ?: Locale.US)) {
"ongoing" -> MangaState.ONGOING
"completed" -> MangaState.FINISHED
else -> null
}
private fun Element.asMangaTag() = MangaTag(
title = ownText(),
key = attr("href").removeSuffix('/').substringAfterLast('/')
.replace('-', '+'),
source = source,
)
@MangaSourceParser("MANGAOWLS", "BeautyManga", "en")
class BeautyManga(context: MangaLoaderContext) : Madara5Parser(context, MangaSource.MANGAOWLS, "beautymanga.com") {
}
}

100
src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Madara6Parser.kt
Unescape Escape View File

@ -10,61 +10,61 @@ import java.text.SimpleDateFormat
import java.util.*
internal abstract class Madara6Parser(
context: MangaLoaderContext,
source: MangaSource,
domain: String,
context: MangaLoaderContext,
source: MangaSource,
domain: String,
) : MadaraParser(context, source, domain) {
override val datePattern: String = "dd MMMM yyyy"
override val datePattern: String = "dd MMMM yyyy"
override suspend fun getDetails(manga: Manga): Manga {
return coroutineScope {
val chapters = async { loadChapters(manga.url) }
val body = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml().body()
parseDetails(manga, body, chapters.await())
}
}
override suspend fun getDetails(manga: Manga): Manga {
return coroutineScope {
val chapters = async { loadChapters(manga.url) }
val body = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml().body()
parseDetails(manga, body, chapters.await())
}
}
protected fun Element.tableValue(): Element {
for (p in parents()) {
val children = p.children()
if (children.size == 2) {
return children[1]
}
}
parseFailed("Cannot find tableValue for node ${text()}")
}
protected fun Element.tableValue(): Element {
for (p in parents()) {
val children = p.children()
if (children.size == 2) {
return children[1]
}
}
parseFailed("Cannot find tableValue for node ${text()}")
}
protected abstract fun String.asMangaState(): MangaState?
protected abstract fun String.asMangaState(): MangaState?
protected fun Element.asMangaTag() = MangaTag(
title = ownText(),
key = attr("href").removeSuffix('/').substringAfterLast('/')
.replace('-', '+'),
source = source,
)
protected fun Element.asMangaTag() = MangaTag(
title = ownText(),
key = attr("href").removeSuffix('/').substringAfterLast('/')
.replace('-', '+'),
source = source,
)
protected open suspend fun loadChapters(mangaUrl: String): List<MangaChapter> {
val url = mangaUrl.toAbsoluteUrl(getDomain()).removeSuffix('/') + "/ajax/chapters/"
val dateFormat = SimpleDateFormat(datePattern, sourceLocale ?: Locale.ROOT)
val doc = context.httpPost(url, emptyMap()).parseHtml()
return doc.select("li.wp-manga-chapter").asReversed().mapChapters { i, li ->
val a = li.selectFirstOrThrow("a")
val href = a.attrAsRelativeUrl("href")
MangaChapter(
id = generateUid(href),
url = href,
name = a.text(),
number = i + 1,
branch = null,
uploadDate = dateFormat.tryParse(
li.selectFirst(".chapter-release-date")?.text()?.trim(),
),
scanlator = null,
source = source,
)
}
}
protected open suspend fun loadChapters(mangaUrl: String): List<MangaChapter> {
val url = mangaUrl.toAbsoluteUrl(domain).removeSuffix('/') + "/ajax/chapters/"
val dateFormat = SimpleDateFormat(datePattern, sourceLocale ?: Locale.ROOT)
val doc = webClient.httpPost(url, emptyMap()).parseHtml()
return doc.select("li.wp-manga-chapter").asReversed().mapChapters { i, li ->
val a = li.selectFirstOrThrow("a")
val href = a.attrAsRelativeUrl("href")
MangaChapter(
id = generateUid(href),
url = href,
name = a.text(),
number = i + 1,
branch = null,
uploadDate = dateFormat.tryParse(
li.selectFirst(".chapter-release-date")?.text()?.trim(),
),
scanlator = null,
source = source,
)
}
}
protected abstract fun parseDetails(manga: Manga, body: Element, chapters: List<MangaChapter>): Manga
}
protected abstract fun parseDetails(manga: Manga, body: Element, chapters: List<MangaChapter>): Manga
}

927
src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt
Unescape Escape View File

@ -16,503 +16,482 @@ import java.text.SimpleDateFormat
import java.util.*
internal abstract class MadaraParser(
override val context: MangaLoaderContext,
source: MangaSource,
domain: String,
) : PagedMangaParser(source, pageSize = 12) {
override val configKeyDomain = ConfigKey.Domain(domain, null)
override val sortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.UPDATED,
SortOrder.POPULARITY,
)
protected open val tagPrefix = "manga-genre/"
protected open val isNsfwSource = false
protected open val datePattern = "MMMM dd, yyyy"
init {
paginator.firstPage = 0
searchPaginator.firstPage = 0
}
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val tag = tags.oneOrThrowIfMany()
val payload = createRequestTemplate()
payload["page"] = page.toString()
payload["vars[meta_key]"] = when (sortOrder) {
SortOrder.POPULARITY -> "_wp_manga_views"
SortOrder.UPDATED -> "_latest_update"
else -> "_wp_manga_views"
}
payload["vars[wp-manga-genre]"] = tag?.key.orEmpty()
payload["vars[s]"] = query?.urlEncoded().orEmpty()
val doc = context.httpPost(
"https://${getDomain()}/wp-admin/admin-ajax.php",
payload,
).parseHtml()
return doc.select("div.row.c-tabs-item__content").ifEmpty {
doc.select("div.page-item-detail.manga")
}.map { div ->
val href = div.selectFirst("a")?.attrAsRelativeUrlOrNull("href")
?: div.parseFailed("Link not found")
val summary = div.selectFirst(".tab-summary") ?: div.selectFirst(".item-summary")
Manga(
id = generateUid(href),
url = href,
publicUrl = href.toAbsoluteUrl(div.host ?: getDomain()),
coverUrl = div.selectFirst("img")?.src().orEmpty(),
title = (summary?.selectFirst("h3") ?: summary?.selectFirst("h4"))?.text().orEmpty(),
altTitle = null,
rating = div.selectFirst("span.total_votes")?.ownText()
?.toFloatOrNull()?.div(5f) ?: -1f,
tags = summary?.selectFirst(".mg_genres")?.select("a")?.mapToSet { a ->
MangaTag(
key = a.attr("href").removeSuffix("/").substringAfterLast('/'),
title = a.text().toTitleCase(),
source = source,
)
}.orEmpty(),
author = summary?.selectFirst(".mg_author")?.selectFirst("a")?.ownText(),
state = when (
summary?.selectFirst(".mg_status")?.selectFirst(".summary-content")
?.ownText()?.trim()?.lowercase()
) {
"ongoing" -> MangaState.ONGOING
"completed" -> MangaState.FINISHED
else -> null
},
source = source,
isNsfw = isNsfwSource,
)
}
}
override suspend fun getTags(): Set<MangaTag> {
val doc = context.httpGet("https://${getDomain()}/manga/").parseHtml()
val body = doc.body()
val root1 = body.selectFirst("header")?.selectFirst("ul.second-menu")
val root2 = body.selectFirst("div.genres_wrap")?.selectFirst("ul.list-unstyled")
if (root1 == null && root2 == null) {
doc.parseFailed("Root not found")
}
val list = root1?.select("li").orEmpty() + root2?.select("li").orEmpty()
val keySet = HashSet<String>(list.size)
return list.mapNotNullToSet { li ->
val a = li.selectFirst("a") ?: return@mapNotNullToSet null
val href = a.attr("href").removeSuffix("/")
.substringAfterLast(tagPrefix, "")
if (href.isEmpty() || !keySet.add(href)) {
return@mapNotNullToSet null
}
MangaTag(
key = href,
title = a.ownText().trim().ifEmpty {
a.selectFirst(".menu-image-title")?.text()?.trim() ?: return@mapNotNullToSet null
}.toTitleCase(),
source = source,
)
}
}
override suspend fun getDetails(manga: Manga): Manga = coroutineScope {
val fullUrl = manga.url.toAbsoluteUrl(getDomain())
val doc = context.httpGet(fullUrl).parseHtml()
val chaptersDeferred = async { getChapters(manga, doc) }
val root = doc.body().selectFirst("div.profile-manga")
?.selectFirst("div.summary_content")
?.selectFirst("div.post-content")
?: throw ParseException("Root not found", fullUrl)
val root2 = doc.body().selectFirst("div.content-area")
?.selectFirst("div.c-page")
?: throw ParseException("Root2 not found", fullUrl)
manga.copy(
tags = root.selectFirst("div.genres-content")?.select("a")
?.mapNotNullToSet { a ->
MangaTag(
key = a.attr("href").removeSuffix("/").substringAfterLast('/'),
title = a.text().toTitleCase(),
source = source,
)
} ?: manga.tags,
description = root2.selectFirst("div.description-summary")
?.selectFirst("div.summary__content")
?.select("p")
?.filterNot { it.ownText().startsWith("A brief description") }
?.joinToString { it.html() },
chapters = chaptersDeferred.await(),
)
}
protected open suspend fun getChapters(manga: Manga, doc: Document): List<MangaChapter> {
val root2 = doc.body().selectFirstOrThrow("div.content-area")
.selectFirstOrThrow("div.c-page")
val dateFormat = SimpleDateFormat(datePattern, sourceLocale ?: Locale.US)
return root2.select("li").asReversed().mapChapters { i, li ->
val a = li.selectFirst("a")
val href = a?.attrAsRelativeUrlOrNull("href") ?: li.parseFailed("Link is missing")
MangaChapter(
id = generateUid(href),
name = a.ownText(),
number = i + 1,
url = href,
uploadDate = parseChapterDate(
dateFormat,
li.selectFirst("span.chapter-release-date i")?.text(),
),
source = source,
scanlator = null,
branch = null,
)
}
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(getDomain())
val doc = context.httpGet(fullUrl).parseHtml()
val root = doc.body().selectFirst("div.main-col-inner")
?.selectFirst("div.reading-content")
?: throw ParseException("Root not found", fullUrl)
return root.select("div.page-break").map { div ->
val img = div.selectFirst("img") ?: div.parseFailed("Page image not found")
val url = img.src()?.toRelativeUrl(getDomain()) ?: div.parseFailed("Image src not found")
MangaPage(
id = generateUid(url),
url = url,
preview = null,
referer = fullUrl,
source = source,
)
}
}
protected fun parseChapterDate(dateFormat: DateFormat, date: String?): Long {
date ?: return 0
return when {
date.endsWith(" ago", ignoreCase = true) -> {
parseRelativeDate(date)
}
// Handle translated 'ago' in Portuguese.
date.endsWith(" atrás", ignoreCase = true) -> {
parseRelativeDate(date)
}
// Handle translated 'ago' in Turkish.
date.endsWith(" önce", ignoreCase = true) -> {
parseRelativeDate(date)
}
// Handle 'yesterday' and 'today', using midnight
date.startsWith("year", ignoreCase = true) -> {
Calendar.getInstance().apply {
add(Calendar.DAY_OF_MONTH, -1) // yesterday
set(Calendar.HOUR_OF_DAY, 0)
set(Calendar.MINUTE, 0)
set(Calendar.SECOND, 0)
set(Calendar.MILLISECOND, 0)
}.timeInMillis
}
date.startsWith("today", ignoreCase = true) -> {
Calendar.getInstance().apply {
set(Calendar.HOUR_OF_DAY, 0)
set(Calendar.MINUTE, 0)
set(Calendar.SECOND, 0)
set(Calendar.MILLISECOND, 0)
}.timeInMillis
}
date.contains(Regex("""\d(st|nd|rd|th)""")) -> {
// Clean date (e.g. 5th December 2019 to 5 December 2019) before parsing it
date.split(" ").map {
if (it.contains(Regex("""\d\D\D"""))) {
it.replace(Regex("""\D"""), "")
} else {
it
}
}
.let { dateFormat.tryParse(it.joinToString(" ")) }
}
else -> dateFormat.tryParse(date)
}
}
// Parses dates in this form:
// 21 hours ago
private fun parseRelativeDate(date: String): Long {
val number = Regex("""(\d+)""").find(date)?.value?.toIntOrNull() ?: return 0
val cal = Calendar.getInstance()
return when {
WordSet(
"hari",
"gün",
"jour",
"día",
"dia",
"day",
).anyWordIn(date) -> cal.apply { add(Calendar.DAY_OF_MONTH, -number) }.timeInMillis
WordSet("jam", "saat", "heure", "hora", "hour").anyWordIn(date) -> cal.apply {
add(
Calendar.HOUR,
-number,
)
}.timeInMillis
WordSet("menit", "dakika", "min", "minute", "minuto").anyWordIn(date) -> cal.apply {
add(
Calendar.MINUTE,
-number,
)
}.timeInMillis
WordSet("detik", "segundo", "second").anyWordIn(date) -> cal.apply {
add(
Calendar.SECOND,
-number,
)
}.timeInMillis
WordSet("month").anyWordIn(date) -> cal.apply { add(Calendar.MONTH, -number) }.timeInMillis
WordSet("year").anyWordIn(date) -> cal.apply { add(Calendar.YEAR, -number) }.timeInMillis
else -> 0
}
}
private fun Element.src(): String? {
var result = absUrl("data-src")
if (result.isEmpty()) result = absUrl("data-cfsrc")
if (result.isEmpty()) result = absUrl("src")
return if (result.isEmpty()) null else result
}
private fun createRequestTemplate() =
(
"action=madara_load_more&page=1&template=madara-core%2Fcontent%2Fcontent-search&vars%5Bs%5D=&vars%5B" +
"orderby%5D=meta_value_num&vars%5Bpaged%5D=1&vars%5Btemplate%5D=search&vars%5Bmeta_query" +
"%5D%5B0%5D%5Brelation%5D=AND&vars%5Bmeta_query%5D%5Brelation%5D=OR&vars%5Bpost_type" +
"%5D=wp-manga&vars%5Bpost_status%5D=publish&vars%5Bmeta_key%5D=_latest_update&vars%5Border" +
"%5D=desc&vars%5Bmanga_archives_item_layout%5D=default"
).split('&')
.map {
val pos = it.indexOf('=')
it.substring(0, pos) to it.substring(pos + 1)
}.toMutableMap()
@MangaSourceParser("MANGAREAD", "MangaRead", "en")
class MangaRead(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGAREAD, "www.mangaread.org") {
override val tagPrefix = "genres/"
override val datePattern = "dd.MM.yyyy"
}
@MangaSourceParser("MANGAWEEBS", "MangaWeebs", "en")
class MangaWeebs(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGAWEEBS, "mangaweebs.in") {
override val datePattern = "dd MMMM HH:mm"
}
@MangaSourceParser("HachiManga", "HACHIMANGA", "ja")
class HachiManga(context: MangaLoaderContext) :
MadaraParser(context, MangaSource.HACHIMANGA, "hachimanga.com") {
override val sourceLocale: Locale = Locale.ENGLISH
}
@MangaSourceParser("PIANMANGA", "PianManga", "en")
class PianManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.PIANMANGA, "pianmanga.me")
@MangaSourceParser("MANGAROSIE", "MangaRosie", "en")
class MangaRosie(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGAROSIE, "mangarosie.in")
@MangaSourceParser("MANGATX", "MangaTx", "en")
class MangaTx(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGATX, "mangatx.com")
@MangaSourceParser("MANGAEFFECT", "MangaEffect", "en")
class MangaEffect(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGAEFFECT, "mangaeffect.com") {
override val datePattern = "dd.MM.yyyy"
}
@MangaSourceParser("AQUAMANGA", "AquaManga", "en")
class AquaManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.AQUAMANGA, "aquamanga.com") {
override fun getFaviconUrl(): String {
return "https://${getDomain()}/wp-content/uploads/2021/03/cropped-cropped-favicon-1-32x32.png"
}
}
@MangaSourceParser("MANGALEK", "MangaLek", "ar")
class MangaLek(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGALEK, "mangalek.com") {
override fun getFaviconUrl(): String {
return "https://${getDomain()}/wp-content/logo192.png"
}
}
@MangaSourceParser("HARIMANGA", "HariManga", "en")
class HariManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.HARIMANGA, "harimanga.com") {
override val datePattern = "MM/dd/yyyy"
}
@MangaSourceParser("KISSMANGA", "KissManga", "en")
class KissManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.KISSMANGA, "kissmanga.in")
@MangaSourceParser("MANGAROCK", "MangaRock", "en")
class MangaRock(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGAROCK, "mangarockteam.com")
@MangaSourceParser("FREEMANGA", "FreeManga", "en")
class FreeManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.FREEMANGA, "freemanga.me")
@MangaSourceParser("MANGA_KOMI", "MangaKomi", "en")
class MangaKomi(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGA_KOMI, "mangakomi.io")
@MangaSourceParser("NEATMANGA", "NeatManga", "en")
class NeatManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.NEATMANGA, "neatmangas.com")
@MangaSourceParser("MANHWACLAN", "ManhwaClan", "en")
class ManhwaClan(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANHWACLAN, "manhwaclan.com")
@MangaSourceParser("MANGA_3S", "Manga3s", "en")
class Manga3s(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGA_3S, "manga3s.com") {
override val tagPrefix = "manhwa-genre/"
}
@MangaSourceParser("MANHWAKOOL", "Manhwa Kool", "en")
class ManhwaKool(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANHWAKOOL, "manhwakool.com") {
override val datePattern: String = "MM/dd"
override fun getFaviconUrl(): String {
return "https://${getDomain()}/wp-content/uploads/2021/10/cropped-logo-kool-32x32.jpeg"
}
}
@MangaSourceParser("TOPMANHUA", "Top Manhua", "en")
class TopManhua(context: MangaLoaderContext) : MadaraParser(context, MangaSource.TOPMANHUA, "www.topmanhua.com") {
override val tagPrefix = "manhua-genre/"
override val datePattern = "MM/dd/yyyy"
}
@MangaSourceParser("S2MANGA", "S2Manga", "en")
class S2Manga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.S2MANGA, "s2manga.com")
@MangaSourceParser("SKY_MANGA", "Sky Manga", "en")
class SkyManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.SKY_MANGA, "skymanga.xyz") {
context: MangaLoaderContext,
source: MangaSource,
domain: String,
) : PagedMangaParser(context, source, pageSize = 12) {
override val configKeyDomain = ConfigKey.Domain(domain, null)
override val sortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.UPDATED,
SortOrder.POPULARITY,
)
protected open val tagPrefix = "manga-genre/"
protected open val isNsfwSource = false
protected open val datePattern = "MMMM dd, yyyy"
init {
paginator.firstPage = 0
searchPaginator.firstPage = 0
}
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val tag = tags.oneOrThrowIfMany()
val payload = createRequestTemplate()
payload["page"] = page.toString()
payload["vars[meta_key]"] = when (sortOrder) {
SortOrder.POPULARITY -> "_wp_manga_views"
SortOrder.UPDATED -> "_latest_update"
else -> "_wp_manga_views"
}
payload["vars[wp-manga-genre]"] = tag?.key.orEmpty()
payload["vars[s]"] = query?.urlEncoded().orEmpty()
val doc = webClient.httpPost(
"https://$domain/wp-admin/admin-ajax.php",
payload,
).parseHtml()
return doc.select("div.row.c-tabs-item__content").ifEmpty {
doc.select("div.page-item-detail.manga")
}.map { div ->
val href = div.selectFirst("a")?.attrAsRelativeUrlOrNull("href")
?: div.parseFailed("Link not found")
val summary = div.selectFirst(".tab-summary") ?: div.selectFirst(".item-summary")
Manga(
id = generateUid(href),
url = href,
publicUrl = href.toAbsoluteUrl(div.host ?: domain),
coverUrl = div.selectFirst("img")?.src().orEmpty(),
title = (summary?.selectFirst("h3") ?: summary?.selectFirst("h4"))?.text().orEmpty(),
altTitle = null,
rating = div.selectFirst("span.total_votes")?.ownText()
?.toFloatOrNull()?.div(5f) ?: -1f,
tags = summary?.selectFirst(".mg_genres")?.select("a")?.mapToSet { a ->
MangaTag(
key = a.attr("href").removeSuffix("/").substringAfterLast('/'),
title = a.text().toTitleCase(),
source = source,
)
}.orEmpty(),
author = summary?.selectFirst(".mg_author")?.selectFirst("a")?.ownText(),
state = when (
summary?.selectFirst(".mg_status")?.selectFirst(".summary-content")
?.ownText()?.trim()?.lowercase()
) {
"ongoing" -> MangaState.ONGOING
"completed" -> MangaState.FINISHED
else -> null
},
source = source,
isNsfw = isNsfwSource,
)
}
}
override suspend fun getTags(): Set<MangaTag> {
val doc = webClient.httpGet("https://$domain/manga/").parseHtml()
val body = doc.body()
val root1 = body.selectFirst("header")?.selectFirst("ul.second-menu")
val root2 = body.selectFirst("div.genres_wrap")?.selectFirst("ul.list-unstyled")
if (root1 == null && root2 == null) {
doc.parseFailed("Root not found")
}
val list = root1?.select("li").orEmpty() + root2?.select("li").orEmpty()
val keySet = HashSet<String>(list.size)
return list.mapNotNullToSet { li ->
val a = li.selectFirst("a") ?: return@mapNotNullToSet null
val href = a.attr("href").removeSuffix("/")
.substringAfterLast(tagPrefix, "")
if (href.isEmpty() || !keySet.add(href)) {
return@mapNotNullToSet null
}
MangaTag(
key = href,
title = a.ownText().trim().ifEmpty {
a.selectFirst(".menu-image-title")?.text()?.trim() ?: return@mapNotNullToSet null
}.toTitleCase(),
source = source,
)
}
}
override suspend fun getDetails(manga: Manga): Manga = coroutineScope {
val fullUrl = manga.url.toAbsoluteUrl(domain)
val doc = webClient.httpGet(fullUrl).parseHtml()
val chaptersDeferred = async { getChapters(manga, doc) }
val root = doc.body().selectFirst("div.profile-manga")
?.selectFirst("div.summary_content")
?.selectFirst("div.post-content")
?: throw ParseException("Root not found", fullUrl)
val root2 = doc.body().selectFirst("div.content-area")
?.selectFirst("div.c-page")
?: throw ParseException("Root2 not found", fullUrl)
manga.copy(
tags = root.selectFirst("div.genres-content")?.select("a")
?.mapNotNullToSet { a ->
MangaTag(
key = a.attr("href").removeSuffix("/").substringAfterLast('/'),
title = a.text().toTitleCase(),
source = source,
)
} ?: manga.tags,
description = root2.selectFirst("div.description-summary")
?.selectFirst("div.summary__content")
?.select("p")
?.filterNot { it.ownText().startsWith("A brief description") }
?.joinToString { it.html() },
chapters = chaptersDeferred.await(),
)
}
protected open suspend fun getChapters(manga: Manga, doc: Document): List<MangaChapter> {
val root2 = doc.body().selectFirstOrThrow("div.content-area")
.selectFirstOrThrow("div.c-page")
val dateFormat = SimpleDateFormat(datePattern, sourceLocale ?: Locale.US)
return root2.select("li").asReversed().mapChapters { i, li ->
val a = li.selectFirst("a")
val href = a?.attrAsRelativeUrlOrNull("href") ?: li.parseFailed("Link is missing")
MangaChapter(
id = generateUid(href),
name = a.ownText(),
number = i + 1,
url = href,
uploadDate = parseChapterDate(
dateFormat,
li.selectFirst("span.chapter-release-date i")?.text(),
),
source = source,
scanlator = null,
branch = null,
)
}
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(domain)
val doc = webClient.httpGet(fullUrl).parseHtml()
val root = doc.body().selectFirst("div.main-col-inner")
?.selectFirst("div.reading-content")
?: throw ParseException("Root not found", fullUrl)
return root.select("div.page-break").map { div ->
val img = div.selectFirst("img") ?: div.parseFailed("Page image not found")
val url = img.src()?.toRelativeUrl(domain) ?: div.parseFailed("Image src not found")
MangaPage(
id = generateUid(url),
url = url,
preview = null,
referer = fullUrl,
source = source,
)
}
}
protected fun parseChapterDate(dateFormat: DateFormat, date: String?): Long {
date ?: return 0
return when {
date.endsWith(" ago", ignoreCase = true) -> {
parseRelativeDate(date)
}
// Handle translated 'ago' in Portuguese.
date.endsWith(" atrás", ignoreCase = true) -> {
parseRelativeDate(date)
}
// Handle translated 'ago' in Turkish.
date.endsWith(" önce", ignoreCase = true) -> {
parseRelativeDate(date)
}
// Handle 'yesterday' and 'today', using midnight
date.startsWith("year", ignoreCase = true) -> {
Calendar.getInstance().apply {
add(Calendar.DAY_OF_MONTH, -1) // yesterday
set(Calendar.HOUR_OF_DAY, 0)
set(Calendar.MINUTE, 0)
set(Calendar.SECOND, 0)
set(Calendar.MILLISECOND, 0)
}.timeInMillis
}
date.startsWith("today", ignoreCase = true) -> {
Calendar.getInstance().apply {
set(Calendar.HOUR_OF_DAY, 0)
set(Calendar.MINUTE, 0)
set(Calendar.SECOND, 0)
set(Calendar.MILLISECOND, 0)
}.timeInMillis
}
date.contains(Regex("""\d(st|nd|rd|th)""")) -> {
// Clean date (e.g. 5th December 2019 to 5 December 2019) before parsing it
date.split(" ").map {
if (it.contains(Regex("""\d\D\D"""))) {
it.replace(Regex("""\D"""), "")
} else {
it
}
}
.let { dateFormat.tryParse(it.joinToString(" ")) }
}
else -> dateFormat.tryParse(date)
}
}
// Parses dates in this form:
// 21 hours ago
private fun parseRelativeDate(date: String): Long {
val number = Regex("""(\d+)""").find(date)?.value?.toIntOrNull() ?: return 0
val cal = Calendar.getInstance()
return when {
WordSet(
"hari",
"gün",
"jour",
"día",
"dia",
"day",
).anyWordIn(date) -> cal.apply { add(Calendar.DAY_OF_MONTH, -number) }.timeInMillis
WordSet("jam", "saat", "heure", "hora", "hour").anyWordIn(date) -> cal.apply {
add(
Calendar.HOUR,
-number,
)
}.timeInMillis
WordSet("menit", "dakika", "min", "minute", "minuto").anyWordIn(date) -> cal.apply {
add(
Calendar.MINUTE,
-number,
)
}.timeInMillis
WordSet("detik", "segundo", "second").anyWordIn(date) -> cal.apply {
add(
Calendar.SECOND,
-number,
)
}.timeInMillis
WordSet("month").anyWordIn(date) -> cal.apply { add(Calendar.MONTH, -number) }.timeInMillis
WordSet("year").anyWordIn(date) -> cal.apply { add(Calendar.YEAR, -number) }.timeInMillis
else -> 0
}
}
private fun Element.src(): String? {
var result = absUrl("data-src")
if (result.isEmpty()) result = absUrl("data-cfsrc")
if (result.isEmpty()) result = absUrl("src")
return if (result.isEmpty()) null else result
}
private fun createRequestTemplate() =
(
"action=madara_load_more&page=1&template=madara-core%2Fcontent%2Fcontent-search&vars%5Bs%5D=&vars%5B" +
"orderby%5D=meta_value_num&vars%5Bpaged%5D=1&vars%5Btemplate%5D=search&vars%5Bmeta_query" +
"%5D%5B0%5D%5Brelation%5D=AND&vars%5Bmeta_query%5D%5Brelation%5D=OR&vars%5Bpost_type" +
"%5D=wp-manga&vars%5Bpost_status%5D=publish&vars%5Bmeta_key%5D=_latest_update&vars%5Border" +
"%5D=desc&vars%5Bmanga_archives_item_layout%5D=default"
).split('&')
.map {
val pos = it.indexOf('=')
it.substring(0, pos) to it.substring(pos + 1)
}.toMutableMap()
@MangaSourceParser("MANGAREAD", "MangaRead", "en")
class MangaRead(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGAREAD, "www.mangaread.org") {
override val tagPrefix = "genres/"
override val datePattern = "dd.MM.yyyy"
}
@MangaSourceParser("MANGAWEEBS", "MangaWeebs", "en")
class MangaWeebs(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGAWEEBS, "mangaweebs.in") {
override val datePattern = "dd MMMM HH:mm"
}
@MangaSourceParser("HACHIMANGA", "HachiManga", "ja")
class HachiManga(context: MangaLoaderContext) :
MadaraParser(context, MangaSource.HACHIMANGA, "hachimanga.com") {
override val sourceLocale: Locale = Locale.ENGLISH
}
@MangaSourceParser("PIANMANGA", "PianManga", "en")
class PianManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.PIANMANGA, "pianmanga.me")
@MangaSourceParser("MANGAROSIE", "MangaRosie", "en")
class MangaRosie(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGAROSIE, "mangarosie.in")
@MangaSourceParser("MANGATX", "MangaTx", "en")
class MangaTx(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGATX, "mangatx.com")
@MangaSourceParser("MANGAEFFECT", "MangaEffect", "en")
class MangaEffect(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGAEFFECT, "mangaeffect.com") {
override val datePattern = "dd.MM.yyyy"
}
@MangaSourceParser("AQUAMANGA", "AquaManga", "en")
class AquaManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.AQUAMANGA, "aquamanga.com") {
}
@MangaSourceParser("MANGALEK", "MangaLek", "ar")
class MangaLek(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGALEK, "mangalek.com") {
}
@MangaSourceParser("HARIMANGA", "HariManga", "en")
class HariManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.HARIMANGA, "harimanga.com") {
override val datePattern = "MM/dd/yyyy"
}
@MangaSourceParser("KISSMANGA", "KissManga", "en")
class KissManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.KISSMANGA, "kissmanga.in")
@MangaSourceParser("MANGAROCK", "MangaRock", "en")
class MangaRock(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGAROCK, "mangarockteam.com")
@MangaSourceParser("FREEMANGA", "FreeManga", "en")
class FreeManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.FREEMANGA, "freemanga.me")
@MangaSourceParser("MANGA_KOMI", "MangaKomi", "en")
class MangaKomi(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGA_KOMI, "mangakomi.io")
@MangaSourceParser("NEATMANGA", "NeatManga", "en")
class NeatManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.NEATMANGA, "neatmangas.com")
@MangaSourceParser("MANHWACLAN", "ManhwaClan", "en")
class ManhwaClan(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANHWACLAN, "manhwaclan.com")
override val isNsfwSource = true
@MangaSourceParser("MANGA_3S", "Manga3s", "en")
class Manga3s(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGA_3S, "manga3s.com") {
override val tagPrefix = "manhwa-genre/"
}
override fun getFaviconUrl(): String {
return "https://${getDomain()}/wp-content/uploads/cropped-sky-tv-1-32x32.png"
}
}
@MangaSourceParser("MANHWAKOOL", "Manhwa Kool", "en")
class ManhwaKool(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANHWAKOOL, "manhwakool.com") {
override val datePattern: String = "MM/dd"
}
@MangaSourceParser("BAKAMAN", "BakaMan", "th")
class BakaMan(context: MangaLoaderContext) : MadaraParser(context, MangaSource.BAKAMAN, "bakaman.net") {
@MangaSourceParser("TOPMANHUA", "Top Manhua", "en")
class TopManhua(context: MangaLoaderContext) : MadaraParser(context, MangaSource.TOPMANHUA, "www.topmanhua.com") {
override val tagPrefix = "manhua-genre/"
override val datePattern = "MM/dd/yyyy"
}
override val isNsfwSource = true
}
@MangaSourceParser("S2MANGA", "S2Manga", "en")
class S2Manga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.S2MANGA, "s2manga.com")
@MangaSourceParser("MANGA_DISTRICT", "Manga District", "en")
class MangaDistrict(context: MangaLoaderContext) :
MadaraParser(context, MangaSource.MANGA_DISTRICT, "mangadistrict.com") {
@MangaSourceParser("SKY_MANGA", "Sky Manga", "en")
class SkyManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.SKY_MANGA, "skymanga.xyz") {
override val tagPrefix = "publication-genre/"
override val isNsfwSource = true
override val isNsfwSource = true
}
}
@MangaSourceParser("HENTAI_4FREE", "Hentai4Free", "en")
class Hentai4Free(context: MangaLoaderContext) :
MadaraParser(context, MangaSource.HENTAI_4FREE, "hentai4free.net") {
@MangaSourceParser("BAKAMAN", "BakaMan", "th")
class BakaMan(context: MangaLoaderContext) : MadaraParser(context, MangaSource.BAKAMAN, "bakaman.net") {
override val tagPrefix = "hentai-tag/"
override val isNsfwSource = true
}
override val isNsfwSource = true
@MangaSourceParser("MANGA_DISTRICT", "Manga District", "en")
class MangaDistrict(context: MangaLoaderContext) :
MadaraParser(context, MangaSource.MANGA_DISTRICT, "mangadistrict.com") {
override suspend fun getTags(): Set<MangaTag> {
val doc = context.httpGet("https://${getDomain()}/").parseHtml()
val body = doc.body()
val root1 = body.selectFirst("header")?.selectFirst("ul.second-menu")
val list = root1?.select("li").orEmpty()
val keySet = HashSet<String>(list.size)
return list.mapNotNullToSet { li ->
val a = li.selectFirst("a") ?: return@mapNotNullToSet null
val href = a.attr("href").removeSuffix("/")
.substringAfterLast(tagPrefix, "")
if (href.isEmpty() || !keySet.add(href)) {
return@mapNotNullToSet null
}
MangaTag(
key = href,
title = a.ownText().trim().toTitleCase(),
source = source,
)
}
}
}
override val tagPrefix = "publication-genre/"
@MangaSourceParser("ALLPORN_COMIC", "All Porn Comic", "en")
class AllPornComic(context: MangaLoaderContext) :
MadaraParser(context, MangaSource.ALLPORN_COMIC, "allporncomic.com") {
override val isNsfwSource = true
}
override val tagPrefix = "porncomic-genre/"
@MangaSourceParser("HENTAI_4FREE", "Hentai4Free", "en")
class Hentai4Free(context: MangaLoaderContext) :
MadaraParser(context, MangaSource.HENTAI_4FREE, "hentai4free.net") {
override val isNsfwSource = true
override val tagPrefix = "hentai-tag/"
override fun getFaviconUrl(): String {
return "https://cdn.${getDomain()}/wp-content/uploads/2019/01/cropped-cropped-pcround-32x32.png"
}
}
override val isNsfwSource = true
@MangaSourceParser("CAT_300", "Cat300", "th")
class Cat300(context: MangaLoaderContext) : MadaraParser(context, MangaSource.CAT_300, "cat300.com") {
override suspend fun getTags(): Set<MangaTag> {
val doc = webClient.httpGet("https://$domain/").parseHtml()
val body = doc.body()
val root1 = body.selectFirst("header")?.selectFirst("ul.second-menu")
val list = root1?.select("li").orEmpty()
val keySet = HashSet<String>(list.size)
return list.mapNotNullToSet { li ->
val a = li.selectFirst("a") ?: return@mapNotNullToSet null
val href = a.attr("href").removeSuffix("/")
.substringAfterLast(tagPrefix, "")
if (href.isEmpty() || !keySet.add(href)) {
return@mapNotNullToSet null
}
MangaTag(
key = href,
title = a.ownText().trim().toTitleCase(),
source = source,
)
}
}
}
override val isNsfwSource = true
}
@MangaSourceParser("BIBIMANGA", "BibiManga", "en")
class BibiManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.BIBIMANGA, "bibimanga.com") {
override val isNsfwSource = true
}
@MangaSourceParser("TREE_MANGA", "Tree Manga", "en")
class TreeManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.TREE_MANGA, "treemanga.com") {
@MangaSourceParser("ALLPORN_COMIC", "All Porn Comic", "en")
class AllPornComic(context: MangaLoaderContext) :
MadaraParser(context, MangaSource.ALLPORN_COMIC, "allporncomic.com") {
override val datePattern = "MM/dd/yyyy"
override fun getFaviconUrl(): String {
return "https://${getDomain()}/wp-content/uploads/2017/10/lgoo-treemanga-2-1.jpg"
}
}
override val tagPrefix = "porncomic-genre/"
@MangaSourceParser("MANGACV", "Manga Cv", "en")
class MangaCv(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGACV, "mangacv.com") {
override val isNsfwSource = true
override fun getFaviconUrl(): String {
return "https://${getDomain()}/wp-content/uploads/2020/10/cropped-mangaCV-32x32.png"
}
}
}
@MangaSourceParser("MANGA_MANHUA", "Manga Manhua", "en")
class MangaManhua(context: MangaLoaderContext) :
MadaraParser(context, MangaSource.MANGA_MANHUA, "mangamanhua.online")
@MangaSourceParser("CAT_300", "Cat300", "th")
class Cat300(context: MangaLoaderContext) : MadaraParser(context, MangaSource.CAT_300, "cat300.com") {
@MangaSourceParser("MANGA_247", "247MANGA", "en")
class Manga247(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGA_247, "247manga.com") {
override val tagPrefix = "manhwa-genre/"
}
override val isNsfwSource = true
}
@MangaSourceParser("MANGA_365", "365Manga", "en")
class Manga365(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGA_365, "365manga.com")
@MangaSourceParser("BIBIMANGA", "BibiManga", "en")
class BibiManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.BIBIMANGA, "bibimanga.com") {
@MangaSourceParser("MANGACLASH", "Mangaclash", "en")
class Mangaclash(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGACLASH, "mangaclash.com") {
override val datePattern = "MM/dd/yyyy"
}
override val isNsfwSource = true
}
@MangaSourceParser("ZINMANGA", "ZINMANGA", "en")
class Zinmanga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.ZINMANGA, "zinmanga.com")
@MangaSourceParser("TREE_MANGA", "Tree Manga", "en")
class TreeManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.TREE_MANGA, "treemanga.com") {
override val datePattern = "MM/dd/yyyy"
}
@MangaSourceParser("MANGACV", "Manga Cv", "en")
class MangaCv(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGACV, "mangacv.com") {
}
@MangaSourceParser("MANGA_MANHUA", "Manga Manhua", "en")
class MangaManhua(context: MangaLoaderContext) :
MadaraParser(context, MangaSource.MANGA_MANHUA, "mangamanhua.online")
@MangaSourceParser("MANGA_247", "247MANGA", "en")
class Manga247(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGA_247, "247manga.com") {
override val tagPrefix = "manhwa-genre/"
}
@MangaSourceParser("MANGA_365", "365Manga", "en")
class Manga365(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGA_365, "365manga.com")
@MangaSourceParser("MANGACLASH", "Mangaclash", "en")
class Mangaclash(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGACLASH, "mangaclash.com") {
override val datePattern = "MM/dd/yyyy"
}
@MangaSourceParser("ZINMANGA", "ZINMANGA", "en")
class Zinmanga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.ZINMANGA, "zinmanga.com")
}

67
src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangalinkParser.kt
Unescape Escape View File

@ -8,44 +8,39 @@ import org.koitharu.kotatsu.parsers.exception.ParseException
import org.koitharu.kotatsu.parsers.model.Manga
import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.model.MangaTag
import org.koitharu.kotatsu.parsers.util.mapNotNullToSet
import org.koitharu.kotatsu.parsers.util.parseHtml
import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl
import org.koitharu.kotatsu.parsers.util.toTitleCase
import org.koitharu.kotatsu.parsers.util.*
@MangaSourceParser("MANGALINK_AR", "Mangalink", "ar")
internal class MangalinkParser(context: MangaLoaderContext) :
MadaraParser(context, MangaSource.MANGALINK_AR, "mangalink.online") {
MadaraParser(context, MangaSource.MANGALINK_AR, "mangalink.online") {
override suspend fun getDetails(manga: Manga): Manga = coroutineScope {
val fullUrl = manga.url.toAbsoluteUrl(getDomain())
val doc = context.httpGet(fullUrl).parseHtml()
val chaptersDeferred = async { getChapters(manga, doc) }
val root = doc.body().selectFirst("div.profile-manga")
?.selectFirst("div.summary_content")
?.selectFirst("div.post-content")
?: throw ParseException("Root not found", fullUrl)
val root2 = doc.body().selectFirst("div.content-area")
?.selectFirst("div.c-page")
?: throw ParseException("Root2 not found", fullUrl)
manga.copy(
tags = root.selectFirst("div.genres-content")?.select("a")
?.mapNotNullToSet { a ->
MangaTag(
key = a.attr("href").removeSuffix("/").substringAfterLast('/'),
title = a.text().toTitleCase(),
source = source,
)
} ?: manga.tags,
description = root2.selectFirst("div.description-summary")
?.selectFirst("div.summary__content")
?.select("p")
?.filterNot { it.ownText().startsWith("A brief description") }
?.joinToString { it.html() },
chapters = chaptersDeferred.await(),
)
}
override suspend fun getDetails(manga: Manga): Manga = coroutineScope {
val fullUrl = manga.url.toAbsoluteUrl(domain)
val doc = webClient.httpGet(fullUrl).parseHtml()
val chaptersDeferred = async { getChapters(manga, doc) }
val root = doc.body().selectFirst("div.profile-manga")
?.selectFirst("div.summary_content")
?.selectFirst("div.post-content")
?: throw ParseException("Root not found", fullUrl)
val root2 = doc.body().selectFirst("div.content-area")
?.selectFirst("div.c-page")
?: throw ParseException("Root2 not found", fullUrl)
manga.copy(
tags = root.selectFirst("div.genres-content")?.select("a")
?.mapNotNullToSet { a ->
MangaTag(
key = a.attr("href").removeSuffix("/").substringAfterLast('/'),
title = a.text().toTitleCase(),
source = source,
)
} ?: manga.tags,
description = root2.selectFirst("div.description-summary")
?.selectFirst("div.summary__content")
?.select("p")
?.filterNot { it.ownText().startsWith("A brief description") }
?.joinToString { it.html() },
chapters = chaptersDeferred.await(),
)
}
override fun getFaviconUrl(): String =
"https://cdn.${getDomain()}/wp-content/uploads/2020/05/cropped-mangalink-180x180.jpg"
}
}

68
src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MangasOriginesParser.kt
Unescape Escape View File

@ -14,43 +14,39 @@ import java.util.*
@MangaSourceParser("MANGAS_ORIGINES", "Mangas Origines", "fr")
internal class MangasOriginesParser(context: MangaLoaderContext) :
Madara6Parser(context, MangaSource.MANGAS_ORIGINES, "mangas-origines.fr") {
Madara6Parser(context, MangaSource.MANGAS_ORIGINES, "mangas-origines.fr") {
override val tagPrefix = "catalogues-genre/"
override val tagPrefix = "catalogues-genre/"
override fun getFaviconUrl(): String {
return "https://${getDomain()}/wp-content/uploads/2020/11/Mangas-150x150.png"
}
override fun parseDetails(manga: Manga, body: Element, chapters: List<MangaChapter>): Manga {
val root = body.selectFirstOrThrow(".site-content")
val postContent = root.selectFirstOrThrow(".post-content")
val tags = postContent.getElementsContainingOwnText("Genre")
.firstOrNull()?.tableValue()
?.getElementsByAttributeValueContaining("href", tagPrefix)
?.mapToSet { a -> a.asMangaTag() } ?: manga.tags
return manga.copy(
largeCoverUrl = root.selectFirst("picture")
?.selectFirst("img[data-src]")
?.attrAsAbsoluteUrlOrNull("data-src"),
description = (root.selectFirst(".detail-content")
?: root.selectFirstOrThrow(".manga-excerpt")).html(),
author = postContent.getElementsContainingOwnText("Auteur")
.firstOrNull()?.tableValue()?.text()?.trim(),
state = postContent.getElementsContainingOwnText("STATUS")
.firstOrNull()?.tableValue()?.text()?.asMangaState(),
tags = tags,
isNsfw = body.hasClass("adult-content"),
chapters = chapters,
)
}
override fun parseDetails(manga: Manga, body: Element, chapters: List<MangaChapter>): Manga {
val root = body.selectFirstOrThrow(".site-content")
val postContent = root.selectFirstOrThrow(".post-content")
val tags = postContent.getElementsContainingOwnText("Genre")
.firstOrNull()?.tableValue()
?.getElementsByAttributeValueContaining("href", tagPrefix)
?.mapToSet { a -> a.asMangaTag() } ?: manga.tags
return manga.copy(
largeCoverUrl = root.selectFirst("picture")
?.selectFirst("img[data-src]")
?.attrAsAbsoluteUrlOrNull("data-src"),
description = (root.selectFirst(".detail-content")
?: root.selectFirstOrThrow(".manga-excerpt")).html(),
author = postContent.getElementsContainingOwnText("Auteur")
.firstOrNull()?.tableValue()?.text()?.trim(),
state = postContent.getElementsContainingOwnText("STATUS")
.firstOrNull()?.tableValue()?.text()?.asMangaState(),
tags = tags,
isNsfw = body.hasClass("adult-content"),
chapters = chapters,
)
}
override fun String.asMangaState() = when (trim().lowercase(Locale.FRANCE)) {
"en cours" -> MangaState.ONGOING
"abandonné",
"terminé",
-> MangaState.FINISHED
override fun String.asMangaState() = when (trim().lowercase(Locale.FRANCE)) {
"en cours" -> MangaState.ONGOING
"abandonné",
"terminé",
-> MangaState.FINISHED
else -> null
}
}
else -> null
}
}

72
src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/PrismaScansParser.kt
Unescape Escape View File

@ -14,45 +14,41 @@ import java.util.*
@MangaSourceParser("PRISMA_SCANS", "Prisma Scans", "pt")
internal class PrismaScansParser(context: MangaLoaderContext) :
Madara6Parser(context, MangaSource.PRISMA_SCANS, "prismascans.net") {
Madara6Parser(context, MangaSource.PRISMA_SCANS, "prismascans.net") {
override val tagPrefix = "manga-genre/"
override val datePattern = "MMM dd, yyyy"
override val tagPrefix = "manga-genre/"
override val datePattern = "MMM dd, yyyy"
override fun getFaviconUrl(): String {
return "https://${getDomain()}/wp-content/uploads/2022/07/cropped-branca-1-192x192.png"
}
override fun parseDetails(manga: Manga, body: Element, chapters: List<MangaChapter>): Manga {
val root = body.selectFirstOrThrow(".site-content")
val postContent = root.selectFirstOrThrow(".post-content")
val tags = postContent.getElementsContainingOwnText("Gênero")
.firstOrNull()?.tableValue()
?.getElementsByAttributeValueContaining("href", tagPrefix)
?.mapToSet { a -> a.asMangaTag() } ?: manga.tags
return manga.copy(
largeCoverUrl = root.selectFirst("picture")
?.selectFirst("img[data-src]")
?.attrAsAbsoluteUrlOrNull("data-src"),
description = root.selectFirstOrThrow(".manga-excerpt").firstElementChild()?.html(),
author = postContent.getElementsContainingOwnText("Artista")
.firstOrNull()?.tableValue()?.text()?.trim(),
altTitle = postContent.getElementsContainingOwnText("Título Alternativo")
.firstOrNull()?.tableValue()?.text()?.trim(),
state = postContent.getElementsContainingOwnText("Status")
.firstOrNull()?.tableValue()?.text()?.asMangaState(),
tags = tags,
isNsfw = body.hasClass("adult-content"),
chapters = chapters,
)
}
override fun parseDetails(manga: Manga, body: Element, chapters: List<MangaChapter>): Manga {
val root = body.selectFirstOrThrow(".site-content")
val postContent = root.selectFirstOrThrow(".post-content")
val tags = postContent.getElementsContainingOwnText("Gênero")
.firstOrNull()?.tableValue()
?.getElementsByAttributeValueContaining("href", tagPrefix)
?.mapToSet { a -> a.asMangaTag() } ?: manga.tags
return manga.copy(
largeCoverUrl = root.selectFirst("picture")
?.selectFirst("img[data-src]")
?.attrAsAbsoluteUrlOrNull("data-src"),
description = root.selectFirstOrThrow(".manga-excerpt").firstElementChild()?.html(),
author = postContent.getElementsContainingOwnText("Artista")
.firstOrNull()?.tableValue()?.text()?.trim(),
altTitle = postContent.getElementsContainingOwnText("Título Alternativo")
.firstOrNull()?.tableValue()?.text()?.trim(),
state = postContent.getElementsContainingOwnText("Status")
.firstOrNull()?.tableValue()?.text()?.asMangaState(),
tags = tags,
isNsfw = body.hasClass("adult-content"),
chapters = chapters,
)
}
override fun String.asMangaState() = when (trim().lowercase(sourceLocale ?: Locale.ROOT)) {
"em lançamento" -> MangaState.ONGOING
"completo",
"cancelado",
-> MangaState.FINISHED
override fun String.asMangaState() = when (trim().lowercase(sourceLocale ?: Locale.ROOT)) {
"em lançamento" -> MangaState.ONGOING
"completo",
"cancelado",
-> MangaState.FINISHED
else -> null
}
}
else -> null
}
}

138
src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/ReaperScansParser.kt
Unescape Escape View File

@ -9,80 +9,78 @@ import java.util.*
@MangaSourceParser("REAPER_SCANS_ID", "ReaperScansID", "in")
internal class ReaperScansParser(context: MangaLoaderContext) :
Madara6Parser(context, MangaSource.REAPER_SCANS_ID, "reaperscans.id") {
Madara6Parser(context, MangaSource.REAPER_SCANS_ID, "reaperscans.id") {
override val datePattern = "MMMM dd, yyyy"
override val tagPrefix = "genre/"
override val sourceLocale: Locale = Locale.ENGLISH
override val datePattern = "MMMM dd, yyyy"
override val tagPrefix = "genre/"
override val sourceLocale: Locale = Locale.ENGLISH
override fun String.asMangaState(): MangaState? = when (this) {
"OnGoing",
"Upcoming",
-> MangaState.ONGOING
override fun String.asMangaState(): MangaState? = when (this) {
"OnGoing",
"Upcoming",
-> MangaState.ONGOING
"Completed",
"Dropped",
-> MangaState.FINISHED
"Completed",
"Dropped",
-> MangaState.FINISHED
else -> null
}
else -> null
}
override fun parseDetails(manga: Manga, body: Element, chapters: List<MangaChapter>): Manga {
val root = body.selectFirstOrThrow(".site-content")
val postContent = root.requireElementById("nav-info")
val tags = postContent.getElementsContainingOwnText("Gênero")
.firstOrNull()?.tableValue()
?.getElementsByAttributeValueContaining("href", tagPrefix)
?.mapToSet { a -> a.asMangaTag() } ?: manga.tags
return manga.copy(
rating = postContent.selectFirstOrThrow(".post-rating")
.selectFirstOrThrow(".total_votes").text().toFloat() / 5f,
largeCoverUrl = root.selectFirst(".summary_image")
?.selectFirst("img[data-src]")
?.attrAsAbsoluteUrlOrNull("data-src")
.assertNotNull("largeCoverUrl"),
description = root.requireElementById("nav-profile")
.selectFirstOrThrow(".description-summary")
.firstElementChild()?.html(),
author = postContent.getElementsContainingOwnText("Author(s)")
.firstOrNull()?.tableValue()?.text()?.trim(),
altTitle = postContent.getElementsContainingOwnText("Alternative")
.firstOrNull()?.tableValue()?.text()?.trim(),
state = postContent.getElementsContainingOwnText("Status")
.firstOrNull()?.tableValue()?.text()?.asMangaState(),
tags = tags,
isNsfw = body.hasClass("adult-content"),
chapters = chapters,
)
}
override fun parseDetails(manga: Manga, body: Element, chapters: List<MangaChapter>): Manga {
val root = body.selectFirstOrThrow(".site-content")
val postContent = root.requireElementById("nav-info")
val tags = postContent.getElementsContainingOwnText("Gênero")
.firstOrNull()?.tableValue()
?.getElementsByAttributeValueContaining("href", tagPrefix)
?.mapToSet { a -> a.asMangaTag() } ?: manga.tags
return manga.copy(
rating = postContent.selectFirstOrThrow(".post-rating")
.selectFirstOrThrow(".total_votes").text().toFloat() / 5f,
largeCoverUrl = root.selectFirst(".summary_image")
?.selectFirst("img[data-src]")
?.attrAsAbsoluteUrlOrNull("data-src")
.assertNotNull("largeCoverUrl"),
description = root.requireElementById("nav-profile")
.selectFirstOrThrow(".description-summary")
.firstElementChild()?.html(),
author = postContent.getElementsContainingOwnText("Author(s)")
.firstOrNull()?.tableValue()?.text()?.trim(),
altTitle = postContent.getElementsContainingOwnText("Alternative")
.firstOrNull()?.tableValue()?.text()?.trim(),
state = postContent.getElementsContainingOwnText("Status")
.firstOrNull()?.tableValue()?.text()?.asMangaState(),
tags = tags,
isNsfw = body.hasClass("adult-content"),
chapters = chapters,
)
}
override suspend fun getTags(): Set<MangaTag> {
val doc = context.httpGet("https://${getDomain()}/semua-komik/").parseHtml()
val body = doc.body()
val root1 = body.selectFirst("header")?.selectFirst("ul.second-menu")
val root2 = body.selectFirst("div.genres_wrap")?.selectFirst("ul.list-unstyled")
if (root1 == null && root2 == null) {
doc.parseFailed("Root not found")
}
val list = root1?.select("li").orEmpty() + root2?.select("li").orEmpty()
val keySet = HashSet<String>(list.size)
return list.mapNotNullToSet { li ->
val a = li.selectFirst("a") ?: return@mapNotNullToSet null
val href = a.attr("href").removeSuffix("/")
.substringAfterLast(tagPrefix, "")
if (href.isEmpty() || !keySet.add(href)) {
return@mapNotNullToSet null
}
MangaTag(
key = href,
title = a.ownText().trim().ifEmpty {
a.selectFirst(".menu-image-title")?.text()?.trim() ?: return@mapNotNullToSet null
}.toTitleCase(),
source = source,
)
}
}
override suspend fun getTags(): Set<MangaTag> {
val doc = webClient.httpGet("https://${domain}/semua-komik/").parseHtml()
val body = doc.body()
val root1 = body.selectFirst("header")?.selectFirst("ul.second-menu")
val root2 = body.selectFirst("div.genres_wrap")?.selectFirst("ul.list-unstyled")
if (root1 == null && root2 == null) {
doc.parseFailed("Root not found")
}
val list = root1?.select("li").orEmpty() + root2?.select("li").orEmpty()
val keySet = HashSet<String>(list.size)
return list.mapNotNullToSet { li ->
val a = li.selectFirst("a") ?: return@mapNotNullToSet null
val href = a.attr("href").removeSuffix("/")
.substringAfterLast(tagPrefix, "")
if (href.isEmpty() || !keySet.add(href)) {
return@mapNotNullToSet null
}
MangaTag(
key = href,
title = a.ownText().trim().ifEmpty {
a.selectFirst(".menu-image-title")?.text()?.trim() ?: return@mapNotNullToSet null
}.toTitleCase(),
source = source,
)
}
}
override fun getFaviconUrl(): String =
"https://wuz.${getDomain()}/wp-content/uploads/2022/03/20063242/cropped-ms-icon-310x310-2-192x192.png"
}
}

77
src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/TatakaeScansParser.kt
Unescape Escape View File

@ -3,7 +3,10 @@ package org.koitharu.kotatsu.parsers.site.madara
import org.jsoup.nodes.Element
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.model.Manga
import org.koitharu.kotatsu.parsers.model.MangaChapter
import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.model.MangaState
import org.koitharu.kotatsu.parsers.util.attrAsAbsoluteUrlOrNull
import org.koitharu.kotatsu.parsers.util.mapToSet
import org.koitharu.kotatsu.parsers.util.selectFirstOrThrow
@ -11,41 +14,37 @@ import java.util.*
@MangaSourceParser("TATAKAE_SCANS", "Tatakae Scans", "pt")
internal class TatakaeScansParser(context: MangaLoaderContext) :
Madara6Parser(context, MangaSource.TATAKAE_SCANS, "tatakaescan.com") {
override val datePattern: String = "dd 'de' MMMMM 'de' yyyy"
override fun parseDetails(manga: Manga, body: Element, chapters: List<MangaChapter>): Manga {
val root = body.selectFirstOrThrow(".site-content")
val postContent = root.selectFirstOrThrow(".post-content")
val tags = postContent.getElementsContainingOwnText("Gênero")
.firstOrNull()?.tableValue()
?.getElementsByAttributeValueContaining("href", tagPrefix)
?.mapToSet { a -> a.asMangaTag() } ?: manga.tags
return manga.copy(
largeCoverUrl = root.selectFirst("picture")
?.selectFirst("img[data-src]")
?.attrAsAbsoluteUrlOrNull("data-src"),
description = (root.selectFirst(".detail-content")
?: root.selectFirstOrThrow(".manga-excerpt")).html(),
author = postContent.getElementsContainingOwnText("Autor")
.firstOrNull()?.tableValue()?.text()?.trim(),
state = postContent.getElementsContainingOwnText("Status")
.firstOrNull()?.tableValue()?.text()?.asMangaState(),
tags = tags,
isNsfw = body.hasClass("adult-content"),
chapters = chapters,
)
}
override fun String.asMangaState() = when (trim().lowercase(Locale.ROOT)) {
"em lançamento" -> MangaState.ONGOING
else -> null
}
override fun getFaviconUrl(): String {
return "https://${getDomain()}/wp-content/uploads/2022/07/cropped-favicon-180x180.png"
}
}
Madara6Parser(context, MangaSource.TATAKAE_SCANS, "tatakaescan.com") {
override val datePattern: String = "dd 'de' MMMMM 'de' yyyy"
override fun parseDetails(manga: Manga, body: Element, chapters: List<MangaChapter>): Manga {
val root = body.selectFirstOrThrow(".site-content")
val postContent = root.selectFirstOrThrow(".post-content")
val tags = postContent.getElementsContainingOwnText("Gênero")
.firstOrNull()?.tableValue()
?.getElementsByAttributeValueContaining("href", tagPrefix)
?.mapToSet { a -> a.asMangaTag() } ?: manga.tags
return manga.copy(
largeCoverUrl = root.selectFirst("picture")
?.selectFirst("img[data-src]")
?.attrAsAbsoluteUrlOrNull("data-src"),
description = (root.selectFirst(".detail-content")
?: root.selectFirstOrThrow(".manga-excerpt")).html(),
author = postContent.getElementsContainingOwnText("Autor")
.firstOrNull()?.tableValue()?.text()?.trim(),
state = postContent.getElementsContainingOwnText("Status")
.firstOrNull()?.tableValue()?.text()?.asMangaState(),
tags = tags,
isNsfw = body.hasClass("adult-content"),
chapters = chapters,
)
}
override fun String.asMangaState() = when (trim().lowercase(Locale.ROOT)) {
"em lançamento" -> MangaState.ONGOING
else -> null
}
}

1030
src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt
View File

File diff suppressed because it is too large Load Diff

316
src/main/kotlin/org/koitharu/kotatsu/parsers/site/multichan/ChanParser.kt
Unescape Escape View File

@ -1,5 +1,6 @@
package org.koitharu.kotatsu.parsers.site.multichan
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.MangaParserAuthProvider
import org.koitharu.kotatsu.parsers.exception.AuthRequiredException
@ -8,172 +9,175 @@ import org.koitharu.kotatsu.parsers.util.*
import java.text.SimpleDateFormat
import java.util.*
internal abstract class ChanParser(source: MangaSource) : MangaParser(source), MangaParserAuthProvider {
internal abstract class ChanParser(
context: MangaLoaderContext,
source: MangaSource,
) : MangaParser(context, source), MangaParserAuthProvider {
override val sortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.NEWEST,
SortOrder.POPULARITY,
SortOrder.ALPHABETICAL,
)
override val sortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.NEWEST,
SortOrder.POPULARITY,
SortOrder.ALPHABETICAL,
)
override val authUrl: String
get() = "https://${getDomain()}"
override val authUrl: String
get() = "https://${domain}"
override val isAuthorized: Boolean
get() = context.cookieJar.getCookies(getDomain()).any { it.name == "dle_user_id" }
override val isAuthorized: Boolean
get() = context.cookieJar.getCookies(domain).any { it.name == "dle_user_id" }
override suspend fun getList(
offset: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val domain = getDomain()
val url = when {
!query.isNullOrEmpty() -> {
if (offset != 0) {
return emptyList()
}
"https://$domain/?do=search&subaction=search&story=${query.urlEncoded()}"
}
override suspend fun getList(
offset: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val domain = domain
val url = when {
!query.isNullOrEmpty() -> {
if (offset != 0) {
return emptyList()
}
"https://$domain/?do=search&subaction=search&story=${query.urlEncoded()}"
}
!tags.isNullOrEmpty() -> tags.joinToString(
prefix = "https://$domain/tags/",
postfix = "&n=${getSortKey2(sortOrder)}?offset=$offset",
separator = "+",
) { tag -> tag.key }
!tags.isNullOrEmpty() -> tags.joinToString(
prefix = "https://$domain/tags/",
postfix = "&n=${getSortKey2(sortOrder)}?offset=$offset",
separator = "+",
) { tag -> tag.key }
else -> "https://$domain/${getSortKey(sortOrder)}?offset=$offset"
}
val doc = context.httpGet(url).parseHtml()
val root = doc.body().selectFirst("div.main_fon")?.getElementById("content")
?: doc.parseFailed("Cannot find root")
return root.select("div.content_row").mapNotNull { row ->
val a = row.selectFirst("div.manga_row1")?.selectFirst("h2")?.selectFirst("a")
?: return@mapNotNull null
val href = a.attrAsRelativeUrl("href")
Manga(
id = generateUid(href),
url = href,
publicUrl = href.toAbsoluteUrl(a.host ?: domain),
altTitle = a.attr("title"),
title = a.text().substringAfterLast('(').substringBeforeLast(')'),
author = row.getElementsByAttributeValueStarting(
"href",
"/mangaka",
).firstOrNull()?.text(),
coverUrl = row.selectFirst("div.manga_images")?.selectFirst("img")
?.absUrl("src").orEmpty(),
tags = runCatching {
row.selectFirst("div.genre")?.select("a")?.mapToSet {
MangaTag(
title = it.text().toTagName(),
key = it.attr("href").substringAfterLast('/').urlEncoded(),
source = source,
)
}
}.getOrNull().orEmpty(),
rating = RATING_UNKNOWN,
state = null,
isNsfw = false,
source = source,
)
}
}
else -> "https://$domain/${getSortKey(sortOrder)}?offset=$offset"
}
val doc = webClient.httpGet(url).parseHtml()
val root = doc.body().selectFirst("div.main_fon")?.getElementById("content")
?: doc.parseFailed("Cannot find root")
return root.select("div.content_row").mapNotNull { row ->
val a = row.selectFirst("div.manga_row1")?.selectFirst("h2")?.selectFirst("a")
?: return@mapNotNull null
val href = a.attrAsRelativeUrl("href")
Manga(
id = generateUid(href),
url = href,
publicUrl = href.toAbsoluteUrl(a.host ?: domain),
altTitle = a.attr("title"),
title = a.text().substringAfterLast('(').substringBeforeLast(')'),
author = row.getElementsByAttributeValueStarting(
"href",
"/mangaka",
).firstOrNull()?.text(),
coverUrl = row.selectFirst("div.manga_images")?.selectFirst("img")
?.absUrl("src").orEmpty(),
tags = runCatching {
row.selectFirst("div.genre")?.select("a")?.mapToSet {
MangaTag(
title = it.text().toTagName(),
key = it.attr("href").substringAfterLast('/').urlEncoded(),
source = source,
)
}
}.getOrNull().orEmpty(),
rating = RATING_UNKNOWN,
state = null,
isNsfw = false,
source = source,
)
}
}
override suspend fun getDetails(manga: Manga): Manga {
val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml()
val root = doc.body().getElementById("dle-content") ?: doc.parseFailed("Cannot find root")
val dateFormat = SimpleDateFormat("yyyy-MM-dd", Locale.US)
return manga.copy(
description = root.getElementById("description")?.html()?.substringBeforeLast("<div"),
largeCoverUrl = root.getElementById("cover")?.absUrl("src"),
chapters = root.select("table.table_cha tr:gt(1)").reversed().mapChapters { i, tr ->
val href = tr?.selectFirst("a")?.attrAsRelativeUrlOrNull("href")
?: return@mapChapters null
MangaChapter(
id = generateUid(href),
name = tr.selectFirst("a")?.text().orEmpty(),
number = i + 1,
url = href,
scanlator = null,
branch = null,
uploadDate = dateFormat.tryParse(tr.selectFirst("div.date")?.text()),
source = source,
)
},
)
}
override suspend fun getDetails(manga: Manga): Manga {
val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml()
val root = doc.body().getElementById("dle-content") ?: doc.parseFailed("Cannot find root")
val dateFormat = SimpleDateFormat("yyyy-MM-dd", Locale.US)
return manga.copy(
description = root.getElementById("description")?.html()?.substringBeforeLast("<div"),
largeCoverUrl = root.getElementById("cover")?.absUrl("src"),
chapters = root.select("table.table_cha tr:gt(1)").reversed().mapChapters { i, tr ->
val href = tr?.selectFirst("a")?.attrAsRelativeUrlOrNull("href")
?: return@mapChapters null
MangaChapter(
id = generateUid(href),
name = tr.selectFirst("a")?.text().orEmpty(),
number = i + 1,
url = href,
scanlator = null,
branch = null,
uploadDate = dateFormat.tryParse(tr.selectFirst("div.date")?.text()),
source = source,
)
},
)
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(getDomain())
val doc = context.httpGet(fullUrl).parseHtml()
val scripts = doc.select("script")
for (script in scripts) {
val data = script.html()
val pos = data.indexOf("\"fullimg")
if (pos == -1) {
continue
}
val json = data.substring(pos).substringAfter('[').substringBefore(';')
.substringBeforeLast(']')
val domain = getDomain()
return json.split(",").mapNotNull {
it.trim()
.removeSurrounding('"', '\'')
.toRelativeUrl(domain)
.takeUnless(String::isBlank)
}.map { url ->
MangaPage(
id = generateUid(url),
url = url,
preview = null,
referer = fullUrl,
source = source,
)
}
}
doc.parseFailed("Pages list not found at ${chapter.url}")
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(domain)
val doc = webClient.httpGet(fullUrl).parseHtml()
val scripts = doc.select("script")
for (script in scripts) {
val data = script.html()
val pos = data.indexOf("\"fullimg")
if (pos == -1) {
continue
}
val json = data.substring(pos).substringAfter('[').substringBefore(';')
.substringBeforeLast(']')
val domain = domain
return json.split(",").mapNotNull {
it.trim()
.removeSurrounding('"', '\'')
.toRelativeUrl(domain)
.takeUnless(String::isBlank)
}.map { url ->
MangaPage(
id = generateUid(url),
url = url,
preview = null,
referer = fullUrl,
source = source,
)
}
}
doc.parseFailed("Pages list not found at ${chapter.url}")
}
override suspend fun getTags(): Set<MangaTag> {
val domain = getDomain()
val doc = context.httpGet("https://$domain/mostfavorites&sort=manga").parseHtml()
val root = doc.body().selectFirst("div.main_fon")?.getElementById("side")
?.select("ul")?.last() ?: doc.parseFailed("Cannot find root")
return root.select("li.sidetag").mapToSet { li ->
val a = li.children().lastOrNull() ?: li.parseFailed("a is null")
MangaTag(
title = a.text().toTagName(),
key = a.attr("href").substringAfterLast('/'),
source = source,
)
}
}
override suspend fun getTags(): Set<MangaTag> {
val domain = domain
val doc = webClient.httpGet("https://$domain/mostfavorites&sort=manga").parseHtml()
val root = doc.body().selectFirst("div.main_fon")?.getElementById("side")
?.select("ul")?.last() ?: doc.parseFailed("Cannot find root")
return root.select("li.sidetag").mapToSet { li ->
val a = li.children().lastOrNull() ?: li.parseFailed("a is null")
MangaTag(
title = a.text().toTagName(),
key = a.attr("href").substringAfterLast('/'),
source = source,
)
}
}
override suspend fun getUsername(): String {
val doc = context.httpGet("https://${getDomain()}").parseHtml().body()
val root = doc.requireElementById("top_user")
val a = root.getElementsByAttributeValueContaining("href", "/user/").firstOrNull()
?: throw AuthRequiredException(source)
return a.attr("href").removeSuffix('/').substringAfterLast('/')
}
override suspend fun getUsername(): String {
val doc = webClient.httpGet("https://${domain}").parseHtml().body()
val root = doc.requireElementById("top_user")
val a = root.getElementsByAttributeValueContaining("href", "/user/").firstOrNull()
?: throw AuthRequiredException(source)
return a.attr("href").removeSuffix('/').substringAfterLast('/')
}
private fun getSortKey(sortOrder: SortOrder) =
when (sortOrder) {
SortOrder.ALPHABETICAL -> "catalog"
SortOrder.POPULARITY -> "mostfavorites"
SortOrder.NEWEST -> "manga/new"
else -> "mostfavorites"
}
private fun getSortKey(sortOrder: SortOrder) =
when (sortOrder) {
SortOrder.ALPHABETICAL -> "catalog"
SortOrder.POPULARITY -> "mostfavorites"
SortOrder.NEWEST -> "manga/new"
else -> "mostfavorites"
}
private fun getSortKey2(sortOrder: SortOrder) =
when (sortOrder) {
SortOrder.ALPHABETICAL -> "abcasc"
SortOrder.POPULARITY -> "favdesc"
SortOrder.NEWEST -> "datedesc"
else -> "favdesc"
}
private fun getSortKey2(sortOrder: SortOrder) =
when (sortOrder) {
SortOrder.ALPHABETICAL -> "abcasc"
SortOrder.POPULARITY -> "favdesc"
SortOrder.NEWEST -> "datedesc"
else -> "favdesc"
}
private fun String.toTagName() = replace('_', ' ').toTitleCase()
}
private fun String.toTagName() = replace('_', ' ').toTitleCase()
}

96
src/main/kotlin/org/koitharu/kotatsu/parsers/site/multichan/HenChanParser.kt
Unescape Escape View File

@ -7,54 +7,54 @@ import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
@MangaSourceParser("HENCHAN", "Хентай-тян", "ru")
internal class HenChanParser(override val context: MangaLoaderContext) : ChanParser(MangaSource.HENCHAN) {
internal class HenChanParser(context: MangaLoaderContext) : ChanParser(context, MangaSource.HENCHAN) {
override val configKeyDomain = ConfigKey.Domain(
"y.hentaichan.live",
arrayOf("y.hentaichan.live", "xxx.hentaichan.live", "xx.hentaichan.live", "hentaichan.live", "hentaichan.pro"),
)
override val configKeyDomain = ConfigKey.Domain(
"y.hentaichan.live",
arrayOf("y.hentaichan.live", "xxx.hentaichan.live", "xx.hentaichan.live", "hentaichan.live", "hentaichan.pro"),
)
override suspend fun getList(
offset: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
return super.getList(offset, query, tags, sortOrder).map {
it.copy(
coverUrl = it.coverUrl.replace("_blur", ""),
isNsfw = true,
)
}
}
override suspend fun getList(
offset: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
return super.getList(offset, query, tags, sortOrder).map {
it.copy(
coverUrl = it.coverUrl.replace("_blur", ""),
isNsfw = true,
)
}
}
override suspend fun getDetails(manga: Manga): Manga {
val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml()
val root = doc.body().requireElementById("dle-content")
val readLink = manga.url.replace("manga", "online")
return manga.copy(
description = root.getElementById("description")?.html()?.substringBeforeLast("<div"),
largeCoverUrl = root.getElementById("cover")?.absUrl("src"),
tags = root.selectFirst("div.sidetags")?.select("li.sidetag")?.mapToSet {
val a = it.children().last() ?: doc.parseFailed("Invalid tag")
MangaTag(
title = a.text().toTitleCase(),
key = a.attr("href").substringAfterLast('/'),
source = source,
)
} ?: manga.tags,
chapters = listOf(
MangaChapter(
id = generateUid(readLink),
url = readLink,
source = source,
number = 1,
uploadDate = 0L,
name = manga.title,
scanlator = null,
branch = null,
),
),
)
}
}
override suspend fun getDetails(manga: Manga): Manga {
val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml()
val root = doc.body().requireElementById("dle-content")
val readLink = manga.url.replace("manga", "online")
return manga.copy(
description = root.getElementById("description")?.html()?.substringBeforeLast("<div"),
largeCoverUrl = root.getElementById("cover")?.absUrl("src"),
tags = root.selectFirst("div.sidetags")?.select("li.sidetag")?.mapToSet {
val a = it.children().last() ?: doc.parseFailed("Invalid tag")
MangaTag(
title = a.text().toTitleCase(),
key = a.attr("href").substringAfterLast('/'),
source = source,
)
} ?: manga.tags,
chapters = listOf(
MangaChapter(
id = generateUid(readLink),
url = readLink,
source = source,
number = 1,
uploadDate = 0L,
name = manga.title,
scanlator = null,
branch = null,
),
),
)
}
}

6
src/main/kotlin/org/koitharu/kotatsu/parsers/site/multichan/MangaChanParser.kt
Unescape Escape View File

@ -6,7 +6,7 @@ import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.MangaSource
@MangaSourceParser("MANGACHAN", "Манга-тян", "ru")
internal class MangaChanParser(override val context: MangaLoaderContext) : ChanParser(MangaSource.MANGACHAN) {
internal class MangaChanParser(context: MangaLoaderContext) : ChanParser(context, MangaSource.MANGACHAN) {
override val configKeyDomain = ConfigKey.Domain("manga-chan.me", null)
}
override val configKeyDomain = ConfigKey.Domain("manga-chan.me", null)
}

52
src/main/kotlin/org/koitharu/kotatsu/parsers/site/multichan/YaoiChanParser.kt
Unescape Escape View File

@ -9,31 +9,31 @@ import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.util.*
@MangaSourceParser("YAOICHAN", "Яой-тян", "ru")
internal class YaoiChanParser(override val context: MangaLoaderContext) : ChanParser(MangaSource.YAOICHAN) {
internal class YaoiChanParser(context: MangaLoaderContext) : ChanParser(context, MangaSource.YAOICHAN) {
override val configKeyDomain = ConfigKey.Domain("yaoi-chan.me", null)
override val configKeyDomain = ConfigKey.Domain("yaoi-chan.me", null)
override suspend fun getDetails(manga: Manga): Manga {
val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml()
val root = doc.body().requireElementById("dle-content")
return manga.copy(
description = root.getElementById("description")?.html()?.substringBeforeLast("<div"),
largeCoverUrl = root.getElementById("cover")?.absUrl("src"),
chapters = root.select("table.table_cha").flatMap { table ->
table.select("div.manga")
}.mapNotNull { it.selectFirst("a") }.reversed().mapChapters { i, a ->
val href = a.attrAsRelativeUrl("href")
MangaChapter(
id = generateUid(href),
name = a.text().trim(),
number = i + 1,
url = href,
uploadDate = 0L,
source = source,
scanlator = null,
branch = null,
)
},
)
}
}
override suspend fun getDetails(manga: Manga): Manga {
val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml()
val root = doc.body().requireElementById("dle-content")
return manga.copy(
description = root.getElementById("description")?.html()?.substringBeforeLast("<div"),
largeCoverUrl = root.getElementById("cover")?.absUrl("src"),
chapters = root.select("table.table_cha").flatMap { table ->
table.select("div.manga")
}.mapNotNull { it.selectFirst("a") }.reversed().mapChapters { i, a ->
val href = a.attrAsRelativeUrl("href")
MangaChapter(
id = generateUid(href),
name = a.text().trim(),
number = i + 1,
url = href,
uploadDate = 0L,
source = source,
scanlator = null,
branch = null,
)
},
)
}
}

530
src/main/kotlin/org/koitharu/kotatsu/parsers/site/rulib/MangaLibParser.kt
Unescape Escape View File

@ -20,284 +20,284 @@ import java.text.SimpleDateFormat
import java.util.*
internal open class MangaLibParser(
override val context: MangaLoaderContext,
source: MangaSource,
) : PagedMangaParser(source, pageSize = 60), MangaParserAuthProvider {
context: MangaLoaderContext,
source: MangaSource,
) : PagedMangaParser(context, source, pageSize = 60), MangaParserAuthProvider {
override val configKeyDomain = ConfigKey.Domain("mangalib.me", null)
override val configKeyDomain = ConfigKey.Domain("mangalib.me", null)
override val authUrl: String
get() = "https://${getDomain()}/login"
override val authUrl: String
get() = "https://${domain}/login"
override val sortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.RATING,
SortOrder.ALPHABETICAL,
SortOrder.POPULARITY,
SortOrder.UPDATED,
SortOrder.NEWEST,
)
override val sortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.RATING,
SortOrder.ALPHABETICAL,
SortOrder.POPULARITY,
SortOrder.UPDATED,
SortOrder.NEWEST,
)
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
if (!query.isNullOrEmpty()) {
return if (page == searchPaginator.firstPage) search(query) else emptyList()
}
val url = buildString {
append("https://")
append(getDomain())
append("/manga-list?dir=")
append(getSortKey(sortOrder))
append("&page=")
append(page)
tags?.forEach { tag ->
append("&genres[include][]=")
append(tag.key)
}
}
val doc = context.httpGet(url).parseHtml()
val root = doc.body().getElementById("manga-list") ?: doc.parseFailed("Root not found")
val items = root.selectFirst("div.media-cards-grid")?.select("div.media-card-wrap")
?: return emptyList()
return items.mapNotNull { card ->
val a = card.selectFirst("a.media-card") ?: return@mapNotNull null
val href = a.attrAsRelativeUrl("href")
Manga(
id = generateUid(href),
title = card.selectFirst("h3")?.text().orEmpty(),
coverUrl = a.absUrl("data-src"),
altTitle = null,
author = null,
rating = RATING_UNKNOWN,
url = href,
publicUrl = href.toAbsoluteUrl(a.host ?: getDomain()),
tags = emptySet(),
state = null,
isNsfw = false,
source = source,
)
}
}
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
if (!query.isNullOrEmpty()) {
return if (page == searchPaginator.firstPage) search(query) else emptyList()
}
val url = buildString {
append("https://")
append(domain)
append("/manga-list?dir=")
append(getSortKey(sortOrder))
append("&page=")
append(page)
tags?.forEach { tag ->
append("&genres[include][]=")
append(tag.key)
}
}
val doc = webClient.httpGet(url).parseHtml()
val root = doc.body().getElementById("manga-list") ?: doc.parseFailed("Root not found")
val items = root.selectFirst("div.media-cards-grid")?.select("div.media-card-wrap")
?: return emptyList()
return items.mapNotNull { card ->
val a = card.selectFirst("a.media-card") ?: return@mapNotNull null
val href = a.attrAsRelativeUrl("href")
Manga(
id = generateUid(href),
title = card.selectFirst("h3")?.text().orEmpty(),
coverUrl = a.absUrl("data-src"),
altTitle = null,
author = null,
rating = RATING_UNKNOWN,
url = href,
publicUrl = href.toAbsoluteUrl(a.host ?: domain),
tags = emptySet(),
state = null,
isNsfw = false,
source = source,
)
}
}
override suspend fun getDetails(manga: Manga): Manga {
val fullUrl = manga.url.toAbsoluteUrl(getDomain())
val doc = context.httpGet("$fullUrl?section=info").parseHtml()
val root = doc.body().getElementById("main-page") ?: throw ParseException("Root not found", fullUrl)
val title = root.selectFirst("div.media-header__wrap")?.children()
val info = root.selectFirst("div.media-content")
val chaptersDoc = context.httpGet("$fullUrl?section=chapters").parseHtml()
val scripts = chaptersDoc.select("script")
val dateFormat = SimpleDateFormat("yyy-MM-dd", Locale.US)
var chapters: ChaptersListBuilder? = null
scripts@ for (script in scripts) {
val raw = script.html().lines()
for (line in raw) {
if (line.startsWith("window.__DATA__")) {
val json = JSONObject(line.substringAfter('=').substringBeforeLast(';'))
val list = json.getJSONObject("chapters").getJSONArray("list")
val id = json.optJSONObject("user")?.getLong("id")?.toString() ?: "not"
val total = list.length()
chapters = ChaptersListBuilder(total)
for (i in 0 until total) {
val item = list.getJSONObject(i)
val chapterId = item.getLong("chapter_id")
val scanlator = item.getStringOrNull("username")
val url = buildString {
if (isAuthorized) {
append(manga.url)
append("/v")
append(item.getInt("chapter_volume"))
append("/c")
append(item.getString("chapter_number"))
append("?ui=")
append(id)
} else {
append(manga.url)
append("/v")
append(item.getInt("chapter_volume"))
append("/c")
append(item.getString("chapter_number"))
}
}
val nameChapter = item.getStringOrNull("chapter_name")
val volume = item.getInt("chapter_volume")
val number = item.getString("chapter_number")
val fullNameChapter = "Том $volume. Глава $number"
chapters.add(
MangaChapter(
id = generateUid(chapterId),
url = url,
source = source,
number = total - i,
uploadDate = dateFormat.tryParse(
item.getString("chapter_created_at").substringBefore(" "),
),
scanlator = scanlator,
branch = null,
name = if (nameChapter.isNullOrBlank()) fullNameChapter else "$fullNameChapter - $nameChapter",
),
)
}
chapters.reverse()
break@scripts
}
}
}
return manga.copy(
title = title?.getOrNull(0)?.text()?.takeUnless(String::isBlank) ?: manga.title,
altTitle = title?.getOrNull(1)?.text()?.substringBefore('/')?.trim(),
rating = root.selectFirst("div.media-stats-item__score")
?.selectFirst("span")
?.text()?.toFloatOrNull()?.div(5f) ?: manga.rating,
author = info?.getElementsMatchingOwnText("Автор")?.firstOrNull()
?.nextElementSibling()?.text() ?: manga.author,
tags = info?.selectFirst("div.media-tags")
?.select("a.media-tag-item")?.mapNotNullToSet { a ->
val href = a.attr("href")
if (href.contains("genres")) {
MangaTag(
title = a.text().toTitleCase(),
key = href.substringAfterLast('='),
source = source,
)
} else null
} ?: manga.tags,
isNsfw = isNsfw(doc),
description = info?.selectFirst("div.media-description__text")?.html(),
chapters = chapters?.toList(),
)
}
override suspend fun getDetails(manga: Manga): Manga {
val fullUrl = manga.url.toAbsoluteUrl(domain)
val doc = webClient.httpGet("$fullUrl?section=info").parseHtml()
val root = doc.body().getElementById("main-page") ?: throw ParseException("Root not found", fullUrl)
val title = root.selectFirst("div.media-header__wrap")?.children()
val info = root.selectFirst("div.media-content")
val chaptersDoc = webClient.httpGet("$fullUrl?section=chapters").parseHtml()
val scripts = chaptersDoc.select("script")
val dateFormat = SimpleDateFormat("yyy-MM-dd", Locale.US)
var chapters: ChaptersListBuilder? = null
scripts@ for (script in scripts) {
val raw = script.html().lines()
for (line in raw) {
if (line.startsWith("window.__DATA__")) {
val json = JSONObject(line.substringAfter('=').substringBeforeLast(';'))
val list = json.getJSONObject("chapters").getJSONArray("list")
val id = json.optJSONObject("user")?.getLong("id")?.toString() ?: "not"
val total = list.length()
chapters = ChaptersListBuilder(total)
for (i in 0 until total) {
val item = list.getJSONObject(i)
val chapterId = item.getLong("chapter_id")
val scanlator = item.getStringOrNull("username")
val url = buildString {
if (isAuthorized) {
append(manga.url)
append("/v")
append(item.getInt("chapter_volume"))
append("/c")
append(item.getString("chapter_number"))
append("?ui=")
append(id)
} else {
append(manga.url)
append("/v")
append(item.getInt("chapter_volume"))
append("/c")
append(item.getString("chapter_number"))
}
}
val nameChapter = item.getStringOrNull("chapter_name")
val volume = item.getInt("chapter_volume")
val number = item.getString("chapter_number")
val fullNameChapter = "Том $volume. Глава $number"
chapters.add(
MangaChapter(
id = generateUid(chapterId),
url = url,
source = source,
number = total - i,
uploadDate = dateFormat.tryParse(
item.getString("chapter_created_at").substringBefore(" "),
),
scanlator = scanlator,
branch = null,
name = if (nameChapter.isNullOrBlank()) fullNameChapter else "$fullNameChapter - $nameChapter",
),
)
}
chapters.reverse()
break@scripts
}
}
}
return manga.copy(
title = title?.getOrNull(0)?.text()?.takeUnless(String::isBlank) ?: manga.title,
altTitle = title?.getOrNull(1)?.text()?.substringBefore('/')?.trim(),
rating = root.selectFirst("div.media-stats-item__score")
?.selectFirst("span")
?.text()?.toFloatOrNull()?.div(5f) ?: manga.rating,
author = info?.getElementsMatchingOwnText("Автор")?.firstOrNull()
?.nextElementSibling()?.text() ?: manga.author,
tags = info?.selectFirst("div.media-tags")
?.select("a.media-tag-item")?.mapNotNullToSet { a ->
val href = a.attr("href")
if (href.contains("genres")) {
MangaTag(
title = a.text().toTitleCase(),
key = href.substringAfterLast('='),
source = source,
)
} else null
} ?: manga.tags,
isNsfw = isNsfw(doc),
description = info?.selectFirst("div.media-description__text")?.html(),
chapters = chapters?.toList(),
)
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(getDomain())
val doc = context.httpGet(fullUrl).parseHtml()
if (doc.location().substringBefore('?').endsWith("/register")) {
throw AuthRequiredException(source)
}
val scripts = doc.head().select("script")
val pg = (doc.body().getElementById("pg")?.html() ?: doc.parseFailed("Element #pg not found"))
.substringAfter('=')
.substringBeforeLast(';')
val pages = JSONArray(pg)
for (script in scripts) {
val raw = script.html().trim()
if (raw.contains("window.__info")) {
val json = JSONObject(
raw.substringAfter("window.__info")
.substringAfter('=')
.substringBeforeLast(';'),
)
val domain = json.getJSONObject("servers").run {
getStringOrNull("main") ?: getString(
json.getJSONObject("img").getString("server"),
)
}
val url = json.getJSONObject("img").getString("url")
return pages.mapJSON { x ->
val pageUrl = "$domain/$url${x.getString("u")}"
MangaPage(
id = generateUid(pageUrl),
url = pageUrl,
preview = null,
referer = fullUrl,
source = source,
)
}
}
}
throw ParseException("Script with info not found", fullUrl)
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(domain)
val doc = webClient.httpGet(fullUrl).parseHtml()
if (doc.location().substringBefore('?').endsWith("/register")) {
throw AuthRequiredException(source)
}
val scripts = doc.head().select("script")
val pg = (doc.body().getElementById("pg")?.html() ?: doc.parseFailed("Element #pg not found"))
.substringAfter('=')
.substringBeforeLast(';')
val pages = JSONArray(pg)
for (script in scripts) {
val raw = script.html().trim()
if (raw.contains("window.__info")) {
val json = JSONObject(
raw.substringAfter("window.__info")
.substringAfter('=')
.substringBeforeLast(';'),
)
val domain = json.getJSONObject("servers").run {
getStringOrNull("main") ?: getString(
json.getJSONObject("img").getString("server"),
)
}
val url = json.getJSONObject("img").getString("url")
return pages.mapJSON { x ->
val pageUrl = "$domain/$url${x.getString("u")}"
MangaPage(
id = generateUid(pageUrl),
url = pageUrl,
preview = null,
referer = fullUrl,
source = source,
)
}
}
}
throw ParseException("Script with info not found", fullUrl)
}
override suspend fun getTags(): Set<MangaTag> {
val url = "https://${getDomain()}/manga-list"
val doc = context.httpGet(url).parseHtml()
val scripts = doc.body().select("script")
for (script in scripts) {
val raw = script.html().trim()
if (raw.startsWith("window.__DATA")) {
val json = JSONObject(raw.substringAfter('=').substringBeforeLast(';'))
val genres = json.getJSONObject("filters").getJSONArray("genres")
val result = ArraySet<MangaTag>(genres.length())
for (x in genres.JSONIterator()) {
result += MangaTag(
source = source,
key = x.getInt("id").toString(),
title = x.getString("name").toTitleCase(),
)
}
return result
}
}
throw ParseException("Script with genres not found", url)
}
override suspend fun getTags(): Set<MangaTag> {
val url = "https://${domain}/manga-list"
val doc = webClient.httpGet(url).parseHtml()
val scripts = doc.body().select("script")
for (script in scripts) {
val raw = script.html().trim()
if (raw.startsWith("window.__DATA")) {
val json = JSONObject(raw.substringAfter('=').substringBeforeLast(';'))
val genres = json.getJSONObject("filters").getJSONArray("genres")
val result = ArraySet<MangaTag>(genres.length())
for (x in genres.JSONIterator()) {
result += MangaTag(
source = source,
key = x.getInt("id").toString(),
title = x.getString("name").toTitleCase(),
)
}
return result
}
}
throw ParseException("Script with genres not found", url)
}
override val isAuthorized: Boolean
get() {
return context.cookieJar.getCookies(getDomain()).any {
it.name.startsWith("remember_web_")
}
}
override val isAuthorized: Boolean
get() {
return context.cookieJar.getCookies(domain).any {
it.name.startsWith("remember_web_")
}
}
override suspend fun getUsername(): String {
val body = context.httpGet("https://${LibConst.LIB_SOCIAL_LINK}/messages").parseHtml().body()
if (body.baseUri().endsWith("/login")) {
throw AuthRequiredException(source)
}
return body.selectFirst(".profile-user__username")?.text() ?: body.parseFailed("Cannot find username")
}
override suspend fun getUsername(): String {
val body = webClient.httpGet("https://${LibConst.LIB_SOCIAL_LINK}/messages").parseHtml().body()
if (body.baseUri().endsWith("/login")) {
throw AuthRequiredException(source)
}
return body.selectFirst(".profile-user__username")?.text() ?: body.parseFailed("Cannot find username")
}
protected open fun isNsfw(doc: Document): Boolean {
val sidebar = doc.body().run {
selectFirst(".media-sidebar") ?: selectFirst(".media-info")
} ?: doc.parseFailed("Sidebar not found")
return sidebar.getElementsContainingOwnText("18+").isNotEmpty()
}
protected open fun isNsfw(doc: Document): Boolean {
val sidebar = doc.body().run {
selectFirst(".media-sidebar") ?: selectFirst(".media-info")
} ?: doc.parseFailed("Sidebar not found")
return sidebar.getElementsContainingOwnText("18+").isNotEmpty()
}
private fun getSortKey(sortOrder: SortOrder?) = when (sortOrder) {
SortOrder.RATING -> "desc&sort=rate"
SortOrder.ALPHABETICAL -> "asc&sort=name"
SortOrder.POPULARITY -> "desc&sort=views"
SortOrder.UPDATED -> "desc&sort=last_chapter_at"
SortOrder.NEWEST -> "desc&sort=created_at"
else -> "desc&sort=last_chapter_at"
}
private fun getSortKey(sortOrder: SortOrder?) = when (sortOrder) {
SortOrder.RATING -> "desc&sort=rate"
SortOrder.ALPHABETICAL -> "asc&sort=name"
SortOrder.POPULARITY -> "desc&sort=views"
SortOrder.UPDATED -> "desc&sort=last_chapter_at"
SortOrder.NEWEST -> "desc&sort=created_at"
else -> "desc&sort=last_chapter_at"
}
private suspend fun search(query: String): List<Manga> {
val domain = getDomain()
val json = context.httpGet("https://$domain/search?type=manga&q=$query")
.parseJsonArray()
return json.mapJSON { jo ->
val slug = jo.getString("slug")
val url = "/$slug"
val covers = jo.getJSONObject("covers")
val title = jo.getString("rus_name").ifEmpty { jo.getString("name") }
Manga(
id = generateUid(url),
url = url,
publicUrl = "https://$domain/$slug",
title = title,
altTitle = jo.getString("name").takeUnless { it == title },
author = null,
tags = emptySet(),
rating = jo.getString("rate_avg").toFloatOrNull()?.div(5f) ?: RATING_UNKNOWN,
state = null,
isNsfw = false,
source = source,
coverUrl = covers.getString("thumbnail").toAbsoluteUrl(domain),
largeCoverUrl = covers.getString("default").toAbsoluteUrl(domain),
)
}
}
private suspend fun search(query: String): List<Manga> {
val domain = domain
val json = webClient.httpGet("https://$domain/search?type=manga&q=$query")
.parseJsonArray()
return json.mapJSON { jo ->
val slug = jo.getString("slug")
val url = "/$slug"
val covers = jo.getJSONObject("covers")
val title = jo.getString("rus_name").ifEmpty { jo.getString("name") }
Manga(
id = generateUid(url),
url = url,
publicUrl = "https://$domain/$slug",
title = title,
altTitle = jo.getString("name").takeUnless { it == title },
author = null,
tags = emptySet(),
rating = jo.getString("rate_avg").toFloatOrNull()?.div(5f) ?: RATING_UNKNOWN,
state = null,
isNsfw = false,
source = source,
coverUrl = covers.getString("thumbnail").toAbsoluteUrl(domain),
largeCoverUrl = covers.getString("default").toAbsoluteUrl(domain),
)
}
}
@MangaSourceParser("MANGALIB", "MangaLib", "ru")
class Impl(context: MangaLoaderContext) : MangaLibParser(context, MangaSource.MANGALIB)
@MangaSourceParser("MANGALIB", "MangaLib", "ru")
class Impl(context: MangaLoaderContext) : MangaLibParser(context, MangaSource.MANGALIB)
object LibConst {
object LibConst {
val LIB_SOCIAL_LINK = "lib.social"
val LIB_SOCIAL_LINK = "lib.social"
}
}
}
}

152
src/main/kotlin/org/koitharu/kotatsu/parsers/util/FaviconParser.kt
Unescape Escape View File

@ -1,94 +1,92 @@
package org.koitharu.kotatsu.parsers.util
import okhttp3.Headers
import org.jsoup.nodes.Element
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.model.Favicon
import org.koitharu.kotatsu.parsers.model.Favicons
import org.koitharu.kotatsu.parsers.network.WebClient
import org.koitharu.kotatsu.parsers.util.json.mapJSON
class FaviconParser(
private val context: MangaLoaderContext,
private val domain: String,
private val headers: Headers?,
private val webClient: WebClient,
private val domain: String,
) {
suspend fun parseFavicons(): Favicons {
val url = "https://$domain"
val doc = context.httpGet(url, headers).parseHtml()
val result = HashSet<Favicon>()
val manifestLink = doc.getElementsByAttributeValue("rel", "manifest").firstOrNull()
?.attrAsAbsoluteUrlOrNull("href")
if (manifestLink != null) {
result += parseManifest(manifestLink)
}
val links = doc.getElementsByAttributeValueContaining("rel", "icon")
links.mapNotNullTo(result) { link ->
parseLink(link)
}
if (result.isEmpty()) {
result.add(createFallback())
}
return Favicons(result, url)
}
suspend fun parseFavicons(): Favicons {
val url = "https://$domain"
val doc = webClient.httpGet(url).parseHtml()
val result = HashSet<Favicon>()
val manifestLink = doc.getElementsByAttributeValue("rel", "manifest").firstOrNull()
?.attrAsAbsoluteUrlOrNull("href")
if (manifestLink != null) {
result += parseManifest(manifestLink)
}
val links = doc.getElementsByAttributeValueContaining("rel", "icon")
links.mapNotNullTo(result) { link ->
parseLink(link)
}
if (result.isEmpty()) {
result.add(createFallback())
}
return Favicons(result, url)
}
private fun parseLink(link: Element): Favicon? {
val href = link.attrAsAbsoluteUrlOrNull("href")
if (href == null || href.endsWith('/')) {
return null
}
val sizes = link.attr("sizes")
return Favicon(
url = href,
size = parseSize(sizes),
rel = link.attrOrNull("rel"),
)
}
private fun parseLink(link: Element): Favicon? {
val href = link.attrAsAbsoluteUrlOrNull("href")
if (href == null || href.endsWith('/')) {
return null
}
val sizes = link.attr("sizes")
return Favicon(
url = href,
size = parseSize(sizes),
rel = link.attrOrNull("rel"),
)
}
private fun parseSize(sizes: String): Int {
if (sizes.isEmpty() || sizes == "any") {
return 0
}
return sizes.substringBefore(' ')
.split('x', 'X', '*')
.firstNotNullOfOrNull { it.toIntOrNull() }
?: 0
}
private fun parseSize(sizes: String): Int {
if (sizes.isEmpty() || sizes == "any") {
return 0
}
return sizes.substringBefore(' ')
.split('x', 'X', '*')
.firstNotNullOfOrNull { it.toIntOrNull() }
?: 0
}
private suspend fun parseManifest(url: String): List<Favicon> {
val json = context.httpGet(url, headers).parseJson()
val icons = json.getJSONArray("icons")
return icons.mapJSON { jo ->
Favicon(
url = jo.getString("src").resolveLink(),
size = parseSize(jo.getString("sizes")),
rel = null,
)
}
}
private suspend fun parseManifest(url: String): List<Favicon> {
val json = webClient.httpGet(url).parseJson()
val icons = json.getJSONArray("icons")
return icons.mapJSON { jo ->
Favicon(
url = jo.getString("src").resolveLink(),
size = parseSize(jo.getString("sizes")),
rel = null,
)
}
}
private fun createFallback(): Favicon {
val href = "https://$domain/favicon.ico"
return Favicon(
url = href,
size = 0,
rel = null,
)
}
private fun createFallback(): Favicon {
val href = "https://$domain/favicon.ico"
return Favicon(
url = href,
size = 0,
rel = null,
)
}
private fun String.resolveLink(): String {
return when {
startsWith("http:") || startsWith("https:") -> {
this
}
private fun String.resolveLink(): String {
return when {
startsWith("http:") || startsWith("https:") -> {
this
}
startsWith('/') -> {
"https://$domain$this"
}
startsWith('/') -> {
"https://$domain$this"
}
else -> {
"https://$domain/$this"
}
}
}
}
else -> {
"https://$domain/$this"
}
}
}
}

78
src/main/kotlin/org/koitharu/kotatsu/parsers/util/MangaParserEnv.kt
Unescape Escape View File

@ -0,0 +1,78 @@
package org.koitharu.kotatsu.parsers.util
import okhttp3.HttpUrl
import org.jsoup.nodes.Element
import org.koitharu.kotatsu.parsers.InternalParsersApi
import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.exception.ParseException
import org.koitharu.kotatsu.parsers.model.Manga
import org.koitharu.kotatsu.parsers.model.MangaChapter
import org.koitharu.kotatsu.parsers.model.MangaPage
import org.koitharu.kotatsu.parsers.model.MangaTag
/**
* Create a unique id for [Manga]/[MangaChapter]/[MangaPage].
* @param url must be relative url, without a domain
* @see [Manga.id]
* @see [MangaChapter.id]
* @see [MangaPage.id]
*/
@InternalParsersApi
fun MangaParser.generateUid(url: String): Long {
var h = 1125899906842597L
source.name.forEach { c ->
h = 31 * h + c.code
}
url.forEach { c ->
h = 31 * h + c.code
}
return h
}
/**
* Create a unique id for [Manga]/[MangaChapter]/[MangaPage].
* @param id an internal identifier
* @see [Manga.id]
* @see [MangaChapter.id]
* @see [MangaPage.id]
*/
@InternalParsersApi
fun MangaParser.generateUid(id: Long): Long {
var h = 1125899906842597L
source.name.forEach { c ->
h = 31 * h + c.code
}
h = 31 * h + id
return h
}
@InternalParsersApi
fun Element.parseFailed(message: String? = null): Nothing {
throw ParseException(message, ownerDocument()?.location() ?: baseUri(), null)
}
@InternalParsersApi
fun Set<MangaTag>?.oneOrThrowIfMany(): MangaTag? {
return when {
isNullOrEmpty() -> null
size == 1 -> first()
else -> throw IllegalArgumentException("Multiple genres are not supported by this source")
}
}
val MangaParser.domain: String
get() {
return config[configKeyDomain]
}
fun MangaParser.getDomain(subdomain: String): String {
val domain = domain
return subdomain + "." + domain.removePrefix("www.")
}
fun MangaParser.urlBuilder(): HttpUrl.Builder {
return HttpUrl.Builder()
.scheme("https")
.host(domain)
}

20
src/main/kotlin/org/koitharu/kotatsu/parsers/util/OkHttp.kt
Unescape Escape View File

@ -4,16 +4,26 @@ package org.koitharu.kotatsu.parsers.util
import kotlinx.coroutines.suspendCancellableCoroutine
import okhttp3.Call
import okhttp3.Headers
import okhttp3.Response
suspend fun Call.await(): Response = suspendCancellableCoroutine { continuation ->
val callback = ContinuationCallCallback(this, continuation)
enqueue(callback)
continuation.invokeOnCancellation(callback)
val callback = ContinuationCallCallback(this, continuation)
enqueue(callback)
continuation.invokeOnCancellation(callback)
}
val Response.mimeType: String?
get() = body?.contentType()?.run { "$type/$subtype" }
get() = body?.contentType()?.run { "$type/$subtype" }
val Response.contentDisposition: String?
get() = header("Content-Disposition")
get() = header("Content-Disposition")
fun Headers.Builder.mergeWith(other: Headers, replaceExisting: Boolean): Headers.Builder {
for ((name, value) in other) {
if (replaceExisting || this[name] == null) {
this[name] = value
}
}
return this
}

44
src/test/kotlin/org/koitharu/kotatsu/parsers/AuthCheckExtension.kt
Unescape Escape View File

@ -8,27 +8,27 @@ import org.koitharu.kotatsu.parsers.util.runCatchingCancellable
class AuthCheckExtension : BeforeAllCallback {
private val loaderContext: MangaLoaderContext = MangaLoaderContextMock()
private val loaderContext: MangaLoaderContext = MangaLoaderContextMock
override fun beforeAll(context: ExtensionContext) {
for (source in MangaSource.values()) {
if (source == MangaSource.LOCAL || source == MangaSource.DUMMY) {
continue
}
val parser = source.newParser(loaderContext)
if (parser is MangaParserAuthProvider) {
checkAuthorization(source, parser)
}
}
}
override fun beforeAll(context: ExtensionContext) {
for (source in MangaSource.values()) {
if (source == MangaSource.LOCAL || source == MangaSource.DUMMY) {
continue
}
val parser = source.newParser(loaderContext)
if (parser is MangaParserAuthProvider) {
checkAuthorization(source, parser)
}
}
}
private fun checkAuthorization(source: MangaSource, parser: MangaParserAuthProvider) = runTest {
runCatchingCancellable {
parser.getUsername()
}.onSuccess { username ->
println("Signed in to ${source.name} as $username")
}.onFailure { error ->
System.err.println("Auth failed for ${source.name}: ${error.javaClass.name}(${error.message})")
}
}
}
private fun checkAuthorization(source: MangaSource, parser: MangaParserAuthProvider) = runTest {
runCatchingCancellable {
parser.getUsername()
}.onSuccess { username ->
println("Signed in to ${source.name} as $username")
}.onFailure { error ->
System.err.println("Auth failed for ${source.name}: ${error.javaClass.name}(${error.message})")
}
}
}

35
src/test/kotlin/org/koitharu/kotatsu/parsers/CloudFlareInterceptor.kt
Unescape Escape View File

@ -4,26 +4,25 @@ import okhttp3.Interceptor
import okhttp3.Response
import okhttp3.internal.closeQuietly
import org.koitharu.kotatsu.parsers.exception.CloudFlareProtectedException
import java.net.HttpURLConnection.HTTP_FORBIDDEN
import java.net.HttpURLConnection.HTTP_UNAVAILABLE
import java.net.HttpURLConnection
private const val HEADER_SERVER = "Server"
private const val SERVER_CLOUDFLARE = "cloudflare"
class CloudFlareInterceptor : Interceptor {
internal class CloudFlareInterceptor : Interceptor {
override fun intercept(chain: Interceptor.Chain): Response {
val request = chain.request()
val response = chain.proceed(request)
if (response.code == HTTP_FORBIDDEN || response.code == HTTP_UNAVAILABLE) {
if (response.header(HEADER_SERVER)?.startsWith(SERVER_CLOUDFLARE) == true) {
response.closeQuietly()
throw CloudFlareProtectedException(
url = response.request.url.toString(),
headers = request.headers,
)
}
}
return response
}
}
override fun intercept(chain: Interceptor.Chain): Response {
val request = chain.request()
val response = chain.proceed(request)
if (response.code == HttpURLConnection.HTTP_FORBIDDEN || response.code == HttpURLConnection.HTTP_UNAVAILABLE) {
if (response.header(HEADER_SERVER)?.startsWith(SERVER_CLOUDFLARE) == true) {
response.closeQuietly()
throw CloudFlareProtectedException(
url = response.request.url.toString(),
headers = request.headers,
)
}
}
return response
}
}

47
src/test/kotlin/org/koitharu/kotatsu/parsers/CommonHeadersInterceptor.kt
Unescape Escape View File

@ -0,0 +1,47 @@
package org.koitharu.kotatsu.parsers
import okhttp3.Interceptor
import okhttp3.Request
import okhttp3.Response
import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.util.domain
import org.koitharu.kotatsu.parsers.util.mergeWith
private const val HEADER_USER_AGENT = "User-Agent"
private const val HEADER_REFERER = "Referer"
internal class CommonHeadersInterceptor(
private val userAgent: String,
) : Interceptor {
override fun intercept(chain: Interceptor.Chain): Response {
val request = chain.request()
val source = request.tag(MangaSource::class.java)
val parser = source?.newParser(MangaLoaderContextMock)
val sourceHeaders = parser?.headers
val headersBuilder = request.headers.newBuilder()
if (sourceHeaders != null) {
headersBuilder.mergeWith(sourceHeaders, replaceExisting = false)
}
if (headersBuilder[HEADER_USER_AGENT] == null) {
headersBuilder[HEADER_USER_AGENT] = userAgent
}
if (headersBuilder[HEADER_REFERER] == null && parser != null) {
headersBuilder[HEADER_REFERER] = "https://${parser.domain}/"
}
val newRequest = request.newBuilder().headers(headersBuilder.build()).build()
return if (parser is Interceptor) {
parser.intercept(ProxyChain(chain, newRequest))
} else {
return chain.proceed(newRequest)
}
}
private class ProxyChain(
private val delegate: Interceptor.Chain,
private val request: Request,
) : Interceptor.Chain by delegate {
override fun request(): Request = request
}
}

96
src/test/kotlin/org/koitharu/kotatsu/parsers/MangaLoaderContextMock.kt
Unescape Escape View File

@ -10,59 +10,59 @@ import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.util.await
import java.util.concurrent.TimeUnit
internal class MangaLoaderContextMock : MangaLoaderContext() {
internal object MangaLoaderContextMock : MangaLoaderContext() {
private val userAgent = "Kotatsu/%s (Android %s; %s; %s %s; %s)".format(
/*BuildConfig.VERSION_NAME*/ "3.0",
/*Build.VERSION.RELEASE*/ "r",
/*Build.MODEL*/ "",
/*Build.BRAND*/ "",
/*Build.DEVICE*/ "",
/*Locale.getDefault().language*/ "en",
)
private val userAgent = "Kotatsu/%s (Android %s; %s; %s %s; %s)".format(
/*BuildConfig.VERSION_NAME*/ "4.3",
/*Build.VERSION.RELEASE*/ "r",
/*Build.MODEL*/ "",
/*Build.BRAND*/ "",
/*Build.DEVICE*/ "",
/*Locale.getDefault().language*/ "en",
)
override val cookieJar = InMemoryCookieJar()
override val cookieJar = InMemoryCookieJar()
override val httpClient: OkHttpClient = OkHttpClient.Builder()
.cookieJar(cookieJar)
.addInterceptor(UserAgentInterceptor(userAgent))
.addInterceptor(CloudFlareInterceptor())
.connectTimeout(20, TimeUnit.SECONDS)
.readTimeout(60, TimeUnit.SECONDS)
.writeTimeout(20, TimeUnit.SECONDS)
.build()
override val httpClient: OkHttpClient = OkHttpClient.Builder()
.cookieJar(cookieJar)
.addInterceptor(CommonHeadersInterceptor(userAgent))
.addInterceptor(CloudFlareInterceptor())
.connectTimeout(20, TimeUnit.SECONDS)
.readTimeout(60, TimeUnit.SECONDS)
.writeTimeout(20, TimeUnit.SECONDS)
.build()
init {
loadTestCookies()
}
init {
loadTestCookies()
}
override suspend fun evaluateJs(script: String): String? {
return QuackContext.create().use {
it.evaluate(script)?.toString()
}
}
override suspend fun evaluateJs(script: String): String? {
return QuackContext.create().use {
it.evaluate(script)?.toString()
}
}
override fun getConfig(source: MangaSource): MangaSourceConfig {
return SourceConfigMock()
}
override fun getConfig(source: MangaSource): MangaSourceConfig {
return SourceConfigMock()
}
suspend fun doRequest(url: String, referer: String? = null, extraHeaders: Headers? = null): Response {
val request = Request.Builder()
.get()
.url(url)
if (extraHeaders != null) {
request.headers(extraHeaders)
}
if (referer != null) {
request.header("Referer", referer)
}
return httpClient.newCall(request.build()).await()
}
suspend fun doRequest(url: String, referer: String? = null, extraHeaders: Headers? = null): Response {
val request = Request.Builder()
.get()
.url(url)
if (extraHeaders != null) {
request.headers(extraHeaders)
}
if (referer != null) {
request.header("Referer", referer)
}
return httpClient.newCall(request.build()).await()
}
private fun loadTestCookies() {
// https://addons.mozilla.org/ru/firefox/addon/cookies-txt/
javaClass.getResourceAsStream("/cookies.txt")?.use {
cookieJar.loadFromStream(it)
} ?: println("No cookies loaded!")
}
}
private fun loadTestCookies() {
// https://addons.mozilla.org/ru/firefox/addon/cookies-txt/
javaClass.getResourceAsStream("/cookies.txt")?.use {
cookieJar.loadFromStream(it)
} ?: println("No cookies loaded!")
}
}

361
src/test/kotlin/org/koitharu/kotatsu/parsers/MangaParserTest.kt
Unescape Escape View File

@ -8,6 +8,7 @@ import org.junit.jupiter.params.ParameterizedTest
import org.koitharu.kotatsu.parsers.model.Manga
import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.model.SortOrder
import org.koitharu.kotatsu.parsers.util.domain
import org.koitharu.kotatsu.parsers.util.medianOrNull
import org.koitharu.kotatsu.parsers.util.mimeType
import org.koitharu.kotatsu.test_util.isDistinct
@ -19,184 +20,184 @@ import org.koitharu.kotatsu.test_util.maxDuplicates
@ExtendWith(AuthCheckExtension::class)
internal class MangaParserTest {
private val context = MangaLoaderContextMock()
@ParameterizedTest(name = "{index}|list|{0}")
@MangaSources
fun list(source: MangaSource) = runTest {
val parser = source.newParser(context)
val list = parser.getList(20, sortOrder = SortOrder.POPULARITY, tags = null)
checkMangaList(list, "list")
assert(list.all { it.source == source })
}
@ParameterizedTest(name = "{index}|pagination|{0}")
@MangaSources
fun pagination(source: MangaSource) = runTest {
val parser = source.newParser(context)
val page1 = parser.getList(0, sortOrder = null, tags = null)
val page2 = parser.getList(page1.size, sortOrder = null, tags = null)
assert(page1.isNotEmpty()) { "Page 1 is empty" }
assert(page2.isNotEmpty()) { "Page 2 is empty" }
val intersection = page1.intersect(page2.toSet())
assert(intersection.isEmpty()) {
"Pages are intersected by " + intersection.size
}
}
@ParameterizedTest(name = "{index}|search|{0}")
@MangaSources
fun search(source: MangaSource) = runTest {
val parser = source.newParser(context)
val subject = parser.getList(20, sortOrder = SortOrder.POPULARITY, tags = null).minByOrNull {
it.title.length
} ?: error("No manga found")
val query = subject.title
check(query.isNotBlank()) { "Manga title '$query' is blank" }
val list = parser.getList(0, query)
assert(list.isNotEmpty()) { "Empty search results by \"$query\"" }
assert(list.singleOrNull { it.url == subject.url && it.id == subject.id } != null) {
"Single subject '${subject.title} (${subject.publicUrl})' not found in search results"
}
checkMangaList(list, "search('$query')")
assert(list.all { it.source == source })
}
@ParameterizedTest(name = "{index}|tags|{0}")
@MangaSources
fun tags(source: MangaSource) = runTest {
val parser = source.newParser(context)
val tags = parser.getTags()
assert(tags.isNotEmpty()) { "No tags found" }
val keys = tags.map { it.key }
assert(keys.isDistinct())
assert("" !in keys)
val titles = tags.map { it.title }
private val context = MangaLoaderContextMock
@ParameterizedTest(name = "{index}|list|{0}")
@MangaSources
fun list(source: MangaSource) = runTest {
val parser = source.newParser(context)
val list = parser.getList(20, sortOrder = SortOrder.POPULARITY, tags = null)
checkMangaList(list, "list")
assert(list.all { it.source == source })
}
@ParameterizedTest(name = "{index}|pagination|{0}")
@MangaSources
fun pagination(source: MangaSource) = runTest {
val parser = source.newParser(context)
val page1 = parser.getList(0, sortOrder = null, tags = null)
val page2 = parser.getList(page1.size, sortOrder = null, tags = null)
assert(page1.isNotEmpty()) { "Page 1 is empty" }
assert(page2.isNotEmpty()) { "Page 2 is empty" }
val intersection = page1.intersect(page2.toSet())
assert(intersection.isEmpty()) {
"Pages are intersected by " + intersection.size
}
}
@ParameterizedTest(name = "{index}|search|{0}")
@MangaSources
fun search(source: MangaSource) = runTest {
val parser = source.newParser(context)
val subject = parser.getList(20, sortOrder = SortOrder.POPULARITY, tags = null).minByOrNull {
it.title.length
} ?: error("No manga found")
val query = subject.title
check(query.isNotBlank()) { "Manga title '$query' is blank" }
val list = parser.getList(0, query)
assert(list.isNotEmpty()) { "Empty search results by \"$query\"" }
assert(list.singleOrNull { it.url == subject.url && it.id == subject.id } != null) {
"Single subject '${subject.title} (${subject.publicUrl})' not found in search results"
}
checkMangaList(list, "search('$query')")
assert(list.all { it.source == source })
}
@ParameterizedTest(name = "{index}|tags|{0}")
@MangaSources
fun tags(source: MangaSource) = runTest {
val parser = source.newParser(context)
val tags = parser.getTags()
assert(tags.isNotEmpty()) { "No tags found" }
val keys = tags.map { it.key }
assert(keys.isDistinct())
assert("" !in keys)
val titles = tags.map { it.title }
// assert(titles.isDistinct())
assert("" !in titles)
assert(tags.all { it.source == source })
val tag = tags.last()
val list = parser.getList(offset = 0, tags = setOf(tag), sortOrder = null)
checkMangaList(list, "${tag.title} (${tag.key})")
assert(list.all { it.source == source })
}
@ParameterizedTest(name = "{index}|details|{0}")
@MangaSources
fun details(source: MangaSource) = runTest {
val parser = source.newParser(context)
val list = parser.getList(20, sortOrder = SortOrder.POPULARITY, tags = null)
val manga = list[3]
parser.getDetails(manga).apply {
assert(!chapters.isNullOrEmpty()) { "Chapters are null or empty" }
assert(publicUrl.isUrlAbsolute()) { "Manga public url is not absolute: '$publicUrl'" }
assert(description != null) { "Detailed description is null: '$publicUrl'" }
assert(title.startsWith(manga.title)) {
"Titles are mismatch: '$title' and '${manga.title}' for $publicUrl"
}
assert(this.source == source)
val c = checkNotNull(chapters)
assert(c.isDistinctBy { it.id }) {
"Chapters are not distinct by id: ${c.maxDuplicates { it.id }} for $publicUrl"
}
assert(c.isDistinctBy { it.number to it.branch }) {
"Chapters are not distinct by number: ${c.maxDuplicates { it.number to it.branch }} for $publicUrl"
}
assert(c.all { it.source == source })
checkImageRequest(coverUrl, publicUrl)
largeCoverUrl?.let {
checkImageRequest(it, publicUrl)
}
}
}
@ParameterizedTest(name = "{index}|pages|{0}")
@MangaSources
fun pages(source: MangaSource) = runTest {
val parser = source.newParser(context)
val list = parser.getList(20, sortOrder = SortOrder.POPULARITY, tags = null)
val manga = list.first()
val chapter = parser.getDetails(manga).chapters?.firstOrNull() ?: error("Chapter is null")
val pages = parser.getPages(chapter)
assert(pages.isNotEmpty())
assert(pages.isDistinctBy { it.id })
assert(pages.all { it.source == source })
val page = pages.medianOrNull() ?: error("No page")
val pageUrl = parser.getPageUrl(page)
assert(pageUrl.isNotEmpty())
assert(pageUrl.isUrlAbsolute())
checkImageRequest(pageUrl, page.referer)
}
@ParameterizedTest(name = "{index}|favicon|{0}")
@MangaSources
fun favicon(source: MangaSource) = runTest {
val parser = source.newParser(context)
val favicons = parser.getFavicons()
val types = setOf("png", "svg", "ico", "gif", "jpg", "jpeg")
assert(favicons.isNotEmpty())
favicons.forEach {
assert(it.url.isUrlAbsolute()) { "Favicon url is not absolute: ${it.url}" }
assert(it.type in types) { "Unknown icon type: ${it.type}" }
}
val favicon = favicons.find(24)
checkNotNull(favicon)
checkImageRequest(favicon.url, favicons.referer)
}
@ParameterizedTest(name = "{index}|domain|{0}")
@MangaSources
fun domain(source: MangaSource) = runTest {
val parser = source.newParser(context)
val defaultDomain = parser.getDomain()
val url = HttpUrl.Builder()
.host(defaultDomain)
.scheme("https")
.toString()
val response = context.doRequest(url, extraHeaders = parser.headers)
val realUrl = response.request.url
val realDomain = realUrl.topPrivateDomain()
val realHost = realUrl.host
assert(defaultDomain == realHost || defaultDomain == realDomain) {
"Domain mismatch:\nRequired:\t\t\t$defaultDomain\nActual:\t\t\t$realDomain\nHost:\t\t\t$realHost"
}
}
@ParameterizedTest(name = "{index}|authorization|{0}")
@MangaSources
@Disabled
fun authorization(source: MangaSource) = runTest {
val parser = source.newParser(context)
if (parser is MangaParserAuthProvider) {
val username = parser.getUsername()
assert(username.isNotBlank()) { "Username is blank" }
println("Signed in to ${source.name} as $username")
}
}
private suspend fun checkMangaList(list: List<Manga>, cause: String) {
assert(list.isNotEmpty()) { "Manga list for '$cause' is empty" }
assert(list.isDistinctBy { it.id }) { "Manga list for '$cause' contains duplicated ids" }
for (item in list) {
assert(item.url.isNotEmpty()) { "Url is empty" }
assert(!item.url.isUrlAbsolute()) { "Url looks like absolute: ${item.url}" }
assert(item.coverUrl.isUrlAbsolute()) { "Cover url is not absolute: ${item.coverUrl}" }
assert(item.title.isNotEmpty()) { "Title for ${item.publicUrl} is empty" }
assert(item.publicUrl.isUrlAbsolute())
}
val testItem = list.random()
checkImageRequest(testItem.coverUrl, testItem.publicUrl)
}
private suspend fun checkImageRequest(url: String, referer: String?) {
context.doRequest(url, referer).use {
assert(it.isSuccessful) { "Request failed: ${it.code}(${it.message}): $url" }
assert(it.mimeType?.startsWith("image/") == true) {
"Wrong response mime type: ${it.mimeType}"
}
}
}
}
assert("" !in titles)
assert(tags.all { it.source == source })
val tag = tags.last()
val list = parser.getList(offset = 0, tags = setOf(tag), sortOrder = null)
checkMangaList(list, "${tag.title} (${tag.key})")
assert(list.all { it.source == source })
}
@ParameterizedTest(name = "{index}|details|{0}")
@MangaSources
fun details(source: MangaSource) = runTest {
val parser = source.newParser(context)
val list = parser.getList(20, sortOrder = SortOrder.POPULARITY, tags = null)
val manga = list[3]
parser.getDetails(manga).apply {
assert(!chapters.isNullOrEmpty()) { "Chapters are null or empty" }
assert(publicUrl.isUrlAbsolute()) { "Manga public url is not absolute: '$publicUrl'" }
assert(description != null) { "Detailed description is null: '$publicUrl'" }
assert(title.startsWith(manga.title)) {
"Titles are mismatch: '$title' and '${manga.title}' for $publicUrl"
}
assert(this.source == source)
val c = checkNotNull(chapters)
assert(c.isDistinctBy { it.id }) {
"Chapters are not distinct by id: ${c.maxDuplicates { it.id }} for $publicUrl"
}
assert(c.isDistinctBy { it.number to it.branch }) {
"Chapters are not distinct by number: ${c.maxDuplicates { it.number to it.branch }} for $publicUrl"
}
assert(c.all { it.source == source })
checkImageRequest(coverUrl, publicUrl)
largeCoverUrl?.let {
checkImageRequest(it, publicUrl)
}
}
}
@ParameterizedTest(name = "{index}|pages|{0}")
@MangaSources
fun pages(source: MangaSource) = runTest {
val parser = source.newParser(context)
val list = parser.getList(20, sortOrder = SortOrder.POPULARITY, tags = null)
val manga = list.first()
val chapter = parser.getDetails(manga).chapters?.firstOrNull() ?: error("Chapter is null")
val pages = parser.getPages(chapter)
assert(pages.isNotEmpty())
assert(pages.isDistinctBy { it.id })
assert(pages.all { it.source == source })
val page = pages.medianOrNull() ?: error("No page")
val pageUrl = parser.getPageUrl(page)
assert(pageUrl.isNotEmpty())
assert(pageUrl.isUrlAbsolute())
checkImageRequest(pageUrl, page.referer)
}
@ParameterizedTest(name = "{index}|favicon|{0}")
@MangaSources
fun favicon(source: MangaSource) = runTest {
val parser = source.newParser(context)
val favicons = parser.getFavicons()
val types = setOf("png", "svg", "ico", "gif", "jpg", "jpeg")
assert(favicons.isNotEmpty())
favicons.forEach {
assert(it.url.isUrlAbsolute()) { "Favicon url is not absolute: ${it.url}" }
assert(it.type in types) { "Unknown icon type: ${it.type}" }
}
val favicon = favicons.find(24)
checkNotNull(favicon)
checkImageRequest(favicon.url, favicons.referer)
}
@ParameterizedTest(name = "{index}|domain|{0}")
@MangaSources
fun domain(source: MangaSource) = runTest {
val parser = source.newParser(context)
val defaultDomain = parser.domain
val url = HttpUrl.Builder()
.host(defaultDomain)
.scheme("https")
.toString()
val response = context.doRequest(url, extraHeaders = parser.headers)
val realUrl = response.request.url
val realDomain = realUrl.topPrivateDomain()
val realHost = realUrl.host
assert(defaultDomain == realHost || defaultDomain == realDomain) {
"Domain mismatch:\nRequired:\t\t\t$defaultDomain\nActual:\t\t\t$realDomain\nHost:\t\t\t$realHost"
}
}
@ParameterizedTest(name = "{index}|authorization|{0}")
@MangaSources
@Disabled
fun authorization(source: MangaSource) = runTest {
val parser = source.newParser(context)
if (parser is MangaParserAuthProvider) {
val username = parser.getUsername()
assert(username.isNotBlank()) { "Username is blank" }
println("Signed in to ${source.name} as $username")
}
}
private suspend fun checkMangaList(list: List<Manga>, cause: String) {
assert(list.isNotEmpty()) { "Manga list for '$cause' is empty" }
assert(list.isDistinctBy { it.id }) { "Manga list for '$cause' contains duplicated ids" }
for (item in list) {
assert(item.url.isNotEmpty()) { "Url is empty" }
assert(!item.url.isUrlAbsolute()) { "Url looks like absolute: ${item.url}" }
assert(item.coverUrl.isUrlAbsolute()) { "Cover url is not absolute: ${item.coverUrl}" }
assert(item.title.isNotEmpty()) { "Title for ${item.publicUrl} is empty" }
assert(item.publicUrl.isUrlAbsolute())
}
val testItem = list.random()
checkImageRequest(testItem.coverUrl, testItem.publicUrl)
}
private suspend fun checkImageRequest(url: String, referer: String?) {
context.doRequest(url, referer).use {
assert(it.isSuccessful) { "Request failed: ${it.code}(${it.message}): $url" }
assert(it.mimeType?.startsWith("image/") == true) {
"Wrong response mime type: ${it.mimeType}"
}
}
}
}

2
src/test/kotlin/org/koitharu/kotatsu/parsers/MangaSources.kt
Unescape Escape View File

@ -4,4 +4,4 @@ import org.junit.jupiter.params.provider.EnumSource
import org.koitharu.kotatsu.parsers.model.MangaSource
@EnumSource(MangaSource::class, names = ["LOCAL", "DUMMY"], mode = EnumSource.Mode.EXCLUDE)
internal annotation class MangaSources
internal annotation class MangaSources

20
src/test/kotlin/org/koitharu/kotatsu/parsers/UserAgentInterceptor.kt
Unescape Escape View File

@ -1,20 +0,0 @@
package org.koitharu.kotatsu.parsers
import okhttp3.Interceptor
import okhttp3.Response
private const val HEADER_USER_AGENT = "User-Agent"
internal class UserAgentInterceptor(
private val userAgent: String,
) : Interceptor {
override fun intercept(chain: Interceptor.Chain): Response {
val request = chain.request()
val newRequest = if (request.header(HEADER_USER_AGENT) == null) {
request.newBuilder().header(HEADER_USER_AGENT, userAgent).build()
} else {
request
}
return chain.proceed(newRequest)
}
}
Write Preview
Loading…
Cancel
Save