44 Commits

Author SHA1 Message Date
9952f8f00e [MangaTrend] New source (close #65) 2022-09-25 17:31:06 +03:00
de0c12c078 Add contribution guidelines 2022-09-14 20:10:18 +03:00
b3a9c5fcda Propagate deprecation from MangaParser to MangaSource 2022-09-07 13:57:00 +03:00
c73d34b04f [Madara] Fix date parsing 2022-09-07 13:45:15 +03:00
583b24854f [Remanga] Omit paid chapters 2022-09-07 13:15:51 +03:00
1c79b29845 Merge branch 'master' of github.com:KotatsuApp/kotatsu-parsers 2022-09-07 13:07:02 +03:00
11f19b1f76 Add asserts 2022-09-07 13:05:23 +03:00
c348d68983 Add deprecation annotation to some sources 2022-09-04 17:53:29 +03:00
6d7506cc1b [CloneManga] Add source 2022-09-04 17:52:54 +03:00
551a1d70ae [MangaInUa] Fix chapters list #43 2022-09-03 10:48:35 +03:00
7ea80b51e1 Merge pull request #46 from VietAnh14/source/nettruyen
[NetTruyen] New source
2022-09-02 16:42:24 +03:00
ca6d5c429c Merge pull request #45 from VietAnh14/source/blogtruyen
[BlogTruyen] New source
2022-09-02 16:42:10 +03:00
vianh
eb1bcac175 [NetTruyen] Synchronize tags with mutex, use WordSet as suggestion 2022-09-02 18:36:00 +07:00
ViAnh
704a063cb3 [NetTruyen] Apply suggestions from code review
Co-authored-by: Koitharu <nvasya95@gmail.com>
2022-09-02 17:44:36 +07:00
vianh
1577afc4a9 [BlogTruyen] Synchronize tags with mutex 2022-09-02 17:35:43 +07:00
ViAnh
aec404c0a1 Apply suggestions from code review
Co-authored-by: Koitharu <nvasya95@gmail.com>
2022-09-02 17:23:50 +07:00
vianh
1bad7eeb75 [NetTruyen] Update to use PagedMangaParser 2022-09-02 11:15:00 +07:00
vianh
e3612ef466 [BlogTruyen] Update to use PagedMangaParser 2022-09-02 10:57:28 +07:00
vianh
01cc33e309 [NetTruyen] New source 2022-09-02 00:44:54 +07:00
vianh
599f606aef Remove deprecated favicon url 2022-09-01 22:27:48 +07:00
vianh
c976e0e4e0 [BlogTruyen] New source 2022-09-01 22:17:31 +07:00
f112a06ab6 [IsekaiScanEu] Fix chapters loading #41 2022-08-17 16:45:18 +03:00
8709c3dd0c [Grouple] Fix rating parsing 2022-08-08 12:13:25 +03:00
e2308214a7 [MangaDex] Speed up large chapters list loading (hotfix) 2022-08-04 13:32:53 +03:00
baf6234a2a [MangaDex] Speed up large chapters list loading 2022-08-04 13:19:48 +03:00
85bfe42ddf [Tatakae Scans] Fix favicon 2022-07-30 17:11:08 +03:00
dac06ddd0b [Tatakae Scans] New source 2022-07-30 15:10:44 +03:00
b632571ebc Update readme 2022-07-28 10:04:07 +03:00
7588617316 New source: PrismaScans 2022-07-27 15:56:06 +03:00
dd6de64c44 New manga source: UnionManga 2022-07-27 15:56:06 +03:00
fadb06aabb [Remanga] Fix details parsing 2022-07-20 19:41:49 +03:00
c4abb758f3 Make jsoup dependency api 2022-07-20 18:19:07 +03:00
6495ddf277 Add NotFoundException for 404 response 2022-07-20 17:09:37 +03:00
30071709af Add base url to ParseException 2022-07-19 11:36:26 +03:00
37fda4bd9e Oops! 2022-07-14 17:00:15 +03:00
a1dc901439 Update issue templates 2022-07-14 16:58:54 +03:00
6af8cec134 Update dependencies 2022-07-14 14:35:03 +03:00
67f6af63f8 [Madara] New source Mangas Origines (fr) 2022-07-14 11:59:00 +03:00
be595af075 Paginator class to compute page number by offset 2022-07-14 11:17:55 +03:00
c5c84c8630 Merge branch 'feature/favicons' 2022-07-13 15:13:56 +03:00
abc7eca40b Rename parseFavicons to getFavicons 2022-07-13 15:13:46 +03:00
2d1907569b [MangaLib] Fix authorization required detection 2022-07-12 10:43:29 +03:00
cb7a23950a [BatoTo] Fix pages parsing 2022-07-12 10:38:25 +03:00
941cc61577 Ignore malformed domains while ops with cookies 2022-07-12 10:21:21 +03:00
53 changed files with 1772 additions and 409 deletions

View File

@@ -25,27 +25,7 @@ body:
2. Second step
3. Issue here
validations:
required: true
- type: textarea
id: expected-behavior
attributes:
label: Expected behavior
placeholder: |
Example:
"This should happen..."
validations:
required: true
- type: textarea
id: actual-behavior
attributes:
label: Actual behavior
placeholder: |
Example:
"This happened instead..."
validations:
required: true
required: false
- type: input
id: kotatsu-version
@@ -54,7 +34,7 @@ body:
description: |
You can find your Kotatsu version in **Settings → About**.
placeholder: |
Example: "3.3.1"
Example: "3.3"
validations:
required: true
@@ -67,7 +47,7 @@ body:
placeholder: |
Example: "Android 12"
validations:
required: true
required: false
- type: textarea
id: other-details
@@ -80,15 +60,6 @@ body:
id: acknowledgements
attributes:
label: Acknowledgements
description: Your issue will be closed if you haven't done these steps.
options:
- label: I have searched the existing issues and this is a new ticket, **NOT** a duplicate or related to another open issue.
required: true
- label: I have written a short but informative title.
required: true
- label: I have updated the app to version **[3.3.1](https://github.com/KotatsuApp/Kotatsu/releases/latest)**.
required: true
- label: If this is an issue with the app itself, I should be opening an issue in the [app repository](https://github.com/KotatsuApp/Kotatsu/issues/new/choose).
required: true
- label: I will fill out all of the requested information in this form.
required: true

View File

@@ -25,15 +25,6 @@ body:
id: acknowledgements
attributes:
label: Acknowledgements
description: Your issue will be closed if you haven't done these steps.
options:
- label: I have searched the existing issues and this is a new ticket, **NOT** a duplicate or related to another open issue.
required: true
- label: I have written a short but informative title.
required: true
- label: If this is an issue with the app itself, I should be opening an issue in the [app repository](https://github.com/KotatsuApp/Kotatsu/issues/new/choose).
required: true
- label: I have updated the app to version **[3.3.1](https://github.com/KotatsuApp/Kotatsu/releases/latest)**.
required: true
- label: I will fill out all of the requested information in this form.
required: true

View File

@@ -41,17 +41,10 @@ body:
id: acknowledgements
attributes:
label: Acknowledgements
description: Your issue will be closed if you haven't done these steps.
options:
- label: I have searched the existing issues and this is a new ticket, **NOT** a duplicate or related to another open issue.
required: true
- label: I have written a title with source name.
required: true
- label: I have updated the app to version **[3.3.1](https://github.com/KotatsuApp/Kotatsu/releases/latest)**.
required: true
- label: I have checked that the source does not already exist on the app.
required: true
- label: I have checked that the source does not already exist by searching the [GitHub repository](https://github.com/KotatsuApp/kotatsu-parsers) and verified it does not appear in the code base.
required: true
- label: I will fill out all of the requested information in this form.
required: true

3
.gitignore vendored
View File

@@ -75,4 +75,5 @@ build/
.idea/**/ktlint.xml
.idea/codeStyles/
src/test/resources/cookies.txt
src/test/resources/cookies.txt
local.properties

View File

@@ -1,7 +0,0 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="TestsAndReport" type="CompoundRunConfigurationType">
<toRun name="MangaParserTest" type="GradleRunConfiguration" />
<toRun name="kotatsu-parsers [generateTestsReport]" type="GradleRunConfiguration" />
<method v="2" />
</configuration>
</component>

90
CONTRIBUTING.md Normal file
View File

@@ -0,0 +1,90 @@
# Contributing
The following is guide for creating a Kotatsu parsers. Thanks for taking the time to contribute!
## Prerequisites
Before you start, please note that the ability to use following technologies is **required**.
- Basic [Android development](https://developer.android.com/)
- [Kotlin](https://kotlinlang.org/)
- Web scraping ([JSoup](https://jsoup.org/)) or JSON API
### Tools
- [Android Studio](https://developer.android.com/studio)
- [IntelliJ IDEA](https://www.jetbrains.com/idea/) (Community edition is enough)
- Android device (or emulator)
Kotatsu parsers is not a part of Android application, but you can easily develop and test it directly inside an Android
application project and relocate it to the library project when done.
### Before you start
First, take a look at `kotatsu-parsers` project structure. Each parser is a single class that
extends `MangaParser` class and have a `MangaSourceParser` annotation.
Also pay attention on extensions in `util` package. For example, extensions from `Jsoup` file
should be used instead of existing JSoup functions because they have better nullability support
and improved error messages.
## Writing your parser
So, you want to create a parser, that will provide access to manga from a website.
First, you should explore a website for API availability.
If it does not contain any documentation about
API, [explore network requests](https://firefox-source-docs.mozilla.org/devtools-user/):
some websites use ajax.
- [Example](https://github.com/KotatsuApp/kotatsu-parsers/blob/master/src/main/kotlin/org/koitharu/kotatsu/parsers/site/DesuMeParser.kt)
of Json API usage.
- [Example](https://github.com/KotatsuApp/kotatsu-parsers/blob/master/src/main/kotlin/org/koitharu/kotatsu/parsers/site/AnibelParser.kt)
of GraphQL API usage
- [Example](https://github.com/KotatsuApp/kotatsu-parsers/blob/master/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaTownParser.kt)
of pure HTML parsing.
If website is based on some engine it is rationally to use common base class for this one (for example, Madara wordress
theme
and the `MadaraParser` class)
### Parser class skeleton
Parser class must have exactly one primary constructor parameter of type `MangaLoaderContext` and have an
`MangaSourceParser` annotation that provides internal name, title and language of a manga source.
All functions in `MangaParser` class are documented. Pay attention to some peculiarities:
- Never hardcode domain. Specify default domain in `configKeyDomain` field and obtain an actual one using `getDomain()`.
- All ids must be unique and domain-independent. Use `generateUid` functions with relative url or some internal id which
is unique across the manga source.
- `sortOrders` set should not be empty. If your source is not support sorting, specify one most relevance value.
- If you cannot obtain direct links to pages images inside `getPages` method, it is ok to use an intermediate url
as `Page.url` and fetch a direct link at `getPageUrl` function.
- `getFaviconUrl` function is deprecated, but Kotatsu application before v4 still use it, so it is recommended to
override it too.
- You can use _asserts_ to check some optional fields. For example. `Manga.author` field is not required, but if your
source provide such information, add `assert(it != null)`. This will not have any effect on production but help to
find issues during unit testing.
- If your source website (or it's api) uses pages for pagination instead of offset you should extend `PagedMangaParser`
instead of `MangaParser`.
## Development process
During the development it is recommended (but not necessary) to write it directly
in the Kotatsu android application project. You can use `core.parser.DummyParser` class as a sandbox. `Dummy` manga
source is available in debug Kotatsu build.
Once parser is ready you can relocate your code into `kotatsu-parsers` library project in a `site` package and create a
Pull Request.
### Testing
It is recommended to run unit tests before submitting a PR.
- Temporary modify the `MangaSources` annotation class: specify your parser(s) name(s) and change mode
to `EnumSource.Mode.INCLUDE`
- Run the `MangaParserTest` (`gradlew :test --tests "org.koitharu.kotatsu.parsers.MangaParserTest"`)
- Optionally, you can run the `generateTestsReport` gradle task to get a pretty readable html report from test results.
## Help
If you need a help or have some questions, ask a community in our [Discord server](https://discord.gg/NNJ5RgVBC5).

View File

@@ -2,48 +2,59 @@
Library that provides manga sources.
[![](https://jitpack.io/v/nv95/kotatsu-parsers.svg)](https://jitpack.io/#nv95/kotatsu-parsers) ![Kotlin](https://img.shields.io/github/languages/top/nv95/kotatsu-parsers) ![License](https://img.shields.io/github/license/nv95/Kotatsu) [![Discord](https://img.shields.io/discord/898363402467045416?color=5865f2&label=discord)](https://discord.gg/NNJ5RgVBC5)
[![](https://jitpack.io/v/KotatsuApp/kotatsu-parsers.svg)](https://jitpack.io/#KotatsuApp/kotatsu-parsers) ![Kotlin](https://img.shields.io/github/languages/top/KotatsuApp/kotatsu-parsers) ![License](https://img.shields.io/github/license/KotatsuApp/Kotatsu) [![Discord](https://img.shields.io/discord/898363402467045416?color=5865f2&label=discord)](https://discord.gg/NNJ5RgVBC5)
### Usage
## Usage
1. Add it in your root build.gradle at the end of repositories:
```groovy
allprojects {
repositories {
...
maven { url 'https://jitpack.io' }
}
}
```
```groovy
allprojects {
repositories {
...
maven { url 'https://jitpack.io' }
}
}
```
2. Add the dependency
2. Add the dependency
For Java/Kotlin project:
For Java/Kotlin project:
```groovy
dependencies {
implementation("com.github.nv95:kotatsu-parsers:$parsers_version")
implementation("com.github.KotatsuApp:kotatsu-parsers:$parsers_version")
}
```
For Android project:
For Android project:
```groovy
dependencies {
implementation("com.github.nv95:kotatsu-parsers:$parsers_version") {
implementation("com.github.KotatsuApp:kotatsu-parsers:$parsers_version") {
exclude group: 'org.json', module: 'json'
}
}
```
See for versions at [JitPack](https://jitpack.io/#nv95/kotatsu-parsers)
See for versions at [JitPack](https://jitpack.io/#KotatsuApp/kotatsu-parsers)
3. Usage in code
```kotlin
val parser = MangaSource.MANGADEX.newParser(mangaLoaderContext)
```
`mangaLoaderContext` is an implementation of the `MangaLoaderContext` class.
See [Android](https://github.com/nv95/Kotatsu/blob/devel/app/src/main/java/org/koitharu/kotatsu/core/parser/MangaLoaderContextImpl.kt) and [Non-Android](https://github.com/nv95/kotatsu-dl/blob/master/src/main/kotlin/org/koitharu/kotatsu_dl/env/MangaLoaderContextImpl.kt) implementation examples.
```kotlin
val parser = MangaSource.MANGADEX.newParser(mangaLoaderContext)
```
Note that the `MangaSource.LOCAL` and `MangaSource.DUMMY` parsers cannot be instantiated.
`mangaLoaderContext` is an implementation of the `MangaLoaderContext` class.
See [Android](https://github.com/KotatsuApp/Kotatsu/blob/devel/app/src/main/java/org/koitharu/kotatsu/core/parser/MangaLoaderContextImpl.kt)
and [Non-Android](https://github.com/KotatsuApp/kotatsu-dl/blob/master/src/main/kotlin/org/koitharu/kotatsu_dl/env/MangaLoaderContextImpl.kt)
implementation examples.
Note that the `MangaSource.LOCAL` and `MangaSource.DUMMY` parsers cannot be instantiated.
## Contribution
See [CONTRIBUTING.md](./CONTRIBUTING.md) for the guidelines.
## DMCA disclaimer
The developers of this application does not have any affiliation with the content available in the app. It is collecting
from the sources freely available through any web browser.

View File

@@ -20,6 +20,7 @@ compileKotlin {
freeCompilerArgs += [
'-opt-in=kotlin.RequiresOptIn',
'-opt-in=kotlin.contracts.ExperimentalContracts',
'-opt-in=kotlinx.coroutines.ExperimentalCoroutinesApi',
'-opt-in=org.koitharu.kotatsu.parsers.InternalParsersApi',
]
}
@@ -53,19 +54,19 @@ afterEvaluate {
}
dependencies {
implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-core:1.6.3'
implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-core:1.6.4'
implementation 'com.squareup.okhttp3:okhttp:4.10.0'
implementation 'com.squareup.okio:okio:3.2.0'
implementation 'org.jsoup:jsoup:1.15.1'
api 'org.jsoup:jsoup:1.15.2'
implementation 'org.json:json:20220320'
implementation 'androidx.collection:collection-ktx:1.2.0'
ksp project(':kotatsu-parsers-ksp')
testImplementation 'org.junit.jupiter:junit-jupiter-api:5.8.2'
testImplementation 'org.junit.jupiter:junit-jupiter-engine:5.8.2'
testImplementation 'org.junit.jupiter:junit-jupiter-params:5.8.2'
testImplementation 'org.jetbrains.kotlinx:kotlinx-coroutines-test:1.6.3'
testImplementation 'org.junit.jupiter:junit-jupiter-api:5.9.0'
testImplementation 'org.junit.jupiter:junit-jupiter-engine:5.9.0'
testImplementation 'org.junit.jupiter:junit-jupiter-params:5.9.0'
testImplementation 'org.jetbrains.kotlinx:kotlinx-coroutines-test:1.6.4'
testImplementation 'io.webfolder:quickjs:1.1.0'
}

View File

@@ -69,6 +69,7 @@ class ParserProcessor(
import org.koitharu.kotatsu.parsers.model.MangaSource
@Suppress("DEPRECATION")
fun MangaSource.newParser(context: MangaLoaderContext): MangaParser = when (this) {
""".trimIndent(),
@@ -121,6 +122,7 @@ class ParserProcessor(
logger.error("Only non-abstract can be annotated with @MangaSourceParser", classDeclaration)
}
val annotation = classDeclaration.annotations.single { it.shortName.asString() == "MangaSourceParser" }
val deprecation = classDeclaration.annotations.singleOrNull { it.shortName.asString() == "Deprecated" }
val name = annotation.arguments.single { it.name?.asString() == "name" }.value as String
val title = annotation.arguments.single { it.name?.asString() == "title" }.value as String
val locale = annotation.arguments.single { it.name?.asString() == "locale" }.value as String
@@ -142,11 +144,15 @@ class ParserProcessor(
classDeclaration,
)
}
val className = classDeclaration.qualifiedName?.asString()
factoryWriter?.write("\tMangaSource.$name -> $className(context)\n")
val deprecationString = if (deprecation != null) {
val reason = deprecation.arguments
.find { it.name?.asString() == "message" }?.value?.toString() ?: "Unknown reason"
"@Deprecated(\"$reason\") "
} else ""
val localeComment = localeTitle?.toTitleCase(localeObj)?.let { " /* $it */" }.orEmpty()
sourcesWriter?.write("\t$name(\"$title\", $localeString$localeComment),\n")
sourcesWriter?.write("\t$deprecationString$name(\"$title\", $localeString$localeComment),\n")
}
}
}

View File

@@ -1,12 +1,14 @@
package org.koitharu.kotatsu.parsers
import okhttp3.*
import okhttp3.HttpUrl.Companion.toHttpUrl
import okhttp3.MediaType.Companion.toMediaType
import okhttp3.RequestBody.Companion.toRequestBody
import org.json.JSONObject
import org.jsoup.HttpStatusException
import org.koitharu.kotatsu.parsers.config.MangaSourceConfig
import org.koitharu.kotatsu.parsers.exception.GraphQLException
import org.koitharu.kotatsu.parsers.exception.NotFoundException
import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.util.await
import org.koitharu.kotatsu.parsers.util.parseJson
@@ -23,7 +25,7 @@ abstract class MangaLoaderContext {
* @param url
* @param headers an additional headers for request, may be null
*/
suspend fun httpGet(url: String, headers: Headers? = null): Response {
suspend fun httpGet(url: HttpUrl, headers: Headers? = null): Response {
val request = Request.Builder()
.get()
.url(url)
@@ -33,6 +35,11 @@ abstract class MangaLoaderContext {
return httpClient.newCall(request.build()).await().ensureSuccess()
}
suspend fun httpGet(url: String, headers: Headers? = null): Response {
return httpGet(url.toHttpUrl(), headers)
}
/**
* Do a HEAD http request to specific url
* @param url
@@ -140,12 +147,10 @@ abstract class MangaLoaderContext {
abstract fun getConfig(source: MangaSource): MangaSourceConfig
private fun Response.ensureSuccess() = apply {
private fun Response.ensureSuccess(): Response {
val exception: Exception? = when (code) { // Catch some error codes, not all
404,
in 500..599,
-> HttpStatusException(message, code, request.url.toString())
404 -> NotFoundException(message, request.url.toString())
in 500..599 -> HttpStatusException(message, code, request.url.toString())
else -> null
}
if (exception != null) {
@@ -156,5 +161,6 @@ abstract class MangaLoaderContext {
}
throw exception
}
return this
}
}

View File

@@ -3,6 +3,8 @@ package org.koitharu.kotatsu.parsers
import androidx.annotation.CallSuper
import androidx.annotation.VisibleForTesting
import okhttp3.Headers
import okhttp3.HttpUrl
import org.jsoup.nodes.Element
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.exception.ParseException
import org.koitharu.kotatsu.parsers.model.*
@@ -69,7 +71,7 @@ abstract class MangaParser @InternalParsersApi constructor(val source: MangaSour
* @param offset starting from 0 and used for pagination.
* @param query search query
*/
suspend fun getList(offset: Int, query: String): List<Manga> {
open suspend fun getList(offset: Int, query: String): List<Manga> {
return getList(offset, query, null, defaultSortOrder)
}
@@ -81,7 +83,7 @@ abstract class MangaParser @InternalParsersApi constructor(val source: MangaSour
* @param tags genres for filtering, values from [getTags] and [Manga.tags]. May be null or empty
* @param sortOrder one of [sortOrders] or null for default value
*/
suspend fun getList(offset: Int, tags: Set<MangaTag>?, sortOrder: SortOrder?): List<Manga> {
open suspend fun getList(offset: Int, tags: Set<MangaTag>?, sortOrder: SortOrder?): List<Manga> {
return getList(offset, null, tags, sortOrder ?: defaultSortOrder)
}
@@ -117,7 +119,10 @@ abstract class MangaParser @InternalParsersApi constructor(val source: MangaSour
)
open fun getFaviconUrl() = "https://${getDomain()}/favicon.ico"
open suspend fun parseFavicons(): Favicons {
/**
* Parse favicons from the main page of the source`s website
*/
open suspend fun getFavicons(): Favicons {
return FaviconParser(context, getDomain(), headers).parseFavicons()
}
@@ -137,6 +142,12 @@ abstract class MangaParser @InternalParsersApi constructor(val source: MangaSour
return subdomain + "." + domain.removePrefix("www.")
}
fun urlBuilder(): HttpUrl.Builder {
return HttpUrl.Builder()
.scheme("https")
.host(getDomain())
}
/**
* Create a unique id for [Manga]/[MangaChapter]/[MangaPage].
* @param url must be relative url, without a domain
@@ -174,8 +185,8 @@ abstract class MangaParser @InternalParsersApi constructor(val source: MangaSour
}
@InternalParsersApi
protected fun parseFailed(message: String? = null): Nothing {
throw ParseException(message, null)
protected fun Element.parseFailed(message: String? = null): Nothing {
throw ParseException(message, ownerDocument()?.location() ?: baseUri(), null)
}
@InternalParsersApi

View File

@@ -0,0 +1,50 @@
package org.koitharu.kotatsu.parsers
import org.koitharu.kotatsu.parsers.model.Manga
import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.model.MangaTag
import org.koitharu.kotatsu.parsers.model.SortOrder
import org.koitharu.kotatsu.parsers.util.Paginator
@InternalParsersApi
abstract class PagedMangaParser(
source: MangaSource,
pageSize: Int,
searchPageSize: Int = pageSize,
) : MangaParser(source) {
protected val paginator = Paginator(pageSize)
protected val searchPaginator = Paginator(searchPageSize)
override suspend fun getList(offset: Int, query: String): List<Manga> {
return getList(searchPaginator, offset, query, null, defaultSortOrder)
}
override suspend fun getList(offset: Int, tags: Set<MangaTag>?, sortOrder: SortOrder?): List<Manga> {
return getList(paginator, offset, null, tags, sortOrder ?: defaultSortOrder)
}
@InternalParsersApi
@Deprecated("You should use getListPage for PagedMangaParser", level = DeprecationLevel.HIDDEN)
final override suspend fun getList(
offset: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> = throw UnsupportedOperationException("You should use getListPage for PagedMangaParser")
abstract suspend fun getListPage(page: Int, query: String?, tags: Set<MangaTag>?, sortOrder: SortOrder): List<Manga>
private suspend fun getList(
paginator: Paginator,
offset: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val page = paginator.getPage(offset)
val list = getListPage(page, query, tags, sortOrder)
paginator.onListReceived(offset, page, list.size)
return list
}
}

View File

@@ -0,0 +1,3 @@
package org.koitharu.kotatsu.parsers.exception
class ContentUnavailableException(message: String) : RuntimeException(message)

View File

@@ -0,0 +1,9 @@
package org.koitharu.kotatsu.parsers.exception
import org.jsoup.HttpStatusException
import java.net.HttpURLConnection
class NotFoundException(
message: String,
url: String,
) : HttpStatusException(message, HttpURLConnection.HTTP_NOT_FOUND, url)

View File

@@ -3,6 +3,7 @@ package org.koitharu.kotatsu.parsers.exception
import org.koitharu.kotatsu.parsers.InternalParsersApi
class ParseException @InternalParsersApi @JvmOverloads constructor(
message: String?,
val shortMessage: String?,
val url: String,
cause: Throwable? = null,
) : RuntimeException(message, cause)
) : RuntimeException("$shortMessage at $url", cause)

View File

@@ -5,9 +5,10 @@ import org.json.JSONArray
import org.json.JSONObject
import org.jsoup.nodes.Element
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.exception.ParseException
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import java.nio.charset.StandardCharsets
@@ -17,11 +18,12 @@ import javax.crypto.Cipher
import javax.crypto.spec.IvParameterSpec
import javax.crypto.spec.SecretKeySpec
private const val PAGE_SIZE = 60
private const val PAGE_SIZE_SEARCH = 20
@MangaSourceParser("BATOTO", "Bato.To")
internal class BatoToParser(override val context: MangaLoaderContext) : MangaParser(MangaSource.BATOTO) {
internal class BatoToParser(override val context: MangaLoaderContext) : PagedMangaParser(
source = MangaSource.BATOTO,
pageSize = 60,
searchPageSize = 20,
) {
override val sortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.NEWEST,
@@ -35,17 +37,15 @@ internal class BatoToParser(override val context: MangaLoaderContext) : MangaPar
arrayOf("bato.to", "mto.to", "mangatoto.com", "battwo.com", "batotwo.com", "comiko.net", "batotoo.com"),
)
override suspend fun getList(
offset: Int,
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
if (!query.isNullOrEmpty()) {
return search(offset, query)
return search(page, query)
}
val page = (offset / PAGE_SIZE) + 1
@Suppress("NON_EXHAUSTIVE_WHEN_STATEMENT")
val url = buildString {
append("https://")
@@ -71,8 +71,8 @@ internal class BatoToParser(override val context: MangaLoaderContext) : MangaPar
override suspend fun getDetails(manga: Manga): Manga {
val root = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml()
.getElementById("mainer") ?: parseFailed("Cannot find root")
val details = root.selectFirst(".detail-set") ?: parseFailed("Cannot find detail-set")
.requireElementById("mainer")
val details = root.selectFirstOrThrow(".detail-set")
val attrs = details.selectFirst(".attr-main")?.select(".attr-item")?.associate {
it.child(0).text().trim() to it.child(1)
}.orEmpty()
@@ -105,7 +105,7 @@ internal class BatoToParser(override val context: MangaLoaderContext) : MangaPar
val scripts = context.httpGet(fullUrl).parseHtml().select("script")
for (script in scripts) {
val scriptSrc = script.html()
val p = scriptSrc.indexOf("const images =")
val p = scriptSrc.indexOf("const imgHttpLis =")
if (p == -1) continue
val start = scriptSrc.indexOf('[', p)
val end = scriptSrc.indexOf(';', start)
@@ -113,19 +113,19 @@ internal class BatoToParser(override val context: MangaLoaderContext) : MangaPar
continue
}
val images = JSONArray(scriptSrc.substring(start, end))
val batoJs = scriptSrc.substringBetweenFirst("batojs =", ";")?.trim(' ', '"', '\n')
?: parseFailed("Cannot find batojs")
val server = scriptSrc.substringBetweenFirst("server =", ";")?.trim(' ', '"', '\n')
?: parseFailed("Cannot find server")
val password = context.evaluateJs(batoJs)?.removeSurrounding('"')
?: parseFailed("Cannot evaluate batojs")
val serverDecrypted = decryptAES(server, password).removeSurrounding('"')
val batoPass = scriptSrc.substringBetweenFirst("batoPass =", ";")?.trim(' ', '"', '\n')
?: script.parseFailed("Cannot find batoPass")
val batoWord = scriptSrc.substringBetweenFirst("batoWord =", ";")?.trim(' ', '"', '\n')
?: script.parseFailed("Cannot find batoWord")
val password = context.evaluateJs(batoPass)?.removeSurrounding('"')
?: script.parseFailed("Cannot evaluate batoPass")
val args = JSONArray(decryptAES(batoWord, password))
val result = ArrayList<MangaPage>(images.length())
repeat(images.length()) { i ->
val url = images.getString(i)
result += MangaPage(
id = generateUid(url),
url = if (url.startsWith("http")) url else "$serverDecrypted$url",
url = url + "?" + args.getString(i),
referer = fullUrl,
preview = null,
source = source,
@@ -133,13 +133,13 @@ internal class BatoToParser(override val context: MangaLoaderContext) : MangaPar
}
return result
}
parseFailed("Cannot find images list")
throw ParseException("Cannot find images list", fullUrl)
}
override suspend fun getTags(): Set<MangaTag> {
val scripts = context.httpGet(
"https://${getDomain()}/browse",
).parseHtml().select("script")
).parseHtml().selectOrThrow("script")
for (script in scripts) {
val genres = script.html().substringBetweenFirst("const _genres =", ";") ?: continue
val jo = JSONObject(genres)
@@ -154,13 +154,12 @@ internal class BatoToParser(override val context: MangaLoaderContext) : MangaPar
}
return result
}
parseFailed("Cannot find gernes list")
throw ParseException("Cannot find gernes list", scripts[0].baseUri())
}
override fun getFaviconUrl(): String = "https://styles.amarkcdn.com/img/batoto/favicon.ico?v0"
private suspend fun search(offset: Int, query: String): List<Manga> {
val page = (offset / PAGE_SIZE_SEARCH) + 1
private suspend fun search(page: Int, query: String): List<Manga> {
val url = buildString {
append("https://")
append(getDomain())
@@ -175,7 +174,7 @@ internal class BatoToParser(override val context: MangaLoaderContext) : MangaPar
private fun getActivePage(body: Element): Int = body.select("nav ul.pagination > li.page-item.active")
.lastOrNull()
?.text()
?.toIntOrNull() ?: parseFailed("Cannot determine current page")
?.toIntOrNull() ?: body.parseFailed("Cannot determine current page")
private suspend fun parseList(url: String, page: Int): List<Manga> {
val body = context.httpGet(url).parseHtml().body()
@@ -186,11 +185,11 @@ internal class BatoToParser(override val context: MangaLoaderContext) : MangaPar
if (activePage != page) {
return emptyList()
}
val root = body.getElementById("series-list") ?: parseFailed("Cannot find root")
val root = body.requireElementById("series-list")
return root.children().map { div ->
val a = div.selectFirst("a") ?: parseFailed()
val a = div.selectFirstOrThrow("a")
val href = a.attrAsRelativeUrl("href")
val title = div.selectFirst(".item-title")?.text() ?: parseFailed("Title not found")
val title = div.selectFirstOrThrow(".item-title").text()
Manga(
id = generateUid(href),
title = title,

View File

@@ -0,0 +1,248 @@
package org.koitharu.kotatsu.parsers.site
import androidx.collection.ArrayMap
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
import org.json.JSONArray
import org.jsoup.nodes.Document
import org.jsoup.nodes.Element
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import java.text.SimpleDateFormat
import java.util.*
import kotlin.collections.HashSet
@MangaSourceParser("BLOGTRUYEN", "BlogTruyen", "vi")
class BlogTruyenParser(override val context: MangaLoaderContext) :
PagedMangaParser(MangaSource.BLOGTRUYEN, pageSize = 20) {
override val configKeyDomain: ConfigKey.Domain
get() = ConfigKey.Domain("blogtruyen.vn", null)
override val sortOrders: Set<SortOrder>
get() = EnumSet.of(SortOrder.UPDATED)
private val mutex = Mutex()
private val dateFormat = SimpleDateFormat("dd/MM/yyyy HH:mm", Locale.US)
private var cacheTags: ArrayMap<String, MangaTag>? = null
override suspend fun getDetails(manga: Manga): Manga {
val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml()
val descriptionElement = doc.selectFirstOrThrow("div.description")
val statusText = descriptionElement
.selectFirst("p:contains(Trạng thái) > span.color-red")
?.text()
val state = when (statusText) {
"Đang tiến hành" -> MangaState.ONGOING
"Đã hoàn thành" -> MangaState.FINISHED
else -> null
}
val rating = doc.selectFirst("span.total-vote")?.attr("ng-init")?.let { text ->
val like = text.substringAfter("TotalLike=")
.substringBefore(';')
.toIntOrNull() ?: return@let RATING_UNKNOWN
val dislike = text.substringAfter("TotalDisLike=")
.toIntOrNull() ?: return@let RATING_UNKNOWN
when {
like == 0 && dislike == 0 -> RATING_UNKNOWN
else -> like.toFloat() / (like + dislike)
}
}
val tagMap = getOrCreateTagMap()
val tags = descriptionElement.select("p > span.category").mapNotNullToSet {
val tagName = it.selectFirst("a")?.text()?.trim() ?: return@mapNotNullToSet null
tagMap[tagName]
}
return manga.copy(
tags = tags,
author = descriptionElement.selectFirst("p:contains(Tác giả) > a")?.text(),
description = doc.selectFirst(".detail .content")?.html(),
chapters = parseChapterList(doc),
largeCoverUrl = doc.selectLast("div.thumbnail > img")?.attrAsAbsoluteUrlOrNull("src"),
state = state,
rating = rating ?: RATING_UNKNOWN,
isNsfw = doc.getElementById("warningCategory") != null
)
}
private fun parseChapterList(doc: Document): List<MangaChapter> {
val chapterList = doc.select("#list-chapters > p")
return chapterList.asReversed().mapChapters { index, element ->
val titleElement = element.selectFirst("span.title > a") ?: return@mapChapters null
val name = titleElement.text()
val relativeUrl = titleElement.attrAsRelativeUrl("href")
val id = relativeUrl.substringAfter('/').substringBefore('/')
val uploadDate = dateFormat.tryParse(element.select("span.publishedDate").text())
MangaChapter(
id = generateUid(id),
name = name,
number = index + 1,
url = relativeUrl,
scanlator = null,
uploadDate = uploadDate,
branch = null,
source = source
)
}
}
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
return when {
!query.isNullOrEmpty() -> {
val searchUrl = "https://${getDomain()}/timkiem/nangcao/1/0/-1/-1?txt=${query.urlEncoded()}&p=$page"
val searchContent = context.httpGet(searchUrl).parseHtml()
.selectFirst("section.list-manga-bycate > div.list")
parseMangaList(searchContent)
}
!tags.isNullOrEmpty() -> {
val tag = tags.oneOrThrowIfMany()!!
val categoryAjax = "https://${getDomain()}/ajax/Category/AjaxLoadMangaByCategory?id=${tag.key}&orderBy=5&p=$page"
val listContent = context.httpGet(categoryAjax).parseHtml().selectFirst("div.list")
parseMangaList(listContent)
}
else -> getNormalList(page)
}
}
private suspend fun getNormalList(page: Int): List<Manga> {
val pageLink = "https://${getDomain()}/page-$page"
val doc = context.httpGet(pageLink).parseHtml()
val listElements = doc.selectFirstOrThrow("section.list-mainpage.listview")
.select("div.bg-white.storyitem")
return listElements.mapNotNull {
val linkTag = it.selectFirst("div.fl-l > a") ?: return@mapNotNull null
val relativeUrl = linkTag.attrAsRelativeUrl("href")
val tagMap = getOrCreateTagMap()
val tags = it.select("footer > div.category > a").mapNotNullToSet { a ->
tagMap[a.text()]
}
Manga(
id = generateUid(relativeUrl),
title = linkTag.attr("title"),
altTitle = null,
description = it.selectFirst("p.al-j.break.line-height-15")?.text(),
url = relativeUrl,
publicUrl = relativeUrl.toAbsoluteUrl(getDomain()),
coverUrl = linkTag.selectLast("img")?.attr("src").orEmpty(),
source = source,
tags = tags,
isNsfw = false,
rating = RATING_UNKNOWN,
author = null,
state = null,
)
}
}
private fun parseMangaList(listElement: Element?): List<Manga> {
listElement ?: return emptyList()
return listElement.select("span.tiptip[data-tiptip]").mapNotNull {
val mangaInfo = listElement.getElementById(it.attr("data-tiptip")) ?: return@mapNotNull null
val a = it.selectFirst("a") ?: return@mapNotNull null
val relativeUrl = a.attrAsRelativeUrl("href")
Manga(
id = generateUid(relativeUrl),
title = a.text(),
altTitle = null,
description = mangaInfo.select("div.al-j.fs-12").text(),
url = relativeUrl,
publicUrl = relativeUrl.toAbsoluteUrl(getDomain()),
coverUrl = mangaInfo.selectFirst("div > img.img")?.absUrl("src").orEmpty(),
isNsfw = false,
rating = RATING_UNKNOWN,
tags = emptySet(),
author = null,
state = null,
source = source,
)
}
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
fun generateImageId(index: Int) = generateUid("${chapter.url}/$index")
val doc = context.httpGet(chapter.url.toAbsoluteUrl(getDomain())).parseHtml()
val pages = ArrayList<MangaPage>()
val referer = chapter.url.toAbsoluteUrl(getDomain())
doc.select("#content > img").forEach { img ->
val url = img.attrAsRelativeUrl("src")
pages.add(
MangaPage(
id = generateImageId(pages.lastIndex),
url = url,
referer = referer,
preview = null,
source = source,
)
)
}
// Some chapters use js script to render images
val script = doc.selectLast("#content > script")
if (script != null && script.data().contains("listImageCaption")) {
val imagesStr = script.data().substringBefore(';').substringAfterLast('=').trim()
val imageArr = JSONArray(imagesStr)
for (i in 0 until imageArr.length()) {
val imageUrl = imageArr.getJSONObject(i).getString("url")
pages.add(
MangaPage(
id = generateImageId(pages.lastIndex),
url = imageUrl,
referer = referer,
preview = null,
source = source
)
)
}
}
return pages
}
override suspend fun getTags(): Set<MangaTag> {
val map = getOrCreateTagMap()
val tags = HashSet<MangaTag>(map.size)
for (entry in map) {
tags.add(entry.value)
}
return tags
}
private suspend fun getOrCreateTagMap(): ArrayMap<String, MangaTag> = mutex.withLock {
cacheTags?.let { return@withLock it }
val doc = context.httpGet("/timkiem/nangcao".toAbsoluteUrl(getDomain())).parseHtml()
val tagItems = doc.select("li[data-id]")
val tagMap = ArrayMap<String, MangaTag>(tagItems.size)
for (tag in tagItems) {
val title = tag.text().trim()
tagMap[tag.text().trim()] = MangaTag(
title = title,
key = tag.attr("data-id"),
source = source
)
}
cacheTags = tagMap
tagMap
}
}

View File

@@ -0,0 +1,104 @@
package org.koitharu.kotatsu.parsers.site
import org.koitharu.kotatsu.parsers.*
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import java.util.*
@MangaSourceParser("CLONEMANGA", "CloneManga", "en")
internal class CloneMangaParser(override val context: MangaLoaderContext) : PagedMangaParser(
MangaSource.CLONEMANGA,
pageSize = 1,
) {
override val sortOrders: Set<SortOrder> = Collections.singleton(
SortOrder.POPULARITY,
)
override val configKeyDomain = ConfigKey.Domain("manga.clone-army.org", null)
override fun getFaviconUrl(): String {
return "https://pbs.twimg.com/profile_images/458758466346029056/Ys93EANp_400x400.png"
}
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
if (query != null || page > 1) {
return emptyList()
}
val link = "https://${getDomain()}/viewer_landing.php"
val doc = context.httpGet(link).parseHtml()
val mangas = doc.getElementsByClass("comicPreviewContainer")
return mangas.mapNotNull { item ->
val attr = item.getElementsByClass("comicPreview").attr("style")
val href = item.selectFirst("a")?.attrAsAbsoluteUrl("href") ?: return@mapNotNull null
val cover = attr.substring(attr.indexOf("site/themes"), attr.indexOf(")"))
Manga(
id = generateUid(href),
title = item.selectFirst("h3")?.text() ?: return@mapNotNull null,
coverUrl = "https://${getDomain()}/$cover",
altTitle = null,
author = "Dan Kim",
rating = RATING_UNKNOWN,
url = href,
isNsfw = false,
tags = emptySet(),
state = null,
publicUrl = href.toAbsoluteUrl(getDomain()),
source = source,
)
}
}
override suspend fun getDetails(manga: Manga): Manga {
val doc = context.httpGet(manga.publicUrl).parseHtml()
val series = doc.location()
val numChapters = Regex(
pattern = "&page=(.*)&lang=",
).findAll(
input = doc.getElementsByTag("script")[3].toString(),
)
.elementAt(3).destructured.component1()
.toInt()
val chapters = ArrayList<MangaChapter>()
for (i in 0..numChapters) {
val chapter = MangaChapter(
id = generateUid("$series&page=$i"),
name = "Chapter ${i + 1}",
number = i + 1,
url = "$series&page=$i",
scanlator = null,
branch = null,
uploadDate = 0L,
source = MangaSource.DUMMY,
)
chapters.add(chapter)
}
return manga.copy(chapters = chapters)
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val doc = context.httpGet(chapter.url.toAbsoluteUrl(getDomain())).parseHtml()
val imgUrl = doc.getElementsByClass("subsectionContainer")[0]
.selectFirst("img")
?.attrAsAbsoluteUrlOrNull("src") ?: doc.parseFailed("Something broken")
return listOf(
MangaPage(
id = generateUid(imgUrl),
url = imgUrl,
referer = imgUrl,
preview = null,
source = MangaSource.DUMMY,
),
)
}
override suspend fun getTags(): Set<MangaTag> {
return emptySet()
}
}

View File

@@ -1,8 +1,8 @@
package org.koitharu.kotatsu.parsers.site
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.exception.ParseException
import org.koitharu.kotatsu.parsers.model.*
@@ -13,7 +13,7 @@ import org.koitharu.kotatsu.parsers.util.json.mapJSONToSet
import java.util.*
@MangaSourceParser("DESUME", "Desu.me", "ru")
internal class DesuMeParser(override val context: MangaLoaderContext) : MangaParser(MangaSource.DESUME) {
internal class DesuMeParser(override val context: MangaLoaderContext) : PagedMangaParser(MangaSource.DESUME, 20) {
override val configKeyDomain = ConfigKey.Domain("desu.me", null)
@@ -24,13 +24,13 @@ internal class DesuMeParser(override val context: MangaLoaderContext) : MangaPar
SortOrder.ALPHABETICAL,
)
override suspend fun getList(
offset: Int,
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
if (query != null && offset != 0) {
if (query != null && page != searchPaginator.firstPage) {
return emptyList()
}
val domain = getDomain()
@@ -40,7 +40,7 @@ internal class DesuMeParser(override val context: MangaLoaderContext) : MangaPar
append("/manga/api/?limit=20&order=")
append(getSortKey(sortOrder))
append("&page=")
append((offset / 20) + 1)
append(page)
if (!tags.isNullOrEmpty()) {
append("&genres=")
appendAll(tags, ",") { it.key }
@@ -50,7 +50,8 @@ internal class DesuMeParser(override val context: MangaLoaderContext) : MangaPar
append(query)
}
}
val json = context.httpGet(url).parseJson().getJSONArray("response") ?: parseFailed("Invalid response")
val json = context.httpGet(url).parseJson().getJSONArray("response")
?: throw ParseException("Invalid response", url)
val total = json.length()
val list = ArrayList<Manga>(total)
for (i in 0 until total) {
@@ -83,7 +84,7 @@ internal class DesuMeParser(override val context: MangaLoaderContext) : MangaPar
override suspend fun getDetails(manga: Manga): Manga {
val url = manga.url.toAbsoluteUrl(getDomain())
val json = context.httpGet(url).parseJson().getJSONObject("response")
?: throw ParseException("Invalid response")
?: throw ParseException("Invalid response", url)
val baseChapterUrl = manga.url + "/chapter/"
val chaptersList = json.getJSONObject("chapters").getJSONArray("list")
val totalChapters = chaptersList.length()
@@ -119,7 +120,7 @@ internal class DesuMeParser(override val context: MangaLoaderContext) : MangaPar
val fullUrl = chapter.url.toAbsoluteUrl(getDomain())
val json = context.httpGet(fullUrl)
.parseJson()
.getJSONObject("response") ?: throw ParseException("Invalid response")
.getJSONObject("response") ?: throw ParseException("Invalid response", fullUrl)
return json.getJSONObject("pages").getJSONArray("list").mapJSON { jo ->
MangaPage(
id = generateUid(jo.getLong("id")),
@@ -133,17 +134,17 @@ internal class DesuMeParser(override val context: MangaLoaderContext) : MangaPar
override suspend fun getTags(): Set<MangaTag> {
val doc = context.httpGet("https://${getDomain()}/manga/").parseHtml()
val root = doc.body().getElementById("animeFilter")
?.selectFirst(".catalog-genres") ?: throw ParseException("Root not found")
val root = doc.body().requireElementById("animeFilter")
.selectFirstOrThrow(".catalog-genres")
return root.select("li").mapToSet {
val input = it.selectFirst("input") ?: parseFailed()
val input = it.selectFirstOrThrow("input")
MangaTag(
source = source,
key = input.attr("data-genre-slug").ifEmpty {
parseFailed("data-genre-slug is empty")
it.parseFailed("data-genre-slug is empty")
},
title = input.attr("data-genre-name").toTitleCase().ifEmpty {
parseFailed("data-genre-name is empty")
it.parseFailed("data-genre-name is empty")
},
)
}

View File

@@ -2,12 +2,11 @@ package org.koitharu.kotatsu.parsers.site
import org.jsoup.nodes.Element
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.MangaParserAuthProvider
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.exception.AuthRequiredException
import org.koitharu.kotatsu.parsers.exception.ParseException
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import java.util.*
@@ -19,7 +18,7 @@ private const val DOMAIN_AUTHORIZED = "exhentai.org"
@MangaSourceParser("EXHENTAI", "ExHentai")
internal class ExHentaiParser(
override val context: MangaLoaderContext,
) : MangaParser(MangaSource.EXHENTAI), MangaParserAuthProvider {
) : PagedMangaParser(MangaSource.EXHENTAI, pageSize = 25), MangaParserAuthProvider {
override val sortOrders: Set<SortOrder> = Collections.singleton(
SortOrder.NEWEST,
@@ -57,13 +56,12 @@ internal class ExHentaiParser(
context.cookieJar.insertCookies(DOMAIN_UNAUTHORIZED, "nw=1", "sl=dm_2")
}
override suspend fun getList(
offset: Int,
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val page = (offset / 25f).toIntUp()
var search = query?.urlEncoded().orEmpty()
val url = buildString {
append("https://")
@@ -95,19 +93,19 @@ internal class ExHentaiParser(
val root = body.selectFirst("table.itg")
?.selectFirst("tbody")
?: if (updateDm) {
parseFailed("Cannot find root")
body.parseFailed("Cannot find root")
} else {
updateDm = true
return getList(offset, query, tags, sortOrder)
return getListPage(page, query, tags, sortOrder)
}
updateDm = false
return root.children().mapNotNull { tr ->
if (tr.childrenSize() != 2) return@mapNotNull null
val (td1, td2) = tr.children()
val glink = td2.selectFirst("div.glink") ?: parseFailed("glink not found")
val a = glink.parents().select("a").first() ?: parseFailed("link not found")
val glink = td2.selectFirstOrThrow("div.glink")
val a = glink.parents().select("a").first() ?: glink.parseFailed("link not found")
val href = a.attrAsRelativeUrl("href")
val tagsDiv = glink.nextElementSibling() ?: parseFailed("tags div not found")
val tagsDiv = glink.nextElementSibling() ?: glink.parseFailed("tags div not found")
val mainTag = td2.selectFirst("div.cn")?.let { div ->
MangaTag(
title = div.text().toTitleCase(),
@@ -135,7 +133,7 @@ internal class ExHentaiParser(
override suspend fun getDetails(manga: Manga): Manga {
val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml()
val root = doc.body().selectFirst("div.gm") ?: parseFailed("Cannot find root")
val root = doc.body().selectFirstOrThrow("div.gm")
val cover = root.getElementById("gd1")?.children()?.first()
val title = root.getElementById("gd2")
val taglist = root.getElementById("taglist")
@@ -179,7 +177,7 @@ internal class ExHentaiParser(
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val doc = context.httpGet(chapter.url.toAbsoluteUrl(getDomain())).parseHtml()
val root = doc.body().getElementById("gdt") ?: parseFailed("Root not found")
val root = doc.body().requireElementById("gdt")
return root.select("a").map { a ->
val url = a.attrAsRelativeUrl("href")
MangaPage(
@@ -194,14 +192,12 @@ internal class ExHentaiParser(
override suspend fun getPageUrl(page: MangaPage): String {
val doc = context.httpGet(page.url.toAbsoluteUrl(getDomain())).parseHtml()
return doc.body().getElementById("img")?.absUrl("src")
?: parseFailed("Image not found")
return doc.body().requireElementById("img").attrAsAbsoluteUrl("src")
}
override suspend fun getTags(): Set<MangaTag> {
val doc = context.httpGet("https://${getDomain()}").parseHtml()
val root = doc.body().getElementById("searchbox")?.selectFirst("table")
?: parseFailed("Root not found")
val root = doc.body().requireElementById("searchbox").selectFirstOrThrow("table")
return root.select("div.cs").mapNotNullToSet { div ->
val id = div.id().substringAfterLast('_').toIntOrNull()
?: return@mapNotNullToSet null
@@ -222,7 +218,7 @@ internal class ExHentaiParser(
?: if (doc.getElementById("userlinksguest") != null) {
throw AuthRequiredException(source)
} else {
throw ParseException(null)
doc.parseFailed()
}
return username
}

View File

@@ -1,12 +1,15 @@
package org.koitharu.kotatsu.parsers.site
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.async
import kotlinx.coroutines.awaitAll
import kotlinx.coroutines.coroutineScope
import org.json.JSONObject
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.exception.ParseException
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import org.koitharu.kotatsu.parsers.util.json.*
@@ -14,7 +17,9 @@ import java.text.SimpleDateFormat
import java.util.*
private const val PAGE_SIZE = 20
private const val CHAPTERS_PAGE_SIZE = 60
private const val CHAPTERS_FIRST_PAGE_SIZE = 120
private const val CHAPTERS_MAX_PAGE_SIZE = 500
private const val CHAPTERS_PARALLELISM = 3
private const val CONTENT_RATING =
"contentRating[]=safe&contentRating[]=suggestive&contentRating[]=erotica&contentRating[]=pornographic"
private const val LOCALE_FALLBACK = "en"
@@ -62,6 +67,7 @@ internal class MangaDexParser(override val context: MangaLoaderContext) : MangaP
when (sortOrder) {
SortOrder.UPDATED,
-> "[latestUploadedChapter]=desc"
SortOrder.ALPHABETICAL -> "[title]=asc"
SortOrder.NEWEST -> "[createdAt]=desc"
SortOrder.POPULARITY -> "[followedCount]=desc"
@@ -118,12 +124,13 @@ internal class MangaDexParser(override val context: MangaLoaderContext) : MangaP
override suspend fun getDetails(manga: Manga): Manga = coroutineScope {
val domain = getDomain()
val mangaId = manga.url.removePrefix("/")
val attrsDeferred = async {
context.httpGet(
"https://api.$domain/manga/${manga.url}?includes[]=artist&includes[]=author&includes[]=cover_art",
"https://api.$domain/manga/${mangaId}?includes[]=artist&includes[]=author&includes[]=cover_art",
).parseJson().getJSONObject("data").getJSONObject("attributes")
}
val feedDeferred = async { loadChapters(manga) }
val feedDeferred = async { loadChapters(mangaId) }
val mangaAttrs = attrsDeferred.await()
val feed = feedDeferred.await()
// 2022-01-02T00:27:11+00:00
@@ -134,11 +141,11 @@ internal class MangaDexParser(override val context: MangaLoaderContext) : MangaP
manga.copy(
description = mangaAttrs.getJSONObject("description").selectByLocale()
?: manga.description,
chapters = feed.mapNotNull { jo ->
chapters = feed.mapChapters { _, jo ->
val id = jo.getString("id")
val attrs = jo.getJSONObject("attributes")
if (!attrs.isNull("externalUrl")) {
return@mapNotNull null
return@mapChapters null
}
val locale = attrs.getStringOrNull("translatedLanguage")?.let { Locale.forLanguageTag(it) }
val relations = jo.getJSONArray("relationships").associateByKey("type")
@@ -201,30 +208,62 @@ internal class MangaDexParser(override val context: MangaLoaderContext) : MangaP
return getStringOrNull(LOCALE_FALLBACK) ?: values().nextOrNull() as? String
}
private suspend fun loadChapters(manga: Manga): List<JSONObject> {
val domain = getDomain()
val result = ArrayList<JSONObject>()
while (true) {
val url = buildString {
append("https://api.")
append(domain)
append("/manga/")
append(manga.url)
append("/feed")
append("?limit=")
append(CHAPTERS_PAGE_SIZE)
append("&includes[]=scanlation_group&order[volume]=asc&order[chapter]=asc&offset=")
append(result.size)
append('&')
append(CONTENT_RATING)
}
val json = context.httpGet(url).parseJson()
val data = json.optJSONArray("data")
if (data.isNullOrEmpty()) {
break
}
data.mapJSONTo(result) { it }
private suspend fun loadChapters(mangaId: String): List<JSONObject> {
val firstPage = loadChapters(mangaId, offset = 0, limit = CHAPTERS_FIRST_PAGE_SIZE)
if (firstPage.size >= firstPage.total) {
return firstPage.data
}
val tail = coroutineScope {
val leftCount = firstPage.total - firstPage.size
val pages = (leftCount / CHAPTERS_MAX_PAGE_SIZE.toFloat()).toIntUp()
val dispatcher = Dispatchers.Default.limitedParallelism(CHAPTERS_PARALLELISM)
List(pages) { page ->
val offset = page * CHAPTERS_MAX_PAGE_SIZE + firstPage.size
async(dispatcher) {
loadChapters(mangaId, offset, CHAPTERS_MAX_PAGE_SIZE)
}
}.awaitAll()
}
val result = ArrayList<JSONObject>(firstPage.total)
result += firstPage.data
tail.flatMapTo(result) { it.data }
return result
}
private suspend fun loadChapters(mangaId: String, offset: Int, limit: Int): Chapters {
val url = buildString {
append("https://api.")
append(getDomain())
append("/manga/")
append(mangaId)
append("/feed")
append("?limit=")
append(limit)
append("&includes[]=scanlation_group&order[volume]=asc&order[chapter]=asc&offset=")
append(offset)
append('&')
append(CONTENT_RATING)
}
val json = context.httpGet(url).parseJson()
if (json.getString("result") == "ok") {
return Chapters(
data = json.optJSONArray("data")?.toJSONList().orEmpty(),
total = json.getInt("total"),
)
} else {
val error = json.optJSONArray("errors").mapJSON { jo ->
jo.getString("detail")
}.joinToString("\n")
throw ParseException(error, url)
}
}
private class Chapters(
val data: List<JSONObject>,
val total: Int,
) {
val size: Int
get() = data.size
}
}

View File

@@ -1,8 +1,8 @@
package org.koitharu.kotatsu.parsers.site
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
@@ -12,26 +12,28 @@ import java.util.*
private const val DEF_BRANCH_NAME = "Основний переклад"
@MangaSourceParser("MANGAINUA", "MANGA/in/UA", "uk")
class MangaInUaParser(override val context: MangaLoaderContext) : MangaParser(MangaSource.MANGAINUA) {
class MangaInUaParser(override val context: MangaLoaderContext) : PagedMangaParser(
source = MangaSource.MANGAINUA,
pageSize = 24,
searchPageSize = 10,
) {
override val sortOrders: Set<SortOrder>
get() = Collections.singleton(SortOrder.UPDATED)
override val configKeyDomain: ConfigKey.Domain = ConfigKey.Domain("manga.in.ua", null)
override suspend fun getList(
offset: Int,
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val page = (offset / 24f).toIntUp().inc()
val searchPage = (offset / 10f).toIntUp().inc()
val url = when {
!query.isNullOrEmpty() -> (
"/index.php?do=search" +
"&subaction=search" +
"&search_start=$searchPage" +
"&search_start=$page" +
"&full_search=1" +
"&story=$query" +
"&titleonly=3"
@@ -43,7 +45,7 @@ class MangaInUaParser(override val context: MangaLoaderContext) : MangaParser(Ma
else -> "/mangas/page/$page".toAbsoluteUrl(getDomain())
}
val doc = context.httpGet(url).parseHtml()
val container = doc.body().getElementById("dle-content") ?: parseFailed("Container not found")
val container = doc.body().requireElementById("dle-content")
val items = container.select("div.col-6")
return items.mapNotNull { item ->
val href = item.selectFirst("a")?.attrAsRelativeUrl("href") ?: return@mapNotNull null
@@ -79,12 +81,9 @@ class MangaInUaParser(override val context: MangaLoaderContext) : MangaParser(Ma
override suspend fun getDetails(manga: Manga): Manga {
val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml()
val root =
doc.body().getElementById("dle-content") ?: parseFailed("Cannot find root")
val root = doc.body().requireElementById("dle-content")
val dateFormat = SimpleDateFormat("dd.MM.yyyy", Locale.US)
val chapterNodes = root.selectFirstOrThrow(".linkstocomics")
.select(".ltcitems")
.filterNot { x -> x.selectFirst(".foruserreadedbar") != null }
val chapterNodes = root.selectFirstOrThrow(".linkstocomics").select(".ltcitems")
var prevChapterName: String? = null
var i = 0
return manga.copy(
@@ -123,10 +122,9 @@ class MangaInUaParser(override val context: MangaLoaderContext) : MangaParser(Ma
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(getDomain())
val doc = context.httpGet(fullUrl).parseHtml()
val root =
doc.body().getElementById("comics")?.selectFirst("ul.xfieldimagegallery") ?: parseFailed("Root not found")
val root = doc.body().requireElementById("comics").selectFirstOrThrow("ul.xfieldimagegallery")
return root.select("li").map { ul ->
val img = ul.selectFirst("img") ?: parseFailed("Page image not found")
val img = ul.selectFirstOrThrow("img")
val url = img.attrAsAbsoluteUrl("data-src")
MangaPage(
id = generateUid(url),
@@ -141,8 +139,7 @@ class MangaInUaParser(override val context: MangaLoaderContext) : MangaParser(Ma
override suspend fun getTags(): Set<MangaTag> {
val domain = getDomain()
val doc = context.httpGet("https://$domain/mangas").parseHtml()
val root =
doc.body().getElementById("menu_1")?.selectFirst("div.menu__wrapper") ?: parseFailed("Cannot find root")
val root = doc.body().requireElementById("menu_1").selectFirstOrThrow("div.menu__wrapper")
return root.select("li").mapNotNullToSet { li ->
val a = li.selectFirst("a") ?: return@mapNotNullToSet null
MangaTag(

View File

@@ -4,12 +4,12 @@ import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.exception.ParseException
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import java.text.SimpleDateFormat
import java.util.*
@Deprecated("Permanently shutdown")
@MangaSourceParser("MANGAOWL", "MangaOwl", "en")
internal class MangaOwlParser(override val context: MangaLoaderContext) : MangaParser(MangaSource.MANGAOWL) {
@@ -52,7 +52,7 @@ internal class MangaOwlParser(override val context: MangaLoaderContext) : MangaP
}
}
val doc = context.httpGet(link).parseHtml()
val slides = doc.body().select("ul.slides") ?: parseFailed("An error occurred while parsing")
val slides = doc.body().selectOrThrow("ul.slides")
val items = slides.select("div.col-md-2")
return items.mapNotNull { item ->
val href = item.selectFirst("h6 a")?.attrAsRelativeUrlOrNull("href") ?: return@mapNotNull null
@@ -80,12 +80,12 @@ internal class MangaOwlParser(override val context: MangaLoaderContext) : MangaP
override suspend fun getDetails(manga: Manga): Manga {
val doc = context.httpGet(manga.publicUrl).parseHtml()
val info = doc.body().selectFirst("div.single_detail") ?: parseFailed("An error occurred while parsing")
val table = doc.body().selectFirst("div.single-grid-right") ?: parseFailed("An error occurred while parsing")
val info = doc.body().selectFirstOrThrow("div.single_detail")
val table = doc.body().selectFirstOrThrow("div.single-grid-right")
val dateFormat = SimpleDateFormat("MM/dd/yyyy", Locale.US)
val trRegex = "window\\['tr'] = '([^']*)';".toRegex(RegexOption.IGNORE_CASE)
val trElement =
doc.getElementsByTag("script").find { trRegex.find(it.data()) != null } ?: parseFailed("Oops, tr not found")
val trElement = doc.getElementsByTag("script").find { trRegex.find(it.data()) != null }
?: doc.parseFailed("Oops, tr not found")
val tr = trRegex.find(trElement.data())!!.groups[1]!!.value
val s = context.encodeBase64(getDomain().toByteArray())
var isNsfw = manga.isNsfw
@@ -115,7 +115,7 @@ internal class MangaOwlParser(override val context: MangaLoaderContext) : MangaP
.asReversed().mapChapters { i, li ->
val a = li.select("a")
val href = a.attr("data-href").ifEmpty {
parseFailed("Link is missing")
li.parseFailed("Link is missing")
}
MangaChapter(
id = generateUid(href),
@@ -134,9 +134,9 @@ internal class MangaOwlParser(override val context: MangaLoaderContext) : MangaP
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(getDomain())
val doc = context.httpGet(fullUrl).parseHtml()
val root = doc.body().select("div.item img.owl-lazy") ?: throw ParseException("Root not found")
val root = doc.body().selectOrThrow("div.item img.owl-lazy")
return root.map { div ->
val url = div?.attrAsRelativeUrlOrNull("data-src") ?: parseFailed("Page image not found")
val url = div?.attrAsRelativeUrlOrNull("data-src") ?: doc.parseFailed("Page image not found")
MangaPage(
id = generateUid(url),
url = url,
@@ -158,7 +158,7 @@ internal class MangaOwlParser(override val context: MangaLoaderContext) : MangaP
val doc = context.httpGet("https://${getDomain()}/").parseHtml()
val root = doc.body().select("ul.dropdown-menu.multi-column.columns-3").select("li")
return root.mapToSet { p ->
val a = p.selectFirst("a") ?: parseFailed("a is null")
val a = p.selectFirstOrThrow("a")
MangaTag(
title = a.text().toTitleCase(),
key = a.attr("href"),

View File

@@ -4,7 +4,6 @@ import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.exception.ParseException
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import java.text.DateFormat
@@ -55,8 +54,7 @@ internal class MangaTownParser(override val context: MangaLoaderContext) : Manga
}
}
val doc = context.httpGet(url).parseHtml()
val root = doc.body().selectFirst("ul.manga_pic_list")
?: throw ParseException("Root not found")
val root = doc.body().selectFirstOrThrow("ul.manga_pic_list")
return root.select("li").mapNotNull { li ->
val a = li.selectFirst("a.manga_cover")
val href = a?.attrAsRelativeUrlOrNull("href")
@@ -96,8 +94,8 @@ internal class MangaTownParser(override val context: MangaLoaderContext) : Manga
override suspend fun getDetails(manga: Manga): Manga {
val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml()
val root = doc.body().selectFirst("section.main")
?.selectFirst("div.article_content") ?: throw ParseException("Cannot find root")
val root = doc.body().selectFirstOrThrow("section.main")
.selectFirstOrThrow("div.article_content")
val info = root.selectFirst("div.detail_info")?.selectFirst("ul")
val chaptersList = root.selectFirst("div.chapter_content")
?.selectFirst("ul.chapter_list")?.select("li")?.asReversed()
@@ -139,9 +137,8 @@ internal class MangaTownParser(override val context: MangaLoaderContext) : Manga
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(getDomain())
val doc = context.httpGet(fullUrl).parseHtml()
val root = doc.body().selectFirst("div.page_select")
?: throw ParseException("Cannot find root")
return root.selectFirst("select")?.select("option")?.mapNotNull {
val root = doc.body().selectFirstOrThrow("div.page_select")
return root.selectFirstOrThrow("select").selectOrThrow("option").mapNotNull {
val href = it.attrAsRelativeUrlOrNull("value")
if (href == null || href.endsWith("featured.html")) {
return@mapNotNull null
@@ -153,12 +150,12 @@ internal class MangaTownParser(override val context: MangaLoaderContext) : Manga
referer = fullUrl,
source = MangaSource.MANGATOWN,
)
} ?: parseFailed("Pages list not found")
}
}
override suspend fun getPageUrl(page: MangaPage): String {
val doc = context.httpGet(page.url.toAbsoluteUrl(getDomain())).parseHtml()
return doc.getElementById("image")?.absUrl("src") ?: parseFailed("Image not found")
return doc.requireElementById("image").attrAsAbsoluteUrl("src")
}
override suspend fun getTags(): Set<MangaTag> {
@@ -166,7 +163,7 @@ internal class MangaTownParser(override val context: MangaLoaderContext) : Manga
val root = doc.body().selectFirst("aside.right")
?.getElementsContainingOwnText("Genres")
?.first()
?.nextElementSibling() ?: parseFailed("Root not found")
?.nextElementSibling() ?: doc.parseFailed("Root not found")
return root.select("li").mapNotNullToSet { li ->
val a = li.selectFirst("a") ?: return@mapNotNullToSet null
val key = a.attr("href").parseTagKey()

View File

@@ -6,8 +6,8 @@ import kotlinx.coroutines.awaitAll
import kotlinx.coroutines.coroutineScope
import org.jsoup.nodes.Element
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
@@ -15,7 +15,7 @@ import java.text.SimpleDateFormat
import java.util.*
@MangaSourceParser("NHENTAI", "N-Hentai")
class NHentaiParser(override val context: MangaLoaderContext) : MangaParser(MangaSource.NHENTAI) {
class NHentaiParser(override val context: MangaLoaderContext) : PagedMangaParser(MangaSource.NHENTAI, pageSize = 25) {
override val configKeyDomain: ConfigKey.Domain
get() = ConfigKey.Domain("nhentai.net", null)
@@ -23,17 +23,16 @@ class NHentaiParser(override val context: MangaLoaderContext) : MangaParser(Mang
override val sortOrders: Set<SortOrder>
get() = EnumSet.of(SortOrder.NEWEST, SortOrder.POPULARITY)
override suspend fun getList(
offset: Int,
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
if (query.isNullOrEmpty() && tags != null && tags.size > 1) {
return getList(offset, buildQuery(tags), emptySet(), sortOrder)
return getListPage(page, buildQuery(tags), emptySet(), sortOrder)
}
val domain = getDomain()
val page = (offset / 25) + 1
val url = buildString {
append("https://")
append(domain)
@@ -67,8 +66,8 @@ class NHentaiParser(override val context: MangaLoaderContext) : MangaParser(Mang
}
}
}
val root = context.httpGet(url).parseHtml().body().getElementById("content")
?.selectLast("div.index-container") ?: parseFailed("Root not found")
val root = context.httpGet(url).parseHtml().body().requireElementById("content")
.selectLastOrThrow("div.index-container")
val regexBrackets = Regex("\\[[^]]+]|\\([^)]+\\)")
val regexSpaces = Regex("\\s+")
return root.select(".gallery").map { div ->
@@ -101,7 +100,7 @@ class NHentaiParser(override val context: MangaLoaderContext) : MangaParser(Mang
override suspend fun getDetails(manga: Manga): Manga {
val root = context.httpGet(
url = manga.url.toAbsoluteUrl(getDomain())
url = manga.url.toAbsoluteUrl(getDomain()),
).parseHtml().body().requireElementById("bigcontainer")
val img = root.requireElementById("cover").selectFirstOrThrow("img")
val tagContainers = root.requireElementById("tags").select(".tag-container")
@@ -126,12 +125,12 @@ class NHentaiParser(override val context: MangaLoaderContext) : MangaParser(Mang
uploadDate = dateFormat.tryParse(
tagContainers.find { x -> x.ownText() == "Uploaded:" }
?.selectFirst("time")
?.attr("datetime")
?.attr("datetime"),
),
branch = null,
source = source,
)
)
),
),
)
}

View File

@@ -0,0 +1,221 @@
package org.koitharu.kotatsu.parsers.site
import androidx.collection.ArrayMap
import androidx.collection.ArraySet
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
import org.koitharu.kotatsu.parsers.*
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.exception.NotFoundException
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import java.text.SimpleDateFormat
import java.util.*
@MangaSourceParser("NETTRUYEN", "NetTruyen", "vi")
class NetTruyenParser(override val context: MangaLoaderContext) :
PagedMangaParser(MangaSource.NETTRUYEN, pageSize = 36) {
override val configKeyDomain: ConfigKey.Domain
get() = ConfigKey.Domain("www.nettruyenme.com", null)
override val sortOrders: Set<SortOrder>
get() = EnumSet.of(SortOrder.UPDATED, SortOrder.POPULARITY, SortOrder.NEWEST, SortOrder.RATING)
private val mutex = Mutex()
private val dateFormat = SimpleDateFormat("dd/MM/yy", Locale.US)
private var tagCache: ArrayMap<String, MangaTag>? = null
override suspend fun getDetails(manga: Manga): Manga {
val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml()
val rating = doc.selectFirst("span[itemprop=ratingValue]")
?.ownText()
?.toFloatOrNull() ?: 0f
val chapterElements = doc.getElementById("nt_listchapter")?.select("ul > li") ?: doc.parseFailed()
val chapters = chapterElements.asReversed().mapChapters { index, element ->
val a = element.selectFirst("div.chapter > a") ?: return@mapChapters null
val relativeUrl = a.attrAsRelativeUrlOrNull("href") ?: return@mapChapters null
val timeText = element.selectFirst("div.col-xs-4.text-center.no-wrap.small")?.text()
MangaChapter(
id = generateUid(relativeUrl),
name = a.text(),
number = index + 1,
url = relativeUrl,
scanlator = null,
uploadDate = parseChapterTime(timeText),
branch = null,
source = source,
)
}
return manga.copy(
rating = rating / 5,
chapters = chapters,
description = doc.selectFirst("div.detail-content > p")?.html(),
isNsfw = doc.selectFirst("div.alert.alert-danger > strong:contains(Cảnh báo độ tuổi)") != null,
)
}
// 20 giây trước
// 52 phút trước
// 6 giờ trước
// 2 ngày trước
// 19:09 30/07
// 23/12/21
private fun parseChapterTime(timeText: String?): Long {
if (timeText.isNullOrEmpty()) {
return 0L
}
val timeWords = arrayOf("giây", "phút", "giờ", "ngày")
val calendar = Calendar.getInstance()
val timeArr = timeText.split(' ')
if (WordSet(*timeWords).anyWordIn(timeText)) {
val timeSuffix = timeArr.getOrNull(1)
val timeDiff = timeArr.getOrNull(0)?.toIntOrNull() ?: return 0L
when (timeSuffix) {
timeWords[0] -> calendar.add(Calendar.SECOND, -timeDiff)
timeWords[1] -> calendar.add(Calendar.MINUTE, -timeDiff)
timeWords[2] -> calendar.add(Calendar.HOUR, -timeDiff)
timeWords[3] -> calendar.add(Calendar.DATE, -timeDiff)
else -> return 0L
}
} else {
val relativeDate = timeArr.lastOrNull() ?: return 0L
val dateString = when (relativeDate.split('/').size) {
2 -> {
val currentYear = calendar.get(Calendar.YEAR).toString().takeLast(2)
"$relativeDate/$currentYear"
}
3 -> relativeDate
else -> return 0L
}
calendar.timeInMillis = dateFormat.tryParse(dateString)
}
return calendar.time.time
}
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val isSearching = !query.isNullOrEmpty()
val url = buildString {
append("https://")
append(getDomain())
if (isSearching) {
append("/tim-truyen?keyword=")
append(query!!.urlEncoded())
append("&page=")
append(page)
} else {
val tagQuery = tags.orEmpty().joinToString(",") { it.key }
append("/tim-truyen-nang-cao?genres=$tagQuery")
append("&notgenres=&gender=-1&status=-1&minchapter=1&sort=${getSortOrderKey(sortOrder)}")
append("&page=$page")
}
}
val response = if (isSearching) {
val result = runCatching { context.httpGet(url) }
val exception = result.exceptionOrNull()
if (exception is NotFoundException) {
return emptyList()
}
result.getOrThrow()
} else {
context.httpGet(url)
}
val itemsElements = response.parseHtml()
.select("div.ModuleContent > div.items")
.select("div.item")
return itemsElements.mapNotNull { item ->
val tooltipElement = item.selectFirst("div.box_tootip") ?: return@mapNotNull null
val absUrl = item.selectFirst("div.image > a")?.attrAsAbsoluteUrlOrNull("href") ?: return@mapNotNull null
val slug = absUrl.substringAfterLast('/')
val mangaState = when (tooltipElement.selectFirst("div.message_main > p:contains(Tình trạng)")?.ownText()) {
"Đang tiến hành" -> MangaState.ONGOING
"Hoàn thành" -> MangaState.FINISHED
else -> null
}
val tagMap = getOrCreateTagMap()
val tagsElement = tooltipElement.selectFirst("div.message_main > p:contains(Thể loại)")?.ownText().orEmpty()
val mangaTags = tagsElement.split(',').mapNotNullToSet { tagMap[it.trim()] }
Manga(
id = generateUid(slug),
title = tooltipElement.selectFirst("div.title")?.text().orEmpty(),
altTitle = null,
url = absUrl.toRelativeUrl(getDomain()),
publicUrl = absUrl,
rating = RATING_UNKNOWN,
isNsfw = false,
coverUrl = item.selectFirst("div.image a img")?.absUrl("data-original").orEmpty(),
largeCoverUrl = null,
tags = mangaTags,
state = mangaState,
author = tooltipElement.selectFirst("div.message_main > p:contains(Tác giả)")?.ownText(),
description = tooltipElement.selectFirst("div.box_text")?.text(),
chapters = null,
source = source,
)
}
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val pageElements = context.httpGet(chapter.url.toAbsoluteUrl(getDomain())).parseHtml()
.select("div.reading-detail.box_doc > div img")
return pageElements.map { element ->
val url = element.attrAsAbsoluteUrl("data-original")
MangaPage(
id = generateUid(url),
url = url,
referer = getDomain(),
preview = null,
source = source,
)
}
}
override suspend fun getTags(): Set<MangaTag> {
val map = getOrCreateTagMap()
val tagSet = ArraySet<MangaTag>(map.size)
for (entry in map) {
tagSet.add(entry.value)
}
return tagSet
}
private suspend fun getOrCreateTagMap(): ArrayMap<String, MangaTag> = mutex.withLock {
tagCache?.let { return@withLock it }
val doc = context.httpGet("/tim-truyen-nang-cao".toAbsoluteUrl(getDomain())).parseHtml()
val tagItems = doc.select("div.genre-item")
val result = ArrayMap<String, MangaTag>(tagItems.size)
for (item in tagItems) {
val title = item.text().trim()
val key = item.select("span[data-id]").attr("data-id")
result[title] = MangaTag(title = title, key = key, source = source)
}
tagCache = result
result
}
private fun getSortOrderKey(sortOrder: SortOrder) = when (sortOrder) {
SortOrder.UPDATED -> 0
SortOrder.POPULARITY -> 10
SortOrder.NEWEST -> 15
SortOrder.RATING -> 20
else -> throw IllegalArgumentException("Sort order ${sortOrder.name} not supported")
}
}

View File

@@ -54,7 +54,7 @@ class NicovideoSeigaParser(override val context: MangaLoaderContext) :
else -> "https://$domain/manga/list?page=$page&sort=${getSortKey(sortOrder)}"
}
val doc = context.httpGet(url).parseHtml()
val comicList = doc.body().select("#comic_list > ul > li") ?: parseFailed("Container not found")
val comicList = doc.body().select("#comic_list > ul > li") ?: doc.parseFailed("Container not found")
val items = comicList.select("div > .description > div > div")
return items.mapNotNull { item ->
val href =
@@ -89,7 +89,7 @@ class NicovideoSeigaParser(override val context: MangaLoaderContext) :
override suspend fun getDetails(manga: Manga): Manga {
val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain("seiga"))).parseHtml()
val contents = doc.body().selectFirst("#contents") ?: parseFailed("Cannot find root")
val contents = doc.body().selectFirstOrThrow("#contents")
val statusText = contents
.select("div.mg_work_detail > div > div:nth-child(2) > div.tip.content_status.status_series > span")
.text()
@@ -105,7 +105,7 @@ class NicovideoSeigaParser(override val context: MangaLoaderContext) :
isNsfw = contents.select(".icon_adult").isNotEmpty(),
chapters = contents.select("#episode_list > ul > li").mapChapters { i, li ->
val href = li.selectFirst("div > div.description > div.title > a")
?.attrAsRelativeUrl("href") ?: parseFailed()
?.attrAsRelativeUrl("href") ?: li.parseFailed()
MangaChapter(
id = generateUid(href),
name = li.select("div > div.description > div.title > a").text(),
@@ -140,9 +140,9 @@ class NicovideoSeigaParser(override val context: MangaLoaderContext) :
override suspend fun getTags(): Set<MangaTag> {
val doc = context.httpGet("https://${getDomain("seiga")}/manga/list").parseHtml()
val root = doc.body().select("#mg_category_list > ul > li") ?: parseFailed("Cannot find tags")
val root = doc.body().selectOrThrow("#mg_category_list > ul > li")
return root.mapToSet { li ->
val a = li.selectFirst("a") ?: parseFailed("a is null")
val a = li.selectFirstOrThrow("a")
MangaTag(
title = a.text(),
key = a.attrAsRelativeUrlOrNull("href").orEmpty(),
@@ -157,7 +157,7 @@ class NicovideoSeigaParser(override val context: MangaLoaderContext) :
val root = doc.body().select(".search_result__item")
return root.mapNotNull { item ->
val href = item.selectFirst(".search_result__item__thumbnail > a")
?.attrAsRelativeUrl("href") ?: parseFailed()
?.attrAsRelativeUrl("href") ?: doc.parseFailed()
Manga(
id = generateUid(href),
url = href,

View File

@@ -3,22 +3,19 @@ package org.koitharu.kotatsu.parsers.site
import okhttp3.Headers
import okhttp3.HttpUrl.Companion.toHttpUrl
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.exception.ParseException
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import java.text.SimpleDateFormat
import java.util.*
private const val PAGE_SIZE = 26
internal abstract class NineMangaParser(
final override val context: MangaLoaderContext,
source: MangaSource,
defaultDomain: String,
) : MangaParser(source) {
) : PagedMangaParser(source, pageSize = 26) {
override val configKeyDomain = ConfigKey.Domain(defaultDomain, null)
@@ -34,13 +31,12 @@ internal abstract class NineMangaParser(
SortOrder.POPULARITY,
)
override suspend fun getList(
offset: Int,
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val page = (offset / PAGE_SIZE.toFloat()).toIntUp() + 1
val url = buildString {
append("https://")
append(getDomain())
@@ -69,11 +65,11 @@ internal abstract class NineMangaParser(
}
val doc = context.httpGet(url, headers).parseHtml()
val root = doc.body().selectFirst("ul.direlist")
?: throw ParseException("Cannot find root")
?: doc.parseFailed("Cannot find root")
val baseHost = root.baseUri().toHttpUrl().host
return root.select("li").map { node ->
val href = node.selectFirst("a")?.absUrl("href")
?: parseFailed("Link not found")
?: node.parseFailed("Link not found")
val relUrl = href.toRelativeUrl(baseHost)
val dd = node.selectFirst("dd")
Manga(
@@ -99,10 +95,8 @@ internal abstract class NineMangaParser(
manga.url.toAbsoluteUrl(getDomain()) + "?waring=1",
headers,
).parseHtml()
val root = doc.body().selectFirst("div.manga")
?: throw ParseException("Cannot find root")
val infoRoot = root.selectFirst("div.bookintro")
?: throw ParseException("Cannot find info")
val root = doc.body().selectFirstOrThrow("div.manga")
val infoRoot = root.selectFirstOrThrow("div.bookintro")
return manga.copy(
tags = infoRoot.getElementsByAttributeValue("itemprop", "genre").first()
?.select("a")?.mapToSet { a ->
@@ -120,7 +114,7 @@ internal abstract class NineMangaParser(
?.asReversed()?.mapChapters { i, li ->
val a = li.selectFirst("a.chapter_list_a")
val href = a?.attrAsRelativeUrlOrNull("href")
?.replace("%20", " ") ?: parseFailed("Link not found")
?.replace("%20", " ") ?: li.parseFailed("Link not found")
MangaChapter(
id = generateUid(href),
name = a.text(),
@@ -146,14 +140,14 @@ internal abstract class NineMangaParser(
preview = null,
source = source,
)
} ?: throw ParseException("Pages list not found at ${chapter.url}")
} ?: doc.parseFailed("Pages list not found")
}
override suspend fun getPageUrl(page: MangaPage): String {
val doc = context.httpGet(page.url.toAbsoluteUrl(getDomain()), headers).parseHtml()
val root = doc.body()
return root.selectFirst("a.pic_download")?.absUrl("href")
?: throw ParseException("Page image not found")
?: doc.parseFailed("Page image not found")
}
override suspend fun getTags(): Set<MangaTag> {
@@ -168,7 +162,7 @@ internal abstract class NineMangaParser(
key = cateId,
source = source,
)
} ?: parseFailed("Root not found")
} ?: doc.parseFailed("Root not found")
}
private fun parseStatus(status: String) = when {

View File

@@ -6,7 +6,6 @@ import org.koitharu.kotatsu.parsers.MangaParserAuthProvider
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.exception.AuthRequiredException
import org.koitharu.kotatsu.parsers.exception.ParseException
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import java.text.SimpleDateFormat
@@ -59,12 +58,13 @@ internal class NudeMoonParser(
postfix = "&rowstart=$offset",
transform = { it.key.urlEncoded() },
)
else -> "https://$domain/all_manga?${getSortKey(sortOrder)}&rowstart=$offset"
}
val doc = context.httpGet(url).parseHtml()
val root = doc.body().run {
selectFirst("td.main-bg") ?: selectFirst("td.main-body")
} ?: parseFailed("Cannot find root")
} ?: doc.parseFailed("Cannot find root")
return root.select("table.news_pic2").mapNotNull { row ->
val a = row.selectFirst("td.bg_style1")?.selectFirst("a")
?: return@mapNotNull null
@@ -102,7 +102,7 @@ internal class NudeMoonParser(
override suspend fun getDetails(manga: Manga): Manga {
val body = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml().body()
val root = body.selectFirst("table.shoutbox")
?: parseFailed("Cannot find root")
?: body.parseFailed("Cannot find root")
val info = root.select("div.tbl2")
val lastInfo = info.last()
return manga.copy(
@@ -143,7 +143,7 @@ internal class NudeMoonParser(
val script = doc.select("script").firstNotNullOfOrNull {
it.html().takeIf { x -> x.contains(" images = new ") }
} ?: if (isAuthorized) {
parseFailed("Cannot find pages list")
doc.parseFailed("Cannot find pages list")
} else {
throw AuthRequiredException(source)
}
@@ -174,7 +174,7 @@ internal class NudeMoonParser(
val root = doc.body().getElementsContainingOwnText("Поиск манги по тегам")
.firstOrNull()?.parents()?.find { it.tag().normalName() == "tbody" }
?.selectFirst("td.textbox")?.selectFirst("td.small")
?: parseFailed("Tags root not found")
?: doc.parseFailed("Tags root not found")
return root.select("a").mapToSet {
MangaTag(
title = it.text().toTitleCase(),
@@ -197,7 +197,7 @@ internal class NudeMoonParser(
throw if (body.selectFirst("form[name=\"loginform\"]") != null) {
AuthRequiredException(source)
} else {
ParseException("Cannot find username")
body.parseFailed("Cannot find username")
}
}
}

View File

@@ -6,18 +6,16 @@ import org.json.JSONArray
import org.json.JSONException
import org.json.JSONObject
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.MangaParserAuthProvider
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.exception.AuthRequiredException
import org.koitharu.kotatsu.parsers.exception.ContentUnavailableException
import org.koitharu.kotatsu.parsers.exception.ParseException
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import org.koitharu.kotatsu.parsers.util.json.getStringOrNull
import org.koitharu.kotatsu.parsers.util.json.mapJSON
import org.koitharu.kotatsu.parsers.util.json.mapJSONTo
import org.koitharu.kotatsu.parsers.util.json.mapJSONToSet
import org.koitharu.kotatsu.parsers.util.json.*
import java.net.HttpURLConnection
import java.net.URLDecoder
import java.text.DateFormat
@@ -31,7 +29,7 @@ private const val STATUS_FINISHED = 0
@MangaSourceParser("REMANGA", "Remanga", "ru")
internal class RemangaParser(
override val context: MangaLoaderContext,
) : MangaParser(MangaSource.REMANGA), MangaParserAuthProvider {
) : PagedMangaParser(MangaSource.REMANGA, PAGE_SIZE), MangaParserAuthProvider {
override val configKeyDomain = ConfigKey.Domain("remanga.org", null)
override val authUrl: String
@@ -53,8 +51,8 @@ internal class RemangaParser(
private val regexLastUrlPath = Regex("/[^/]+/?$")
override suspend fun getList(
offset: Int,
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
@@ -77,7 +75,7 @@ internal class RemangaParser(
}
urlBuilder
.append("&page=")
.append((offset / PAGE_SIZE) + 1)
.append(page)
.append("&count=")
.append(PAGE_SIZE)
val content = context.httpGet(urlBuilder.toString(), getApiHeaders()).parseJson()
@@ -113,18 +111,18 @@ internal class RemangaParser(
copyCookies()
val domain = getDomain()
val slug = manga.url.find(regexLastUrlPath)
?: throw ParseException("Cannot obtain slug from ${manga.url}")
?: throw ParseException("Cannot obtain slug from ${manga.url}", manga.publicUrl)
val data = context.httpGet(
url = "https://api.$domain/api/titles/$slug/",
url = "https://api.$domain/api/titles$slug/",
headers = getApiHeaders(),
).handle401().parseJson()
val content = try {
data.getJSONObject("content")
} catch (e: JSONException) {
throw ParseException(data.optString("msg"), e)
throw ParseException(data.optString("msg"), manga.publicUrl, e)
}
val branchId = content.getJSONArray("branches").optJSONObject(0)
?.getLong("id") ?: throw ParseException("No branches found")
?.getLong("id") ?: throw ParseException("No branches found", manga.publicUrl)
val chapters = grabChapters(domain, branchId)
val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.US)
return manga.copy(
@@ -142,6 +140,9 @@ internal class RemangaParser(
)
},
chapters = chapters.mapChapters { i, jo ->
if (jo.getBooleanOrDefault("is_paid", false)) {
return@mapChapters null
}
val id = jo.getLong("id")
val name = jo.getString("name").toTitleCase(Locale.ROOT)
val publishers = jo.optJSONArray("publishers")
@@ -182,9 +183,9 @@ internal class RemangaParser(
}
if (pubDate != null && pubDate > System.currentTimeMillis()) {
val at = SimpleDateFormat.getDateInstance(DateFormat.LONG).format(Date(pubDate))
parseFailed("Глава станет доступной $at")
throw ContentUnavailableException("Глава станет доступной $at")
} else {
parseFailed("Глава недоступна")
throw ContentUnavailableException("Глава недоступна")
}
}
val result = ArrayList<MangaPage>(pages.length())
@@ -192,7 +193,7 @@ internal class RemangaParser(
when (val item = pages.get(i)) {
is JSONObject -> result += parsePage(item, referer)
is JSONArray -> item.mapJSONTo(result) { parsePage(it, referer) }
else -> throw ParseException("Unknown json item $item")
else -> throw ParseException("Unknown json item $item", chapter.url)
}
}
return result

View File

@@ -0,0 +1,177 @@
package org.koitharu.kotatsu.parsers.site
import org.jsoup.nodes.Element
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import org.koitharu.kotatsu.parsers.util.json.getStringOrNull
import org.koitharu.kotatsu.parsers.util.json.mapJSON
import java.text.SimpleDateFormat
import java.util.*
@MangaSourceParser("UNION_MANGAS", "Union Mangás", "pt")
class UnionMangasParser(override val context: MangaLoaderContext) : PagedMangaParser(MangaSource.UNION_MANGAS, 40) {
override val sortOrders = EnumSet.of(
SortOrder.ALPHABETICAL,
SortOrder.POPULARITY,
)
override val configKeyDomain = ConfigKey.Domain("unionleitor.top", emptyArray())
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
if (!query.isNullOrEmpty()) {
return if (page == searchPaginator.firstPage) {
search(query)
} else {
emptyList()
}
}
val tag = tags.oneOrThrowIfMany()
val url = urlBuilder()
.addPathSegment("lista-mangas")
.addPathSegment(
when {
tag != null -> tag.key
sortOrder == SortOrder.ALPHABETICAL -> "a-z"
else -> "visualizacoes"
},
).addPathSegment(page.toString())
val doc = context.httpGet(url.build()).parseHtml()
val root = doc.selectFirstOrThrow("div.tamanho-bloco-perfil")
return root.select(".lista-mangas-novos").map { div ->
val a = div.selectFirstOrThrow("a")
val img = div.selectFirstOrThrow("img")
val href = a.attrAsRelativeUrl("href")
Manga(
id = generateUid(href),
url = href,
publicUrl = a.attrAsAbsoluteUrl("href"),
title = div.selectLastOrThrow("a").text(),
coverUrl = img.attrAsAbsoluteUrl("src"),
altTitle = null,
rating = RATING_UNKNOWN,
tags = emptySet(),
description = div.selectLast("div")?.ownText(),
state = null,
author = null,
isNsfw = false,
source = source,
)
}
}
override suspend fun getDetails(manga: Manga): Manga {
val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml()
val root = doc.selectFirstOrThrow(".perfil-manga")
val dateFormat = SimpleDateFormat("dd/MM/yyyy", Locale.ROOT)
return manga.copy(
rating = root.select("h2")
.find { it.ownText().startsWith('#') }
?.ownText()?.drop(1)?.toFloatOrNull()?.div(10f) ?: manga.rating,
largeCoverUrl = root.selectFirst("img.img-thumbnail")?.attrAsAbsoluteUrlOrNull("src"),
description = root.selectFirst(".panel-default")?.selectFirst(".panel-body")?.html(),
author = root.tableValue("Autor")?.ownText(),
altTitle = root.tableValue("Título(s) Alternativo(s)")?.ownText(),
state = when (root.tableValue("Status")?.selectLast(".label")?.text()) {
"Completo" -> MangaState.FINISHED
"Ativo" -> MangaState.ONGOING
else -> null
},
tags = root.tableValue("Gênero(s)")?.select("a")?.mapToSet {
it.toMangaTag()
} ?: manga.tags,
isNsfw = root.selectFirst(".alert-danger")?.html()?.contains("18 anos") == true,
chapters = root.select("div.row.capitulos").asReversed().mapChapters { i, div ->
val a = div.selectFirstOrThrow("a")
val href = a.attrAsRelativeUrl("href")
val title = a.text()
MangaChapter(
id = generateUid(href),
name = title,
number = i + 1,
url = href,
scanlator = div.selectLast("a")?.text()?.takeUnless { it == title },
uploadDate = dateFormat.tryParse(
a.nextElementSibling()?.text()?.removeSurrounding("(", ")"),
),
branch = null,
source = source,
)
},
)
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(getDomain())
val doc = context.httpGet(fullUrl).parseHtml()
val root = doc.body().selectFirstOrThrow("article")
return root.selectOrThrow("img[pag]").mapNotNull { img ->
val href = img.attrAsRelativeUrl("src")
if (href.startsWith("/images/banner")) {
return@mapNotNull null
}
MangaPage(
id = generateUid(href),
url = href,
referer = fullUrl,
preview = null,
source = source,
)
}
}
override suspend fun getTags(): Set<MangaTag> {
val doc = context.httpGet(urlBuilder().addPathSegment("lista-mangas").build()).parseHtml()
val ul = doc.body().selectFirstOrThrow(".nav-tabs").selectFirstOrThrow("ul.dropdown-menu")
return ul.select("li").mapToSet { li ->
li.selectFirstOrThrow("a").toMangaTag()
}
}
private suspend fun search(query: String): List<Manga> {
val domain = getDomain()
val json = context.httpGet(
urlBuilder()
.addPathSegments("assets/busca.php")
.addQueryParameter("nomeManga", query)
.build(),
).parseJson()
return json.getJSONArray("items").mapJSON { jo ->
val href = "/pagina-manga/" + jo.getString("url")
Manga(
id = generateUid(href),
url = href,
publicUrl = href.toAbsoluteUrl(domain),
title = jo.getString("titulo"),
rating = RATING_UNKNOWN,
tags = emptySet(),
author = jo.getStringOrNull("autor"),
coverUrl = jo.getString("imagem"),
state = null,
isNsfw = false,
altTitle = null,
source = source,
)
}
}
private fun Element.tableValue(title: String): Element? {
return select("h4.media-heading")
.find { it.selectFirst("label.subtit-manga")?.text()?.contains(title, ignoreCase = true) == true }
}
private fun Element.toMangaTag() = MangaTag(
title = text().toTitleCase(sourceLocale ?: Locale.ROOT),
key = attr("href").removeSuffix('/').substringAfterLast('/'),
source = source,
)
}

View File

@@ -7,6 +7,7 @@ import org.json.JSONArray
import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.MangaParserAuthProvider
import org.koitharu.kotatsu.parsers.exception.AuthRequiredException
import org.koitharu.kotatsu.parsers.exception.ParseException
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import org.koitharu.kotatsu.parsers.util.json.mapJSON
@@ -79,13 +80,13 @@ internal abstract class GroupleParser(
else -> advancedSearch(domain, tags)
}.parseHtml().body()
val root = (doc.getElementById("mangaBox") ?: doc.getElementById("mangaResults"))
?: parseFailed("Cannot find root")
?: doc.parseFailed("Cannot find root")
val tiles = root.selectFirst("div.tiles.row") ?: if (
root.select(".alert").any { it.ownText() == NOTHING_FOUND }
) {
return emptyList()
} else {
parseFailed("No tiles found")
doc.parseFailed("No tiles found")
}
val baseHost = root.baseUri().toHttpUrl().host
return tiles.select("div.tile").mapNotNull { node ->
@@ -110,11 +111,10 @@ internal abstract class GroupleParser(
altTitle = descDiv.selectFirst("h4")?.text(),
coverUrl = imgDiv.selectFirst("img.lazy")?.attr("data-original")?.replace("_p.", ".").orEmpty(),
rating = runCatching {
node.selectFirst("div.rating")
node.selectFirst(".compact-rate")
?.attr("title")
?.substringBefore(' ')
?.toFloatOrNull()
?.div(10f)
?.div(5f)
}.getOrNull() ?: RATING_UNKNOWN,
author = tileInfo?.selectFirst("a.person-link")?.text(),
isNsfw = false,
@@ -142,7 +142,7 @@ internal abstract class GroupleParser(
override suspend fun getDetails(manga: Manga): Manga {
val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain()), headers).parseHtml()
val root = doc.body().getElementById("mangaBox")?.selectFirst("div.leftContent")
?: parseFailed("Cannot find root")
?: doc.parseFailed("Cannot find root")
val dateFormat = SimpleDateFormat("dd.MM.yy", Locale.US)
val coverImg = root.selectFirst("div.subject-cover")?.selectFirst("img")
return manga.copy(
@@ -218,7 +218,7 @@ internal abstract class GroupleParser(
)
}
}
parseFailed("Pages list not found at ${chapter.url}")
doc.parseFailed("Pages list not found at ${chapter.url}")
}
override suspend fun getPageUrl(page: MangaPage): String {
@@ -232,14 +232,14 @@ internal abstract class GroupleParser(
return url
}
}
val fallbackServer = servers.firstOrNull() ?: parseFailed("Cannot find any page url")
val fallbackServer = servers.firstOrNull() ?: throw ParseException("Cannot find any page url", page.url)
return fallbackServer + path
}
override suspend fun getTags(): Set<MangaTag> {
val doc = context.httpGet("https://${getDomain()}/list/genres/sort_name", headers).parseHtml()
val root = doc.body().getElementById("mangaBox")?.selectFirst("div.leftContent")
?.selectFirst("table.table") ?: parseFailed("Cannot find root")
?.selectFirst("table.table") ?: doc.parseFailed("Cannot find root")
return root.select("a.element-link").mapToSet { a ->
MangaTag(
title = a.text().toTitleCase(),
@@ -254,7 +254,7 @@ internal abstract class GroupleParser(
val element = root.selectFirst("img.user-avatar") ?: throw AuthRequiredException(source)
val res = element.parent()?.text()
return if (res.isNullOrEmpty()) {
parseFailed("Cannot find username")
root.parseFailed("Cannot find username")
} else res
}
@@ -273,21 +273,21 @@ internal abstract class GroupleParser(
val tagsIndex = context.httpGet(url, headers).parseHtml()
.body().selectFirst("form.search-form")
?.select("div.form-group")
?.get(1) ?: parseFailed("Genres filter element not found")
?.get(1) ?: throw ParseException("Genres filter element not found", url)
val tagNames = tags.map { it.title.lowercase() }
val payload = HashMap<String, String>()
var foundGenres = 0
tagsIndex.select("li.property").forEach { li ->
val name = li.text().trim().lowercase()
val id = li.selectFirst("input")?.id()
?: parseFailed("Id for tag $name not found")
?: li.parseFailed("Id for tag $name not found")
payload[id] = if (name in tagNames) {
foundGenres++
"in"
} else ""
}
if (foundGenres != tags.size) {
parseFailed("Some genres are not found")
tagsIndex.parseFailed("Some genres are not found")
}
// Step 2: advanced search
payload["q"] = ""

View File

@@ -11,7 +11,7 @@ internal class ReadmangaParser(
) : GroupleParser(MangaSource.READMANGA_RU, "readmangafun", 1) {
override val configKeyDomain = ConfigKey.Domain(
"readmanga.io",
"readmanga.live",
arrayOf("readmanga.io", "readmanga.live", "readmanga.me"),
)

View File

@@ -0,0 +1,47 @@
package org.koitharu.kotatsu.parsers.site.madara
import org.jsoup.nodes.Document
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.model.Manga
import org.koitharu.kotatsu.parsers.model.MangaChapter
import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.util.*
import java.text.SimpleDateFormat
import java.util.*
@MangaSourceParser("ISEKAISCAN_EU", "IsekaiScan (eu)", "en")
internal class IsekaiScanEuParser(context: MangaLoaderContext) :
MadaraParser(context, MangaSource.ISEKAISCAN_EU, "isekaiscan.eu") {
override val datePattern = "MM/dd/yyyy"
override suspend fun getChapters(manga: Manga, doc: Document): List<MangaChapter> {
val mangaId = doc.body().requireElementById("manga-chapters-holder").attr("data-id")
val ul = context.httpPost(
"https://${getDomain()}/wp-admin/admin-ajax.php",
mapOf(
"action" to "manga_get_chapters",
"manga" to mangaId,
),
).parseHtml().body().selectFirstOrThrow("ul")
val dateFormat = SimpleDateFormat(datePattern, Locale.US)
return ul.select("li").asReversed().mapChapters { i, li ->
val a = li.selectFirst("a")
val href = a?.attrAsRelativeUrlOrNull("href") ?: li.parseFailed("Link is missing")
MangaChapter(
id = generateUid(href),
name = a.ownText(),
number = i + 1,
url = href,
uploadDate = parseChapterDate(
dateFormat,
li.selectFirst("span.chapter-release-date i")?.text(),
),
source = source,
scanlator = null,
branch = null,
)
}
}
}

View File

@@ -4,8 +4,8 @@ import androidx.collection.arraySetOf
import org.jsoup.nodes.Element
import org.koitharu.kotatsu.parsers.InternalParsersApi
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
@@ -16,9 +16,9 @@ abstract class Madara5Parser @InternalParsersApi constructor(
override val context: MangaLoaderContext,
source: MangaSource,
domain: String,
) : MangaParser(source) {
) : PagedMangaParser(source, pageSize = 22) {
protected open val pageSize = 22
protected open val datePattern = "MMMM dd, HH:mm"
protected open val tagPrefix = "/mangas/"
protected open val nsfwTags = arraySetOf("yaoi", "yuri", "mature")
@@ -26,9 +26,12 @@ abstract class Madara5Parser @InternalParsersApi constructor(
override val configKeyDomain = ConfigKey.Domain(domain, null)
@InternalParsersApi
override suspend fun getList(offset: Int, query: String?, tags: Set<MangaTag>?, sortOrder: SortOrder): List<Manga> {
val page = (offset / pageSize.toFloat()).toIntUp()
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val domain = getDomain()
val url = buildString {
append("https://")
@@ -87,7 +90,7 @@ abstract class Madara5Parser @InternalParsersApi constructor(
?.getElementsByAttributeValueContaining("href", tagPrefix)
?.mapToSet { a -> a.asMangaTag() } ?: manga.tags
val mangaId = root.getElementById("manga-chapters-holder")?.attr("data-id")?.toLongOrNull()
?: parseFailed("Cannot find mangaId")
?: root.parseFailed("Cannot find mangaId")
return manga.copy(
description = (root.selectFirst(".detail-content")
?: root.selectFirstOrThrow(".description-summary")).html(),
@@ -104,7 +107,7 @@ abstract class Madara5Parser @InternalParsersApi constructor(
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(getDomain())
val doc = context.httpGet(fullUrl).parseHtml()
val arrayData = doc.getElementById("arraydata") ?: parseFailed("#arraydata not found")
val arrayData = doc.getElementById("arraydata") ?: doc.parseFailed("#arraydata not found")
return arrayData.html().split(',').map { url ->
MangaPage(
id = generateUid(url),
@@ -123,7 +126,7 @@ abstract class Madara5Parser @InternalParsersApi constructor(
}
private suspend fun loadChapters(mangaId: Long): List<MangaChapter> {
val dateFormat = SimpleDateFormat("MMMM dd, HH:mm", sourceLocale ?: Locale.US)
val dateFormat = SimpleDateFormat(datePattern, sourceLocale ?: Locale.US)
val doc = context.httpGet("https://${getDomain()}/ajax-list-chapter?mangaID=$mangaId").parseHtml()
return doc.select("li.wp-manga-chapter").asReversed().mapChapters { i, li ->
val a = li.selectFirstOrThrow("a")

View File

@@ -0,0 +1,70 @@
package org.koitharu.kotatsu.parsers.site.madara
import kotlinx.coroutines.async
import kotlinx.coroutines.coroutineScope
import org.jsoup.nodes.Element
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import java.text.SimpleDateFormat
import java.util.*
internal abstract class Madara6Parser(
context: MangaLoaderContext,
source: MangaSource,
domain: String,
) : MadaraParser(context, source, domain) {
override val datePattern: String = "dd MMMM yyyy"
override suspend fun getDetails(manga: Manga): Manga {
return coroutineScope {
val chapters = async { loadChapters(manga.url) }
val body = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml().body()
parseDetails(manga, body, chapters.await())
}
}
protected fun Element.tableValue(): Element {
for (p in parents()) {
val children = p.children()
if (children.size == 2) {
return children[1]
}
}
parseFailed("Cannot find tableValue for node ${text()}")
}
protected abstract fun String.asMangaState(): MangaState?
protected fun Element.asMangaTag() = MangaTag(
title = ownText(),
key = attr("href").removeSuffix('/').substringAfterLast('/')
.replace('-', '+'),
source = source,
)
protected open suspend fun loadChapters(mangaUrl: String): List<MangaChapter> {
val url = mangaUrl.toAbsoluteUrl(getDomain()).removeSuffix('/') + "/ajax/chapters/"
val dateFormat = SimpleDateFormat(datePattern, sourceLocale ?: Locale.ROOT)
val doc = context.httpPost(url, emptyMap()).parseHtml()
return doc.select("li.wp-manga-chapter").asReversed().mapChapters { i, li ->
val a = li.selectFirstOrThrow("a")
val href = a.attrAsRelativeUrl("href")
MangaChapter(
id = generateUid(href),
url = href,
name = a.text(),
number = i + 1,
branch = null,
uploadDate = dateFormat.tryParse(
li.selectFirst(".chapter-release-date")?.text()?.trim(),
),
scanlator = null,
source = source,
)
}
}
protected abstract fun parseDetails(manga: Manga, body: Element, chapters: List<MangaChapter>): Manga
}

View File

@@ -1,9 +1,12 @@
package org.koitharu.kotatsu.parsers.site.madara
import kotlinx.coroutines.async
import kotlinx.coroutines.coroutineScope
import org.jsoup.nodes.Document
import org.jsoup.nodes.Element
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.exception.ParseException
import org.koitharu.kotatsu.parsers.model.*
@@ -12,13 +15,11 @@ import java.text.DateFormat
import java.text.SimpleDateFormat
import java.util.*
private const val PAGE_SIZE = 12
internal abstract class MadaraParser(
override val context: MangaLoaderContext,
source: MangaSource,
domain: String,
) : MangaParser(source) {
) : PagedMangaParser(source, pageSize = 12) {
override val configKeyDomain = ConfigKey.Domain(domain, null)
@@ -29,30 +30,36 @@ internal abstract class MadaraParser(
protected open val tagPrefix = "manga-genre/"
protected open val isNsfwSource = false
protected open val datePattern = "MMMM dd, yyyy"
override suspend fun getList(
offset: Int,
init {
paginator.firstPage = 0
searchPaginator.firstPage = 0
}
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val tag = tags.oneOrThrowIfMany()
val payload = createRequestTemplate()
payload["page"] = (offset / PAGE_SIZE.toFloat()).toIntUp().toString()
payload["page"] = page.toString()
payload["vars[meta_key]"] = when (sortOrder) {
SortOrder.POPULARITY -> "_wp_manga_views"
SortOrder.UPDATED -> "_latest_update"
else -> "_wp_manga_views"
}
payload["vars[wp-manga-genre]"] = tag?.key.orEmpty()
payload["vars[s]"] = query.orEmpty()
payload["vars[s]"] = query?.urlEncoded().orEmpty()
val doc = context.httpPost(
"https://${getDomain()}/wp-admin/admin-ajax.php",
payload,
).parseHtml()
return doc.select("div.row.c-tabs-item__content").map { div ->
val href = div.selectFirst("a")?.attrAsRelativeUrlOrNull("href")
?: parseFailed("Link not found")
?: div.parseFailed("Link not found")
val summary = div.selectFirst(".tab-summary")
Manga(
id = generateUid(href),
@@ -91,7 +98,7 @@ internal abstract class MadaraParser(
val root1 = body.selectFirst("header")?.selectFirst("ul.second-menu")
val root2 = body.selectFirst("div.genres_wrap")?.selectFirst("ul.list-unstyled")
if (root1 == null && root2 == null) {
parseFailed("Root not found")
doc.parseFailed("Root not found")
}
val list = root1?.select("li").orEmpty() + root2?.select("li").orEmpty()
val keySet = HashSet<String>(list.size)
@@ -112,18 +119,18 @@ internal abstract class MadaraParser(
}
}
override suspend fun getDetails(manga: Manga): Manga {
override suspend fun getDetails(manga: Manga): Manga = coroutineScope {
val fullUrl = manga.url.toAbsoluteUrl(getDomain())
val doc = context.httpGet(fullUrl).parseHtml()
val chaptersDeferred = async { getChapters(manga, doc) }
val root = doc.body().selectFirst("div.profile-manga")
?.selectFirst("div.summary_content")
?.selectFirst("div.post-content")
?: throw ParseException("Root not found")
?: throw ParseException("Root not found", fullUrl)
val root2 = doc.body().selectFirst("div.content-area")
?.selectFirst("div.c-page")
?: throw ParseException("Root2 not found")
val dateFormat = SimpleDateFormat("MMMM dd, yyyy", Locale.US)
return manga.copy(
?: throw ParseException("Root2 not found", fullUrl)
manga.copy(
tags = root.selectFirst("div.genres-content")?.select("a")
?.mapNotNullToSet { a ->
MangaTag(
@@ -137,35 +144,42 @@ internal abstract class MadaraParser(
?.select("p")
?.filterNot { it.ownText().startsWith("A brief description") }
?.joinToString { it.html() },
chapters = root2.select("li").asReversed().mapChapters { i, li ->
val a = li.selectFirst("a")
val href = a?.attrAsRelativeUrlOrNull("href") ?: parseFailed("Link is missing")
MangaChapter(
id = generateUid(href),
name = a.ownText(),
number = i + 1,
url = href,
uploadDate = parseChapterDate(
dateFormat,
li.selectFirst("span.chapter-release-date i")?.text(),
),
source = source,
scanlator = null,
branch = null,
)
},
chapters = chaptersDeferred.await(),
)
}
protected open suspend fun getChapters(manga: Manga, doc: Document): List<MangaChapter> {
val root2 = doc.body().selectFirstOrThrow("div.content-area")
.selectFirstOrThrow("div.c-page")
val dateFormat = SimpleDateFormat(datePattern, sourceLocale ?: Locale.US)
return root2.select("li").asReversed().mapChapters { i, li ->
val a = li.selectFirst("a")
val href = a?.attrAsRelativeUrlOrNull("href") ?: li.parseFailed("Link is missing")
MangaChapter(
id = generateUid(href),
name = a.ownText(),
number = i + 1,
url = href,
uploadDate = parseChapterDate(
dateFormat,
li.selectFirst("span.chapter-release-date i")?.text(),
),
source = source,
scanlator = null,
branch = null,
)
}
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(getDomain())
val doc = context.httpGet(fullUrl).parseHtml()
val root = doc.body().selectFirst("div.main-col-inner")
?.selectFirst("div.reading-content")
?: throw ParseException("Root not found")
?: throw ParseException("Root not found", fullUrl)
return root.select("div.page-break").map { div ->
val img = div.selectFirst("img") ?: parseFailed("Page image not found")
val url = img.src()?.toRelativeUrl(getDomain()) ?: parseFailed("Image src not found")
val img = div.selectFirst("img") ?: div.parseFailed("Page image not found")
val url = img.src()?.toRelativeUrl(getDomain()) ?: div.parseFailed("Image src not found")
MangaPage(
id = generateUid(url),
url = url,
@@ -176,7 +190,7 @@ internal abstract class MadaraParser(
}
}
private fun parseChapterDate(dateFormat: DateFormat, date: String?): Long {
protected fun parseChapterDate(dateFormat: DateFormat, date: String?): Long {
date ?: return 0
return when {
date.endsWith(" ago", ignoreCase = true) -> {
@@ -291,10 +305,13 @@ internal abstract class MadaraParser(
@MangaSourceParser("MANGAREAD", "MangaRead", "en")
class MangaRead(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGAREAD, "www.mangaread.org") {
override val tagPrefix = "genres/"
override val datePattern = "dd.MM.yyyy"
}
@MangaSourceParser("MANGAWEEBS", "MangaWeebs", "en")
class MangaWeebs(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGAWEEBS, "mangaweebs.in")
class MangaWeebs(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGAWEEBS, "mangaweebs.in") {
override val datePattern = "dd MMMM HH:mm"
}
@MangaSourceParser("KINGMANGA", "KingManga", "en")
class KingManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.KINGMANGA, "king-manga.com") {
@@ -304,6 +321,7 @@ internal abstract class MadaraParser(
}
}
@Deprecated("Not working, unknown reason, maybe shutdown or domain expired")
@MangaSourceParser("MANGAHATACHI", "MangahaTachi", "ja")
class MangahaTachi(context: MangaLoaderContext) :
MadaraParser(context, MangaSource.MANGAHATACHI, "mangahatachi.com")
@@ -312,7 +330,7 @@ internal abstract class MadaraParser(
class PianManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.PIANMANGA, "pianmanga.com")
@MangaSourceParser("MANGAROSIE", "MangaRosie", "en")
class MangaRosie(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGAROSIE, "mangarosie.me")
class MangaRosie(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGAROSIE, "mangarosie.love")
@MangaSourceParser("READMANWHA", "ReadManwha", "en")
class ReadManwha(context: MangaLoaderContext) : MadaraParser(context, MangaSource.READMANWHA, "readmanwha.net")
@@ -321,7 +339,9 @@ internal abstract class MadaraParser(
class MangaTx(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGATX, "mangatx.com")
@MangaSourceParser("MANGAEFFECT", "MangaEffect", "en")
class MangaEffect(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGAEFFECT, "mangaeffect.com")
class MangaEffect(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGAEFFECT, "mangaeffect.com") {
override val datePattern = "dd.MM.yyyy"
}
@MangaSourceParser("AQUAMANGA", "AquaManga", "en")
class AquaManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.AQUAMANGA, "aquamanga.com") {
@@ -343,7 +363,9 @@ internal abstract class MadaraParser(
}
@MangaSourceParser("HARIMANGA", "HariManga", "en")
class HariManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.HARIMANGA, "harimanga.com")
class HariManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.HARIMANGA, "harimanga.com") {
override val datePattern = "MM/dd/yyyy"
}
@MangaSourceParser("KISSMANGA", "KissManga", "en")
class KissManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.KISSMANGA, "kissmanga.in")
@@ -351,9 +373,6 @@ internal abstract class MadaraParser(
@MangaSourceParser("MANGAROCK", "MangaRock", "en")
class MangaRock(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGAROCK, "mangarockteam.com")
@MangaSourceParser("ISEKAISCAN_EU", "IsekaiScan (eu)", "en")
class IsekaiScanEu(context: MangaLoaderContext) : MadaraParser(context, MangaSource.ISEKAISCAN_EU, "isekaiscan.eu")
@MangaSourceParser("FREEMANGA", "FreeManga", "en")
class FreeManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.FREEMANGA, "freemanga.me")
@@ -381,7 +400,9 @@ internal abstract class MadaraParser(
class ManhwaClan(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANHWACLAN, "manhwaclan.com")
@MangaSourceParser("MANGA_3S", "Manga3s", "en")
class Manga3s(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGA_3S, "manga3s.com")
class Manga3s(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGA_3S, "manga3s.com") {
override val tagPrefix = "manhwa-genre/"
}
@MangaSourceParser("MANHWAKOOL", "Manhwa Kool", "en")
class ManhwaKool(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANHWAKOOL, "manhwakool.com") {
@@ -393,6 +414,7 @@ internal abstract class MadaraParser(
@MangaSourceParser("TOPMANHUA", "Top Manhua", "en")
class TopManhua(context: MangaLoaderContext) : MadaraParser(context, MangaSource.TOPMANHUA, "www.topmanhua.com") {
override val tagPrefix = "manhua-genre/"
override val datePattern = "MM/dd/yyyy"
}
@MangaSourceParser("X2MANGA", "X2Manga", "en")
@@ -487,6 +509,8 @@ internal abstract class MadaraParser(
@MangaSourceParser("TREE_MANGA", "Tree Manga", "en")
class TreeManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.TREE_MANGA, "treemanga.com") {
override val datePattern = "MM/dd/yyyy"
override fun getFaviconUrl(): String {
return "https://${getDomain()}/wp-content/uploads/2017/10/lgoo-treemanga-2-1.jpg"
}
@@ -521,7 +545,9 @@ internal abstract class MadaraParser(
class Manga365(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGA_365, "365manga.com")
@MangaSourceParser("MANGACLASH", "Mangaclash", "en")
class Mangaclash(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGACLASH, "mangaclash.com")
class Mangaclash(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGACLASH, "mangaclash.com") {
override val datePattern = "MM/dd/yyyy"
}
@MangaSourceParser("ZINMANGA", "ZINMANGA", "en")
class Zinmanga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.ZINMANGA, "zinmanga.com")

View File

@@ -0,0 +1,51 @@
package org.koitharu.kotatsu.parsers.site.madara
import org.jsoup.nodes.Element
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.model.Manga
import org.koitharu.kotatsu.parsers.model.MangaChapter
import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.model.MangaState
import org.koitharu.kotatsu.parsers.util.attrAsAbsoluteUrlOrNull
import org.koitharu.kotatsu.parsers.util.mapToSet
import org.koitharu.kotatsu.parsers.util.selectFirstOrThrow
@MangaSourceParser("MANGATREND", "MangaTrend", "ar")
internal class MangaTrendParser(context: MangaLoaderContext) :
Madara6Parser(context, MangaSource.MANGATREND, "mangaatrend.com") {
override val tagPrefix = "manga-genre/"
override fun getFaviconUrl(): String {
return "https://${getDomain()}/wp-content/uploads/2022/08/23UOmUDN_400x400-150x150.png"
}
override fun parseDetails(manga: Manga, body: Element, chapters: List<MangaChapter>): Manga {
val root = body.selectFirstOrThrow(".site-content")
val postContent = root.selectFirstOrThrow(".post-content")
val tags = postContent.getElementsContainingOwnText("التصنيفات")
.firstOrNull()?.tableValue()
?.getElementsByAttributeValueContaining("href", tagPrefix)
?.mapToSet { a -> a.asMangaTag() } ?: manga.tags
return manga.copy(
largeCoverUrl = root.selectFirst("summary_image")
?.selectFirst("img[src]")
?.attrAsAbsoluteUrlOrNull("src"),
description = (root.selectFirst(".detail-content")
?: root.selectFirstOrThrow(".manga-excerpt")).html(),
author = postContent.getElementsContainingOwnText("المؤلف")
.firstOrNull()?.tableValue()?.text()?.trim(),
state = postContent.getElementsContainingOwnText("الحالة")
.firstOrNull()?.tableValue()?.text()?.asMangaState(),
tags = tags,
isNsfw = body.hasClass("adult-content"),
chapters = chapters,
)
}
override fun String.asMangaState() = when (trim()) {
"OnGoing" -> MangaState.ONGOING
else -> null
}
}

View File

@@ -0,0 +1,56 @@
package org.koitharu.kotatsu.parsers.site.madara
import org.jsoup.nodes.Element
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.model.Manga
import org.koitharu.kotatsu.parsers.model.MangaChapter
import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.model.MangaState
import org.koitharu.kotatsu.parsers.util.attrAsAbsoluteUrlOrNull
import org.koitharu.kotatsu.parsers.util.mapToSet
import org.koitharu.kotatsu.parsers.util.selectFirstOrThrow
import java.util.*
@MangaSourceParser("MANGAS_ORIGINES", "Mangas Origines", "fr")
internal class MangasOriginesParser(context: MangaLoaderContext) :
Madara6Parser(context, MangaSource.MANGAS_ORIGINES, "mangas-origines.fr") {
override val tagPrefix = "catalogues-genre/"
override fun getFaviconUrl(): String {
return "https://${getDomain()}/wp-content/uploads/2020/11/Mangas-150x150.png"
}
override fun parseDetails(manga: Manga, body: Element, chapters: List<MangaChapter>): Manga {
val root = body.selectFirstOrThrow(".site-content")
val postContent = root.selectFirstOrThrow(".post-content")
val tags = postContent.getElementsContainingOwnText("Genre")
.firstOrNull()?.tableValue()
?.getElementsByAttributeValueContaining("href", tagPrefix)
?.mapToSet { a -> a.asMangaTag() } ?: manga.tags
return manga.copy(
largeCoverUrl = root.selectFirst("picture")
?.selectFirst("img[data-src]")
?.attrAsAbsoluteUrlOrNull("data-src"),
description = (root.selectFirst(".detail-content")
?: root.selectFirstOrThrow(".manga-excerpt")).html(),
author = postContent.getElementsContainingOwnText("Auteur")
.firstOrNull()?.tableValue()?.text()?.trim(),
state = postContent.getElementsContainingOwnText("STATUS")
.firstOrNull()?.tableValue()?.text()?.asMangaState(),
tags = tags,
isNsfw = body.hasClass("adult-content"),
chapters = chapters,
)
}
override fun String.asMangaState() = when (trim().lowercase(Locale.FRANCE)) {
"en cours" -> MangaState.ONGOING
"abandonné",
"terminé",
-> MangaState.FINISHED
else -> null
}
}

View File

@@ -0,0 +1,57 @@
package org.koitharu.kotatsu.parsers.site.madara
import org.jsoup.nodes.Element
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.model.Manga
import org.koitharu.kotatsu.parsers.model.MangaChapter
import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.model.MangaState
import org.koitharu.kotatsu.parsers.util.attrAsAbsoluteUrlOrNull
import org.koitharu.kotatsu.parsers.util.mapToSet
import org.koitharu.kotatsu.parsers.util.selectFirstOrThrow
import java.util.*
@MangaSourceParser("PRISMA_SCANS", "Prisma Scans", "pt")
internal class PrismaScansParser(context: MangaLoaderContext) :
Madara6Parser(context, MangaSource.PRISMA_SCANS, "prismascans.net") {
override val tagPrefix = "manga-genre/"
override fun getFaviconUrl(): String {
return "https://${getDomain()}/wp-content/uploads/2022/07/cropped-branca-1-192x192.png"
}
override fun parseDetails(manga: Manga, body: Element, chapters: List<MangaChapter>): Manga {
val root = body.selectFirstOrThrow(".site-content")
val postContent = root.selectFirstOrThrow(".post-content")
val tags = postContent.getElementsContainingOwnText("Gênero")
.firstOrNull()?.tableValue()
?.getElementsByAttributeValueContaining("href", tagPrefix)
?.mapToSet { a -> a.asMangaTag() } ?: manga.tags
return manga.copy(
largeCoverUrl = root.selectFirst("picture")
?.selectFirst("img[data-src]")
?.attrAsAbsoluteUrlOrNull("data-src"),
description = root.selectFirstOrThrow(".description-summary").firstElementChild()?.html(),
author = postContent.getElementsContainingOwnText("Artista")
.firstOrNull()?.tableValue()?.text()?.trim(),
altTitle = postContent.getElementsContainingOwnText("Título Alternativo")
.firstOrNull()?.tableValue()?.text()?.trim(),
state = postContent.getElementsContainingOwnText("Status")
.firstOrNull()?.tableValue()?.text()?.asMangaState(),
tags = tags,
isNsfw = body.hasClass("adult-content"),
chapters = chapters,
)
}
override fun String.asMangaState() = when (trim().lowercase(sourceLocale ?: Locale.ROOT)) {
"em lançamento" -> MangaState.ONGOING
"completo",
"cancelado",
-> MangaState.FINISHED
else -> null
}
}

View File

@@ -0,0 +1,51 @@
package org.koitharu.kotatsu.parsers.site.madara
import org.jsoup.nodes.Element
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.attrAsAbsoluteUrlOrNull
import org.koitharu.kotatsu.parsers.util.mapToSet
import org.koitharu.kotatsu.parsers.util.selectFirstOrThrow
import java.util.*
@MangaSourceParser("TATAKAE_SCANS", "Tatakae Scans", "pt")
internal class TatakaeScansParser(context: MangaLoaderContext) :
Madara6Parser(context, MangaSource.TATAKAE_SCANS, "tatakaescan.com") {
override val datePattern: String = "dd 'de' MMMMM 'de' yyyy"
override fun parseDetails(manga: Manga, body: Element, chapters: List<MangaChapter>): Manga {
val root = body.selectFirstOrThrow(".site-content")
val postContent = root.selectFirstOrThrow(".post-content")
val tags = postContent.getElementsContainingOwnText("Gênero")
.firstOrNull()?.tableValue()
?.getElementsByAttributeValueContaining("href", tagPrefix)
?.mapToSet { a -> a.asMangaTag() } ?: manga.tags
return manga.copy(
largeCoverUrl = root.selectFirst("picture")
?.selectFirst("img[data-src]")
?.attrAsAbsoluteUrlOrNull("data-src"),
description = (root.selectFirst(".detail-content")
?: root.selectFirstOrThrow(".manga-excerpt")).html(),
author = postContent.getElementsContainingOwnText("Autor")
.firstOrNull()?.tableValue()?.text()?.trim(),
state = postContent.getElementsContainingOwnText("Status")
.firstOrNull()?.tableValue()?.text()?.asMangaState(),
tags = tags,
isNsfw = body.hasClass("adult-content"),
chapters = chapters,
)
}
override fun String.asMangaState() = when (trim().lowercase(Locale.ROOT)) {
"em lançamento" -> MangaState.ONGOING
else -> null
}
override fun getFaviconUrl(): String {
return "https://${getDomain()}/wp-content/uploads/2022/07/cropped-favicon-180x180.png"
}
}

View File

@@ -3,7 +3,6 @@ package org.koitharu.kotatsu.parsers.site.multichan
import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.MangaParserAuthProvider
import org.koitharu.kotatsu.parsers.exception.AuthRequiredException
import org.koitharu.kotatsu.parsers.exception.ParseException
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import java.text.SimpleDateFormat
@@ -48,7 +47,7 @@ internal abstract class ChanParser(source: MangaSource) : MangaParser(source), M
}
val doc = context.httpGet(url).parseHtml()
val root = doc.body().selectFirst("div.main_fon")?.getElementById("content")
?: parseFailed("Cannot find root")
?: doc.parseFailed("Cannot find root")
return root.select("div.content_row").mapNotNull { row ->
val a = row.selectFirst("div.manga_row1")?.selectFirst("h2")?.selectFirst("a")
?: return@mapNotNull null
@@ -84,8 +83,7 @@ internal abstract class ChanParser(source: MangaSource) : MangaParser(source), M
override suspend fun getDetails(manga: Manga): Manga {
val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml()
val root =
doc.body().getElementById("dle-content") ?: parseFailed("Cannot find root")
val root = doc.body().getElementById("dle-content") ?: doc.parseFailed("Cannot find root")
val dateFormat = SimpleDateFormat("yyyy-MM-dd", Locale.US)
return manga.copy(
description = root.getElementById("description")?.html()?.substringBeforeLast("<div"),
@@ -135,16 +133,16 @@ internal abstract class ChanParser(source: MangaSource) : MangaParser(source), M
)
}
}
parseFailed("Pages list not found at ${chapter.url}")
doc.parseFailed("Pages list not found at ${chapter.url}")
}
override suspend fun getTags(): Set<MangaTag> {
val domain = getDomain()
val doc = context.httpGet("https://$domain/mostfavorites&sort=manga").parseHtml()
val root = doc.body().selectFirst("div.main_fon")?.getElementById("side")
?.select("ul")?.last() ?: throw ParseException("Cannot find root")
?.select("ul")?.last() ?: doc.parseFailed("Cannot find root")
return root.select("li.sidetag").mapToSet { li ->
val a = li.children().last() ?: throw ParseException("a is null")
val a = li.children().lastOrNull() ?: li.parseFailed("a is null")
MangaTag(
title = a.text().toTagName(),
key = a.attr("href").substringAfterLast('/'),

View File

@@ -3,12 +3,8 @@ package org.koitharu.kotatsu.parsers.site.multichan
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.exception.ParseException
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.mapToSet
import org.koitharu.kotatsu.parsers.util.parseHtml
import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl
import org.koitharu.kotatsu.parsers.util.toTitleCase
import org.koitharu.kotatsu.parsers.util.*
@MangaSourceParser("HENCHAN", "Хентай-тян", "ru")
internal class HenChanParser(override val context: MangaLoaderContext) : ChanParser(MangaSource.HENCHAN) {
@@ -34,13 +30,13 @@ internal class HenChanParser(override val context: MangaLoaderContext) : ChanPar
override suspend fun getDetails(manga: Manga): Manga {
val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml()
val root = doc.body().getElementById("dle-content") ?: throw ParseException("Cannot find root")
val root = doc.body().requireElementById("dle-content")
val readLink = manga.url.replace("manga", "online")
return manga.copy(
description = root.getElementById("description")?.html()?.substringBeforeLast("<div"),
largeCoverUrl = root.getElementById("cover")?.absUrl("src"),
tags = root.selectFirst("div.sidetags")?.select("li.sidetag")?.mapToSet {
val a = it.children().last() ?: parseFailed("Invalid tag")
val a = it.children().last() ?: doc.parseFailed("Invalid tag")
MangaTag(
title = a.text().toTitleCase(),
key = a.attr("href").substringAfterLast('/'),

View File

@@ -3,14 +3,10 @@ package org.koitharu.kotatsu.parsers.site.multichan
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.exception.ParseException
import org.koitharu.kotatsu.parsers.model.Manga
import org.koitharu.kotatsu.parsers.model.MangaChapter
import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.util.attrAsRelativeUrl
import org.koitharu.kotatsu.parsers.util.mapChapters
import org.koitharu.kotatsu.parsers.util.parseHtml
import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl
import org.koitharu.kotatsu.parsers.util.*
@MangaSourceParser("YAOICHAN", "Яой-тян", "ru")
internal class YaoiChanParser(override val context: MangaLoaderContext) : ChanParser(MangaSource.YAOICHAN) {
@@ -19,8 +15,7 @@ internal class YaoiChanParser(override val context: MangaLoaderContext) : ChanPa
override suspend fun getDetails(manga: Manga): Manga {
val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml()
val root =
doc.body().getElementById("dle-content") ?: throw ParseException("Cannot find root")
val root = doc.body().requireElementById("dle-content")
return manga.copy(
description = root.getElementById("description")?.html()?.substringBeforeLast("<div"),
largeCoverUrl = root.getElementById("cover")?.absUrl("src"),

View File

@@ -5,9 +5,9 @@ import org.json.JSONArray
import org.json.JSONObject
import org.jsoup.nodes.Document
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.MangaParserAuthProvider
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.exception.AuthRequiredException
import org.koitharu.kotatsu.parsers.exception.ParseException
@@ -22,7 +22,7 @@ import java.util.*
internal open class MangaLibParser(
override val context: MangaLoaderContext,
source: MangaSource,
) : MangaParser(source), MangaParserAuthProvider {
) : PagedMangaParser(source, pageSize = 60), MangaParserAuthProvider {
override val configKeyDomain = ConfigKey.Domain("mangalib.me", null)
@@ -37,16 +37,15 @@ internal open class MangaLibParser(
SortOrder.NEWEST,
)
override suspend fun getList(
offset: Int,
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
if (!query.isNullOrEmpty()) {
return if (offset == 0) search(query) else emptyList()
return if (page == searchPaginator.firstPage) search(query) else emptyList()
}
val page = (offset / 60f).toIntUp()
val url = buildString {
append("https://")
append(getDomain())
@@ -60,7 +59,7 @@ internal open class MangaLibParser(
}
}
val doc = context.httpGet(url).parseHtml()
val root = doc.body().getElementById("manga-list") ?: throw ParseException("Root not found")
val root = doc.body().getElementById("manga-list") ?: doc.parseFailed("Root not found")
val items = root.selectFirst("div.media-cards-grid")?.select("div.media-card-wrap")
?: return emptyList()
return items.mapNotNull { card ->
@@ -86,7 +85,7 @@ internal open class MangaLibParser(
override suspend fun getDetails(manga: Manga): Manga {
val fullUrl = manga.url.toAbsoluteUrl(getDomain())
val doc = context.httpGet("$fullUrl?section=info").parseHtml()
val root = doc.body().getElementById("main-page") ?: throw ParseException("Root not found")
val root = doc.body().getElementById("main-page") ?: throw ParseException("Root not found", fullUrl)
val title = root.selectFirst("div.media-header__wrap")?.children()
val info = root.selectFirst("div.media-content")
val chaptersDoc = context.httpGet("$fullUrl?section=chapters").parseHtml()
@@ -166,11 +165,11 @@ internal open class MangaLibParser(
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(getDomain())
val doc = context.httpGet(fullUrl).parseHtml()
if (doc.location().endsWith("/register")) {
if (doc.location().substringBefore('?').endsWith("/register")) {
throw AuthRequiredException(source)
}
val scripts = doc.head().select("script")
val pg = (doc.body().getElementById("pg")?.html() ?: parseFailed("Element #pg not found"))
val pg = (doc.body().getElementById("pg")?.html() ?: doc.parseFailed("Element #pg not found"))
.substringAfter('=')
.substringBeforeLast(';')
val pages = JSONArray(pg)
@@ -200,7 +199,7 @@ internal open class MangaLibParser(
}
}
}
throw ParseException("Script with info not found")
throw ParseException("Script with info not found", fullUrl)
}
override suspend fun getTags(): Set<MangaTag> {
@@ -223,7 +222,7 @@ internal open class MangaLibParser(
return result
}
}
throw ParseException("Script with genres not found")
throw ParseException("Script with genres not found", url)
}
override val isAuthorized: Boolean
@@ -238,13 +237,13 @@ internal open class MangaLibParser(
if (body.baseUri().endsWith("/login")) {
throw AuthRequiredException(source)
}
return body.selectFirst(".profile-user__username")?.text() ?: parseFailed("Cannot find username")
return body.selectFirst(".profile-user__username")?.text() ?: body.parseFailed("Cannot find username")
}
protected open fun isNsfw(doc: Document): Boolean {
val sidebar = doc.body().run {
selectFirst(".media-sidebar") ?: selectFirst(".media-info")
} ?: parseFailed("Sidebar not found")
} ?: doc.parseFailed("Sidebar not found")
return sidebar.getElementsContainingOwnText("18+").isNotEmpty()
}

View File

@@ -0,0 +1,8 @@
package org.koitharu.kotatsu.parsers.util
fun <T : Any> T?.assertNotNull(name: String): T? {
assert(this != null) {
"Value $name is null"
}
return this
}

View File

@@ -9,10 +9,7 @@ import okhttp3.HttpUrl
private const val SCHEME_HTTPS = "https"
fun CookieJar.insertCookies(domain: String, vararg cookies: String) {
val url = HttpUrl.Builder()
.scheme(SCHEME_HTTPS)
.host(domain)
.build()
val url = safeUrlOf(domain) ?: return
saveFromResponse(
url,
cookies.mapNotNull {
@@ -22,18 +19,12 @@ fun CookieJar.insertCookies(domain: String, vararg cookies: String) {
}
fun CookieJar.insertCookie(domain: String, cookie: Cookie) {
val url = HttpUrl.Builder()
.scheme(SCHEME_HTTPS)
.host(domain)
.build()
val url = safeUrlOf(domain) ?: return
saveFromResponse(url, listOf(cookie))
}
fun CookieJar.getCookies(domain: String): List<Cookie> {
val url = HttpUrl.Builder()
.scheme(SCHEME_HTTPS)
.host(domain)
.build()
val url = safeUrlOf(domain) ?: return emptyList()
return loadForRequest(url)
}
@@ -47,4 +38,13 @@ fun CookieJar.copyCookies(oldDomain: String, newDomain: String, names: Array<Str
}
url.host(newDomain)
saveFromResponse(url.build(), cookies)
}
private fun safeUrlOf(domain: String): HttpUrl? = try {
HttpUrl.Builder()
.scheme(SCHEME_HTTPS)
.host(domain)
.build()
} catch (_: IllegalArgumentException) {
null
}

View File

@@ -4,6 +4,7 @@ package org.koitharu.kotatsu.parsers.util
import okhttp3.HttpUrl.Companion.toHttpUrlOrNull
import org.jsoup.nodes.Element
import org.jsoup.select.Elements
import org.jsoup.select.Selector
import org.koitharu.kotatsu.parsers.exception.ParseException
@@ -90,12 +91,21 @@ fun Element.styleValueOrNull(property: String): String? {
return css.substringAfter(':').removeSuffix(';').trim()
}
/**
* Like a `expectFirst` but with detailed error message
*/
fun Element.selectFirstOrThrow(cssQuery: String): Element {
return Selector.selectFirst(cssQuery, this) ?: throw ParseException("Cannot find \"$cssQuery\"")
return Selector.selectFirst(cssQuery, this) ?: throw ParseException("Cannot find \"$cssQuery\"", baseUri())
}
fun Element.selectOrThrow(cssQuery: String): Elements {
return Selector.select(cssQuery, this).ifEmpty {
throw ParseException("Empty result for \"$cssQuery\"", baseUri())
}
}
fun Element.requireElementById(id: String): Element {
return getElementById(id) ?: throw ParseException("Cannot find \"#$id\"")
return getElementById(id) ?: throw ParseException("Cannot find \"#$id\"", baseUri())
}
fun Element.selectLast(cssQuery: String): Element? {
@@ -103,5 +113,5 @@ fun Element.selectLast(cssQuery: String): Element? {
}
fun Element.selectLastOrThrow(cssQuery: String): Element {
return selectLast(cssQuery) ?: throw ParseException("Cannot find \"$cssQuery\"")
return selectLast(cssQuery) ?: throw ParseException("Cannot find \"$cssQuery\"", baseUri())
}

View File

@@ -0,0 +1,25 @@
package org.koitharu.kotatsu.parsers.util
import androidx.collection.SparseArrayCompat
import androidx.collection.set
class Paginator constructor(private val initialPageSize: Int) {
var firstPage = 1
private var pages = SparseArrayCompat<Int>()
fun getPage(offset: Int): Int {
if (offset == 0) { // just an optimization
return firstPage
}
pages[offset]?.let { return it }
val pageSize = initialPageSize
val intPage = offset / pageSize
val tail = offset % pageSize
return intPage + firstPage + if (tail == 0) 0 else 1
}
fun onListReceived(offset: Int, page: Int, count: Int) {
pages[offset + count] = if (count > 0) page + 1 else page
}
}

View File

@@ -68,10 +68,13 @@ fun String.toAbsoluteUrl(domain: String): String = when {
}
fun DateFormat.tryParse(str: String?): Long = if (str.isNullOrEmpty()) {
assert(false) { "Date string is null or empty" }
0L
} else {
runCatching {
parse(str)?.time ?: 0L
}.onFailure {
assert(false) { "Cannot parse date $str: ${it.message}" }
}.getOrDefault(0L)
}

View File

@@ -136,4 +136,8 @@ fun JSONArray?.isNullOrEmpty(): Boolean {
}
return this == null || this.length() == 0
}
fun JSONArray.toJSONList(): List<JSONObject> {
return List(length()) { i -> getJSONObject(i) }
}

View File

@@ -30,6 +30,18 @@ internal class MangaParserTest {
assert(list.all { it.source == source })
}
@ParameterizedTest(name = "{index}|pagination|{0}")
@MangaSources
fun pagination(source: MangaSource) = runTest {
val parser = source.newParser(context)
val page1 = parser.getList(0, sortOrder = null, tags = null)
val page2 = parser.getList(page1.size, sortOrder = null, tags = null)
val intersection = page1.intersect(page2.toSet())
assert(intersection.isEmpty()) {
"Pages are intersected by " + intersection.size
}
}
@ParameterizedTest(name = "{index}|search|{0}")
@MangaSources
fun search(source: MangaSource) = runTest {
@@ -120,7 +132,7 @@ internal class MangaParserTest {
@MangaSources
fun favicon(source: MangaSource) = runTest {
val parser = source.newParser(context)
val favicons = parser.parseFavicons()
val favicons = parser.getFavicons()
val types = setOf("png", "svg", "ico", "gif", "jpg", "jpeg")
assert(favicons.isNotEmpty())
favicons.forEach {

View File

@@ -0,0 +1,42 @@
package org.koitharu.kotatsu.parsers.util
import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.Test
internal class PaginatorTest {
@Test
fun singlePaginationTest() {
val paginator = Paginator(24)
assertEquals(1, paginator.getPage(0))
assertEquals(2, paginator.getPage(24))
assertEquals(3, paginator.getPage(48))
}
@Test
fun adaptivePaginationTest() {
val paginator = Paginator(12)
assertEquals(1, paginator.getPage(0))
paginator.onListReceived(0, 1, 24)
assertEquals(2, paginator.getPage(24))
paginator.onListReceived(24, 2, 18)
assertEquals(3, paginator.getPage(42))
}
@Test
fun endReachPaginationTest() {
val pageSize = 24
val paginator = Paginator(pageSize)
var size = 0
repeat(5) { i ->
val offset = i * pageSize
assertEquals(i + 1, paginator.getPage(offset))
paginator.onListReceived(offset, i + 1, pageSize)
size += pageSize
}
val nextPage = paginator.getPage(size)
assertEquals(nextPage, paginator.getPage(size))
paginator.onListReceived(size, nextPage, 0)
assertEquals(nextPage, paginator.getPage(size))
}
}