KotatsuApp/kotatsu-parsers
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: KotatsuApp:4a8c7fa36abcfb7eeb9ba738e84cf4714b5b5dae
Branches Tags
KotatsuApp:master KotatsuApp:feature/filter-ex KotatsuApp:source/neox KotatsuApp:source/ar KotatsuApp:source/komiktap KotatsuApp:source/tmo KotatsuApp:feature/favicons KotatsuApp:feature/ci
..
compare: KotatsuApp:source/ar
Branches Tags
KotatsuApp:master KotatsuApp:feature/filter-ex KotatsuApp:source/neox KotatsuApp:source/ar KotatsuApp:source/komiktap KotatsuApp:source/tmo KotatsuApp:feature/favicons KotatsuApp:feature/ci

1 Commits
4a8c7fa36a ... source/ar

Author SHA1 Message Date
Zakhar Timoshenko
9952f8f00e [MangaTrend] New source (close #65) 2022-09-25 17:31:06 +03:00
938 changed files with 4388 additions and 37182 deletions
Expand all files Collapse all files

4
.editorconfig
View File

@@ -6,8 +6,8 @@ end_of_line = lf
indent_style = tab
max_line_length = 120
tab_width = 4
insert_final_newline = true
disabled_rules = no-wildcard-imports, no-unused-imports
insert_final_newline = false
disabled_rules=no-wildcard-imports,no-unused-imports
[{*.kt,*.kts}]
ij_kotlin_allow_trailing_comma = true

114
.github/ISSUE_TEMPLATE/report_issue.yml vendored
View File

@@ -1,65 +1,65 @@
name: 🐞 Issue report
description: Report a source issue with a source
labels: [ bug ]
description: Report a source issue in Kotatsu
labels: [bug]
body:
- type: input
id: source
attributes:
label: Source information
description: |
You can find the source name in navigation drawer.
placeholder: |
Example: "MangaDex"
validations:
required: true
- type: input
id: source
attributes:
label: Source information
description: |
You can find the source name in navigation drawer.
placeholder: |
Example: "MangaDex"
validations:
required: true
- type: textarea
id: reproduce-steps
attributes:
label: Steps to reproduce
description: Provide an example of the issue.
placeholder: |
Example:
1. First step
2. Second step
3. Issue here
validations:
required: false
- type: textarea
id: reproduce-steps
attributes:
label: Steps to reproduce
description: Provide an example of the issue.
placeholder: |
Example:
1. First step
2. Second step
3. Issue here
validations:
required: false
- type: input
id: kotatsu-version
attributes:
label: Kotatsu version
description: |
You can find your Kotatsu version in **Settings → About**.
placeholder: |
Example: "3.3"
validations:
required: true
- type: input
id: kotatsu-version
attributes:
label: Kotatsu version
description: |
You can find your Kotatsu version in **Settings → About**.
placeholder: |
Example: "3.3"
validations:
required: true
- type: input
id: android-version
attributes:
label: Android version
description: |
You can find this somewhere in your Android settings.
placeholder: |
Example: "Android 12"
validations:
required: false
- type: input
id: android-version
attributes:
label: Android version
description: |
You can find this somewhere in your Android settings.
placeholder: |
Example: "Android 12"
validations:
required: false
- type: textarea
id: other-details
attributes:
label: Other details
placeholder: |
Additional details and attachments.
- type: textarea
id: other-details
attributes:
label: Other details
placeholder: |
Additional details and attachments.
- type: checkboxes
id: acknowledgements
attributes:
label: Acknowledgements
options:
- label: I have searched the existing issues and this is a new ticket, **NOT** a duplicate or related to another open issue.
required: true
- type: checkboxes
id: acknowledgements
attributes:
label: Acknowledgements
options:
- label: I have searched the existing issues and this is a new ticket, **NOT** a duplicate or related to another open issue.
required: true

87
.github/ISSUE_TEMPLATE/request_source.yml vendored
View File

@@ -1,53 +1,50 @@
name: 🌐 Source request
description: Suggest a new source for Kotatsu
labels: [ source request ]
labels: [source request]
body:
- type: markdown
attributes:
value: Please specify source **name** and **language** in the issue title
- type: input
id: name
attributes:
label: Source name
placeholder: |
Example: "Example Scans"
validations:
required: true
- type: input
id: name
attributes:
label: Source name
placeholder: |
Example: "Example Scans"
validations:
required: true
- type: input
id: link
attributes:
label: Source link
placeholder: |
Example: "https://example.org"
validations:
required: true
- type: input
id: link
attributes:
label: Source link
placeholder: |
Example: "https://example.org"
validations:
required: true
- type: input
id: language
attributes:
label: Language
placeholder: |
Example: "English"
validations:
required: true
- type: input
id: language
attributes:
label: Language
placeholder: |
Example: "English"
validations:
required: true
- type: textarea
id: other-details
attributes:
label: Other details
placeholder: |
Additional details and attachments.
- type: textarea
id: other-details
attributes:
label: Other details
placeholder: |
Additional details and attachments.
- type: checkboxes
id: acknowledgements
attributes:
label: Acknowledgements
options:
- label: I have searched the existing issues and this is a new ticket, **NOT** a duplicate or related to another open issue.
required: true
- label: I have checked that the source does not already exist on the app.
required: true
- label: I have checked that the source does not already exist by searching the [GitHub repository](https://github.com/KotatsuApp/kotatsu-parsers) and verified it does not appear in the code base.
required: true
- type: checkboxes
id: acknowledgements
attributes:
label: Acknowledgements
options:
- label: I have searched the existing issues and this is a new ticket, **NOT** a duplicate or related to another open issue.
required: true
- label: I have checked that the source does not already exist on the app.
required: true
- label: I have checked that the source does not already exist by searching the [GitHub repository](https://github.com/KotatsuApp/kotatsu-parsers) and verified it does not appear in the code base.
required: true

35
.github/workflows/test-parsers.yml vendored
View File

@@ -1,22 +1,27 @@
name: Parsers test
on:
workflow_dispatch:
pull_request:
paths:
- 'src/main/kotlin/org/koitharu/kotatsu/parsers/**'
workflow_dispatch:
pull_request:
paths:
- 'src/main/kotlin/org/koitharu/kotatsu/parsers/site/*'
permissions:
contents: read
contents: read
jobs:
build-and-test:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/setup-java@v3
with:
java-version: '11'
distribution: 'temurin'
cache: 'gradle'
- run: ./gradlew assemble
build-and-test:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/setup-java@v3
with:
java-version: '11'
distribution: 'temurin'
cache: 'gradle'
- run: ./gradlew :test --tests "org.koitharu.kotatsu.parsers.MangaParserTest" || true
- run: ./gradlew generateTestsReport
- uses: actions/upload-artifact@v3
with:
name: Report
path: build/test-results-html/TEST-org.koitharu.kotatsu.parsers.MangaParserTest.htm

3
.gitignore vendored
View File

@@ -29,7 +29,6 @@
.idea/compiler.xml
.idea/jarRepositories.xml
.idea/modules.xml
.idea/ktlint-plugin.xml
.idea/*.iml
.idea/modules
*.iml
@@ -77,4 +76,4 @@ build/
.idea/codeStyles/
src/test/resources/cookies.txt
local.properties
local.properties

2
.idea/kotlinc.xml generated
View File

@@ -1,6 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="KotlinJpsPluginSettings">
<option name="version" value="1.9.22" />
<option name="version" value="1.6.21" />
</component>
</project>

13
CONTRIBUTING.md
View File

@@ -35,11 +35,11 @@ If it does not contain any documentation about
API, [explore network requests](https://firefox-source-docs.mozilla.org/devtools-user/):
some websites use ajax.
- [Example](https://github.com/KotatsuApp/kotatsu-parsers/blob/master/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ru/DesuMeParser.kt)
- [Example](https://github.com/KotatsuApp/kotatsu-parsers/blob/master/src/main/kotlin/org/koitharu/kotatsu/parsers/site/DesuMeParser.kt)
of Json API usage.
- [Example](https://github.com/KotatsuApp/kotatsu-parsers/blob/master/src/main/kotlin/org/koitharu/kotatsu/parsers/site/be/AnibelParser.kt)
- [Example](https://github.com/KotatsuApp/kotatsu-parsers/blob/master/src/main/kotlin/org/koitharu/kotatsu/parsers/site/AnibelParser.kt)
of GraphQL API usage
- [Example](https://github.com/KotatsuApp/kotatsu-parsers/blob/master/src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/MangaTownParser.kt)
- [Example](https://github.com/KotatsuApp/kotatsu-parsers/blob/master/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaTownParser.kt)
of pure HTML parsing.
If website is based on some engine it is rationally to use common base class for this one (for example, Madara wordress
@@ -59,13 +59,13 @@ All functions in `MangaParser` class are documented. Pay attention to some pecul
- `sortOrders` set should not be empty. If your source is not support sorting, specify one most relevance value.
- If you cannot obtain direct links to pages images inside `getPages` method, it is ok to use an intermediate url
as `Page.url` and fetch a direct link at `getPageUrl` function.
- `getFaviconUrl` function is deprecated, but Kotatsu application before v4 still use it, so it is recommended to
override it too.
- You can use _asserts_ to check some optional fields. For example. `Manga.author` field is not required, but if your
source provide such information, add `assert(it != null)`. This will not have any effect on production but help to
find issues during unit testing.
- If your source website (or it's api) uses pages for pagination instead of offset you should extend `PagedMangaParser`
instead of `MangaParser`.
- Your parser may also implement the `Interceptor` interface for additional manipulation of all network requests and/or
responses, including image loading.
## Development process
@@ -87,5 +87,4 @@ It is recommended to run unit tests before submitting a PR.
## Help
If you need a help or have some questions, ask a community in our [Telegram chat](https://t.me/kotatsuapp)
or [Discord server](https://discord.gg/NNJ5RgVBC5).
If you need a help or have some questions, ask a community in our [Discord server](https://discord.gg/NNJ5RgVBC5).

27
README.md
View File

@@ -1,12 +1,12 @@
# Kotatsu parsers
This library provides manga sources.
Library that provides manga sources.
[![](https://jitpack.io/v/KotatsuApp/kotatsu-parsers.svg)](https://jitpack.io/#KotatsuApp/kotatsu-parsers) ![Kotlin](https://img.shields.io/github/languages/top/KotatsuApp/kotatsu-parsers) ![License](https://img.shields.io/github/license/KotatsuApp/Kotatsu) [![Telegram](https://img.shields.io/badge/chat-telegram-60ACFF)](https://t.me/kotatsuapp) [![Discord](https://img.shields.io/discord/898363402467045416?color=5865f2&label=discord)](https://discord.gg/NNJ5RgVBC5)
[![](https://jitpack.io/v/KotatsuApp/kotatsu-parsers.svg)](https://jitpack.io/#KotatsuApp/kotatsu-parsers) ![Kotlin](https://img.shields.io/github/languages/top/KotatsuApp/kotatsu-parsers) ![License](https://img.shields.io/github/license/KotatsuApp/Kotatsu) [![Discord](https://img.shields.io/discord/898363402467045416?color=5865f2&label=discord)](https://discord.gg/NNJ5RgVBC5)
## Usage
1. Add it to your root build.gradle at the end of repositories:
1. Add it in your root build.gradle at the end of repositories:
```groovy
allprojects {
@@ -35,25 +35,18 @@ This library provides manga sources.
}
```
Versions are available on [JitPack](https://jitpack.io/#KotatsuApp/kotatsu-parsers)
When used in Android
projects, [core library desugaring](https://developer.android.com/studio/write/java8-support#library-desugaring) with
the [NIO specification](https://developer.android.com/studio/write/java11-nio-support-table) should be enabled to
support Java 8+ features.
See for versions at [JitPack](https://jitpack.io/#KotatsuApp/kotatsu-parsers)
3. Usage in code
```kotlin
val parser = mangaLoaderContext.newParserInstance(MangaSource.MANGADEX)
val parser = MangaSource.MANGADEX.newParser(mangaLoaderContext)
```
`mangaLoaderContext` is an implementation of the `MangaLoaderContext` class.
See examples
of [Android](https://github.com/KotatsuApp/Kotatsu/blob/devel/app/src/main/kotlin/org/koitharu/kotatsu/core/parser/MangaLoaderContextImpl.kt)
and [Non-Android](https://github.com/KotatsuApp/kotatsu-dl/blob/master/src/jvmMain/kotlin/org/koitharu/kotatsu_dl/logic/MangaLoaderContextImpl.kt)
implementation.
See [Android](https://github.com/KotatsuApp/Kotatsu/blob/devel/app/src/main/java/org/koitharu/kotatsu/core/parser/MangaLoaderContextImpl.kt)
and [Non-Android](https://github.com/KotatsuApp/kotatsu-dl/blob/master/src/main/kotlin/org/koitharu/kotatsu_dl/env/MangaLoaderContextImpl.kt)
implementation examples.
Note that the `MangaSource.LOCAL` and `MangaSource.DUMMY` parsers cannot be instantiated.
@@ -63,5 +56,5 @@ See [CONTRIBUTING.md](./CONTRIBUTING.md) for the guidelines.
## DMCA disclaimer
The developers of this application have no affiliation with the content available in the app. It is collected from
sources freely available through any web browser.
The developers of this application does not have any affiliation with the content available in the app. It is collecting
from the sources freely available through any web browser.

43
build.gradle
View File

@@ -2,8 +2,8 @@ import tasks.ReportGenerateTask
plugins {
id 'java-library'
id 'org.jetbrains.kotlin.jvm' version '1.9.22'
id 'com.google.devtools.ksp' version '1.9.22-1.0.17'
id 'org.jetbrains.kotlin.jvm'
id 'com.google.devtools.ksp'
id 'maven-publish'
}
@@ -16,27 +16,28 @@ test {
compileKotlin {
kotlinOptions {
jvmTarget = '1.8'
freeCompilerArgs += [
'-opt-in=kotlin.RequiresOptIn',
'-opt-in=kotlin.contracts.ExperimentalContracts',
'-opt-in=kotlinx.coroutines.ExperimentalCoroutinesApi',
'-opt-in=org.koitharu.kotatsu.parsers.InternalParsersApi',
'-opt-in=kotlin.RequiresOptIn',
'-opt-in=kotlin.contracts.ExperimentalContracts',
'-opt-in=kotlinx.coroutines.ExperimentalCoroutinesApi',
'-opt-in=org.koitharu.kotatsu.parsers.InternalParsersApi',
]
}
}
compileTestKotlin {
kotlinOptions {
jvmTarget = '1.8'
freeCompilerArgs += [
'-opt-in=kotlin.RequiresOptIn',
'-opt-in=kotlinx.coroutines.ExperimentalCoroutinesApi',
'-opt-in=org.koitharu.kotatsu.parsers.InternalParsersApi',
'-opt-in=kotlin.RequiresOptIn',
'-opt-in=kotlinx.coroutines.ExperimentalCoroutinesApi',
'-opt-in=org.koitharu.kotatsu.parsers.InternalParsersApi',
]
}
}
kotlin {
jvmToolchain(8)
sourceSets {
main.kotlin.srcDirs += 'build/generated/ksp/main/kotlin'
}
@@ -53,20 +54,20 @@ afterEvaluate {
}
dependencies {
implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-core:1.7.3'
implementation 'com.squareup.okhttp3:okhttp:4.12.0'
implementation 'com.squareup.okio:okio:3.7.0'
api 'org.jsoup:jsoup:1.17.2'
implementation 'org.json:json:20231013'
implementation 'androidx.collection:collection:1.4.0'
implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-core:1.6.4'
implementation 'com.squareup.okhttp3:okhttp:4.10.0'
implementation 'com.squareup.okio:okio:3.2.0'
api 'org.jsoup:jsoup:1.15.2'
implementation 'org.json:json:20220320'
implementation 'androidx.collection:collection-ktx:1.2.0'
ksp project(':kotatsu-parsers-ksp')
testImplementation 'org.junit.jupiter:junit-jupiter-api:5.10.1'
testImplementation 'org.junit.jupiter:junit-jupiter-engine:5.10.1'
testImplementation 'org.junit.jupiter:junit-jupiter-params:5.10.1'
testImplementation 'org.jetbrains.kotlinx:kotlinx-coroutines-test:1.7.3'
testImplementation 'org.junit.jupiter:junit-jupiter-api:5.9.0'
testImplementation 'org.junit.jupiter:junit-jupiter-engine:5.9.0'
testImplementation 'org.junit.jupiter:junit-jupiter-params:5.9.0'
testImplementation 'org.jetbrains.kotlinx:kotlinx-coroutines-test:1.6.4'
testImplementation 'io.webfolder:quickjs:1.1.0'
}
tasks.register('generateTestsReport', ReportGenerateTask)
task generateTestsReport(type: ReportGenerateTask)

13
buildSrc/build.gradle
View File

@@ -1,18 +1,13 @@
plugins {
id 'org.jetbrains.kotlin.jvm' version '1.9.22'
id('org.jetbrains.kotlin.jvm') version '1.6.21'
}
repositories {
mavenCentral()
}
kotlin {
jvmToolchain(8)
}
dependencies {
implementation gradleApi()
implementation 'org.simpleframework:simple-xml:2.7.1'
implementation 'com.soywiz.korlibs.korte:korte-jvm:4.0.10'
implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-core:1.7.3'
}
implementation 'com.soywiz.korlibs.korte:korte-jvm:3.0.0-Beta5'
implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-core:1.6.3'
}

7
buildSrc/src/main/kotlin/tasks/ReportGenerateTask.kt
View File

@@ -1,6 +1,6 @@
package tasks
import korlibs.template.Template
import com.soywiz.korte.Template
import kotlinx.coroutines.runBlocking
import org.gradle.api.DefaultTask
import org.gradle.api.tasks.TaskAction
@@ -42,13 +42,10 @@ open class ReportGenerateTask : DefaultTask() {
val results = LinkedHashMap<String, LinkedHashMap<String, TestCase>>()
val tests = LinkedHashSet<String>()
for (case in testSuite.testCases) {
if (!case.isValid()) {
continue
}
tests.add(case.testName)
val map = results.getOrPut(case.source) { LinkedHashMap() }
val oldValue = map.put(case.testName, case)
check(oldValue == null) { "Check failed: $oldValue" }
check(oldValue == null)
}
val failPercent = (testSuite.failures.toDouble() / testSuite.tests * 100.0).roundToInt()

10
buildSrc/src/main/kotlin/tasks/model/TestCase.kt
View File

@@ -19,16 +19,14 @@ class TestCase {
var failure: Failure? = null
val index by lazy {
name.split('|').getOrNull(0)?.toIntOrNull() ?: 0
name.split('|')[0].toInt()
}
val testName by lazy {
name.split('|').getOrNull(1).orEmpty()
name.split('|')[1]
}
val source by lazy {
name.split('|').getOrNull(2).orEmpty()
name.split('|')[2]
}
fun isValid() = name.count { it == '|' } == 2
}
}

133
buildSrc/src/main/resources/report.html
View File

@@ -2,15 +2,13 @@
<html lang="en">
<head>
<meta charset="utf-8">
<meta content="width=device-width, initial-scale=1" name="viewport">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>{{ testSuite.name }}</title>
<!-- CSS only -->
<link crossorigin="anonymous" href="https://cdn.jsdelivr.net/npm/bootstrap@5.2.2/dist/css/bootstrap.min.css"
integrity="sha384-Zenh87qX5JnK2Jl0vWa8Ck2rdkQ2Bzep5IDxbcnCeuOxjzrPF/et3URy9Bv1WTRi" rel="stylesheet">
<!-- JavaScript Bundle with Popper -->
<script crossorigin="anonymous"
integrity="sha384-OERcA2EqjJCMA+/3y+gxIOqMEjwtxJY7qPCqsdltbNJuaOe923+mo//f6V8Qbsw3"
src="https://cdn.jsdelivr.net/npm/bootstrap@5.2.2/dist/js/bootstrap.bundle.min.js"></script>
<link href="https://cdn.jsdelivr.net/npm/bootstrap@5.2.0-beta1/dist/css/bootstrap.min.css" rel="stylesheet"
integrity="sha384-0evHe/X+R7YkIZDRvuzKMRqM+OrBnVFBL6DOitfPri4tjfHxaWutUpFmBp4vmVor" crossorigin="anonymous">
<script src="https://cdn.jsdelivr.net/npm/bootstrap@5.2.0-beta1/dist/js/bootstrap.bundle.min.js"
integrity="sha384-pprn3073KE6tl6bjs2QrFaJGz5/SUsLqktiwsUTF55Jfv3qYSDhgCecCxMW52nD2"
crossorigin="anonymous"></script>
<script src="https://cdn.jsdelivr.net/npm/feather-icons/dist/feather.min.js"></script>
</head>
<body class="py-4">
@@ -27,80 +25,67 @@
{{ testSuite.errors }} ({{ error_percent }}%)
</div>
</div>
<table class="table table-hover">
<thead class="sticky-top bg-body">
<tr>
<th scope="col">Source</th>
{% for test in tests %}
<th class="text-center" scope="col" style="min-width: 5em;">{{ test }}</th>
{% endfor %}
</tr>
</thead>
{% for name, cases in results %}
<tr>
<th scope="row">{{ name }}</th>
{% for test in tests %}
{% set case = cases[test] %}
{% if case.failure == null %}
<td class="table-success text-center">
<i data-feather="check"></i>
</td>
{% else %}
{% if case.failure.type == 'java.lang.AssertionError' %}
<td class="table-warning text-center" data-bs-target="#failure_{{ case.hashCode }}"
data-bs-toggle="modal" style="cursor: pointer;">
<i data-feather="alert-triangle"></i>
</td>
{% elseif case.failure.type == 'java.net.SocketTimeoutException' or case.failure.type ==
'java.net.UnknownHostException' %}
<td class="table-secondary text-center" data-bs-target="#failure_{{ case.hashCode }}"
data-bs-toggle="modal" style="cursor: pointer;">
<i data-feather="power"></i>
</td>
{% elseif case.failure.type == 'org.koitharu.kotatsu.parsers.CloudFlareProtectedException' %}
<td class="table-secondary text-center" data-bs-target="#failure_{{ case.hashCode }}"
data-bs-toggle="modal" style="cursor: pointer;">
<i data-feather="shield"></i>
</td>
{% elseif case.failure.type == 'org.koitharu.kotatsu.parsers.exception.AuthRequiredException' %}
<td class="table-secondary text-center" data-bs-target="#failure_{{ case.hashCode }}"
data-bs-toggle="modal" style="cursor: pointer;">
<i data-feather="user-x"></i>
</td>
{% else %}
<td class="table-danger text-center" data-bs-target="#failure_{{ case.hashCode }}"
data-bs-toggle="modal" style="cursor: pointer;">
<i data-feather="x"></i>
</td>
{% endif %}
<!--suppress HtmlUnknownTag -->
<div class="modal fade" id="failure_{{ case.hashCode }}" tabindex="-1">
<div class="modal-dialog modal-lg modal-dialog-scrollable">
<div class="modal-content">
<div class="modal-header">
<h5 class="modal-title">{{ case.testName }} failed</h5>
<button aria-label="Close" class="btn-close" data-bs-dismiss="modal"
type="button"></button>
</div>
<div class="modal-body font-monospace lh-sm bg-light" style="font-size: 0.7em;">
{{ case.failure.textHtml()|raw }}
</div>
<div class="modal-footer">
<button class="btn btn-secondary" data-bs-dismiss="modal" type="button">Close</button>
<div class="table-responsive mt-4">
<table class="table table-hover">
<thead>
<tr>
<th scope="col">Source</th>
{% for test in tests %}
<th scope="col" style="min-width: 5em;">{{ test }}</th>
{% endfor %}
</tr>
</thead>
{% for name, cases in results %}
<tr>
<th scope="row">{{ name }}</th>
{% for test in tests %}
{% set case = cases[test] %}
{% if case.failure == null %}
<td class="table-success text-center">
<i data-feather="check"></i>
</td>
{% else %}
{% if case.failure.type == 'java.lang.AssertionError' %}
<td class="table-warning text-center" style="cursor: pointer;"
data-bs-toggle="modal" data-bs-target="#failure_{{ case.hashCode }}">
<i data-feather="alert-triangle"></i>
</td>
{% else %}
<td class="table-danger text-center" style="cursor: pointer;"
data-bs-toggle="modal" data-bs-target="#failure_{{ case.hashCode }}">
<i data-feather="x"></i>
</td>
{% endif %}
<!--suppress HtmlUnknownTag -->
<div class="modal fade" id="failure_{{ case.hashCode }}" tabindex="-1">
<div class="modal-dialog modal-lg modal-dialog-scrollable">
<div class="modal-content">
<div class="modal-header">
<h5 class="modal-title">{{ case.testName }} failed</h5>
<button type="button" class="btn-close" data-bs-dismiss="modal"
aria-label="Close"></button>
</div>
<div class="modal-body font-monospace lh-sm bg-light" style="font-size: 0.7em;">
{{ case.failure.textHtml()|raw }}
</div>
<div class="modal-footer">
<button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Close</button>
</div>
</div>
</div>
</div>
</div>
{% endif %}
{% endif %}
{% endfor %}
</tr>
{% endfor %}
</tr>
{% endfor %}
</table>
</table>
</div>
</div>
<script>
feather.replace()
</script>
</body>
</html>
</html>

1
gradle.properties
View File

@@ -1,2 +1 @@
kotlin.code.style=official
org.gradle.jvmargs=-Xmx4g -XX:MaxMetaspaceSize=4096m

2
gradle/wrapper/gradle-wrapper.properties vendored
View File

@@ -1,5 +1,5 @@
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-8.2-bin.zip
distributionUrl=https\://services.gradle.org/distributions/gradle-7.3.3-bin.zip
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists

8
kotatsu-parsers-ksp/build.gradle
View File

@@ -2,10 +2,6 @@ plugins {
id 'org.jetbrains.kotlin.jvm'
}
kotlin {
jvmToolchain(8)
}
dependencies {
implementation 'com.google.devtools.ksp:symbol-processing-api:1.9.22-1.0.17'
}
implementation 'com.google.devtools.ksp:symbol-processing-api:1.6.21-1.0.5'
}

23
kotatsu-parsers-ksp/src/main/kotlin/org/koitharu/kotatsu/parsers/ksp/ParserProcessor.kt
View File

@@ -70,13 +70,10 @@ class ParserProcessor(
import org.koitharu.kotatsu.parsers.model.MangaSource
@Suppress("DEPRECATION")
@InternalParsersApi
@Deprecated("", replaceWith = ReplaceWith("context.newParserInstance(this)"))
fun MangaSource.newParser(context: MangaLoaderContext): MangaParser = when (this) {
""".trimIndent(),
)
//language=kotlin
sourcesWriter?.write(
"""
package org.koitharu.kotatsu.parsers.model
@@ -84,9 +81,8 @@ class ParserProcessor(
enum class MangaSource(
val title: String,
val locale: String?,
val contentType: ContentType,
) {
LOCAL("Local", null, ContentType.OTHER),
LOCAL("Local", null),
""".trimIndent(),
)
@@ -109,7 +105,7 @@ class ParserProcessor(
)
sourcesWriter?.write(
"""
DUMMY("Dummy", null, ContentType.OTHER),
DUMMY("Dummy", null),
;
}
""".trimIndent(),
@@ -121,8 +117,6 @@ class ParserProcessor(
private val factoryWriter: Writer?,
) : KSVisitorVoid() {
private val titles = HashMap<String, String>()
override fun visitClassDeclaration(classDeclaration: KSClassDeclaration, data: Unit) {
if (classDeclaration.classKind != ClassKind.CLASS || classDeclaration.isAbstract()) {
logger.error("Only non-abstract can be annotated with @MangaSourceParser", classDeclaration)
@@ -132,7 +126,6 @@ class ParserProcessor(
val name = annotation.arguments.single { it.name?.asString() == "name" }.value as String
val title = annotation.arguments.single { it.name?.asString() == "title" }.value as String
val locale = annotation.arguments.single { it.name?.asString() == "locale" }.value as String
val type = annotation.arguments.single { it.name?.asString() == "type" }.value
val localeString = if (locale.isEmpty()) "null" else "\"$locale\""
val localeObj = if (locale.isEmpty()) null else Locale(locale)
val localeTitle = localeObj?.getDisplayLanguage(localeObj)
@@ -151,13 +144,7 @@ class ParserProcessor(
classDeclaration,
)
}
val className = checkNotNull(classDeclaration.qualifiedName?.asString()) { "Class name is null" }
val prevTitleClass = titles.put(title, className)
if (prevTitleClass != null) {
logger.warn("Source title duplication: \"$title\" is assigned to both $prevTitleClass and $className")
}
val className = classDeclaration.qualifiedName?.asString()
factoryWriter?.write("\tMangaSource.$name -> $className(context)\n")
val deprecationString = if (deprecation != null) {
val reason = deprecation.arguments
@@ -165,7 +152,7 @@ class ParserProcessor(
"@Deprecated(\"$reason\") "
} else ""
val localeComment = localeTitle?.toTitleCase(localeObj)?.let { " /* $it */" }.orEmpty()
sourcesWriter?.write("\t$deprecationString$name(\"$title\", $localeString$localeComment, ContentType.$type),\n")
sourcesWriter?.write("\t$deprecationString$name(\"$title\", $localeString$localeComment),\n")
}
}
}
}

9
settings.gradle
View File

@@ -1,16 +1,19 @@
pluginManagement {
plugins {
id 'com.google.devtools.ksp' version '1.6.21-1.0.5'
id 'org.jetbrains.kotlin.jvm' version '1.6.21'
}
repositories {
google()
mavenCentral()
gradlePluginPortal()
google()
}
}
dependencyResolutionManagement {
repositoriesMode.set(RepositoriesMode.FAIL_ON_PROJECT_REPOS)
repositories {
google()
mavenCentral()
gradlePluginPortal()
}
}

14
src/main/kotlin/org/koitharu/kotatsu/parsers/ErrorMessages.kt
View File

@@ -1,14 +0,0 @@
package org.koitharu.kotatsu.parsers
object ErrorMessages {
const val FILTER_MULTIPLE_STATES_NOT_SUPPORTED = "Multiple states are not supported by this source"
const val FILTER_MULTIPLE_GENRES_NOT_SUPPORTED = "Multiple genres are not supported by this source"
const val FILTER_MULTIPLE_CONTENT_RATING_NOT_SUPPORTED =
"Multiple Content Rating are not supported by this source"
const val FILTER_BOTH_LOCALE_GENRES_NOT_SUPPORTED =
"Filtering by both genres and locale is not supported by this source"
const val FILTER_BOTH_STATES_GENRES_NOT_SUPPORTED =
"Filtering by both genres and states is not supported by this source"
const val SEARCH_NOT_SUPPORTED = "Search is not supported by this source"
}

2
src/main/kotlin/org/koitharu/kotatsu/parsers/InternalParsersApi.kt
View File

@@ -11,4 +11,4 @@ package org.koitharu.kotatsu.parsers
@SinceKotlin("1.3")
@RequiresOptIn
@MustBeDocumented
annotation class InternalParsersApi
annotation class InternalParsersApi()

146
src/main/kotlin/org/koitharu/kotatsu/parsers/MangaLoaderContext.kt
View File

@@ -1,19 +1,136 @@
package org.koitharu.kotatsu.parsers
import okhttp3.CookieJar
import okhttp3.OkHttpClient
import okhttp3.*
import okhttp3.HttpUrl.Companion.toHttpUrl
import okhttp3.MediaType.Companion.toMediaType
import okhttp3.RequestBody.Companion.toRequestBody
import org.json.JSONObject
import org.jsoup.HttpStatusException
import org.koitharu.kotatsu.parsers.config.MangaSourceConfig
import org.koitharu.kotatsu.parsers.exception.GraphQLException
import org.koitharu.kotatsu.parsers.exception.NotFoundException
import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.util.await
import org.koitharu.kotatsu.parsers.util.parseJson
import java.util.*
abstract class MangaLoaderContext {
abstract val httpClient: OkHttpClient
protected abstract val httpClient: OkHttpClient
abstract val cookieJar: CookieJar
@Suppress("DEPRECATION")
fun newParserInstance(source: MangaSource): MangaParser = source.newParser(this)
/**
* Do a GET http request to specific url
* @param url
* @param headers an additional headers for request, may be null
*/
suspend fun httpGet(url: HttpUrl, headers: Headers? = null): Response {
val request = Request.Builder()
.get()
.url(url)
if (headers != null) {
request.headers(headers)
}
return httpClient.newCall(request.build()).await().ensureSuccess()
}
suspend fun httpGet(url: String, headers: Headers? = null): Response {
return httpGet(url.toHttpUrl(), headers)
}
/**
* Do a HEAD http request to specific url
* @param url
* @param headers an additional headers for request, may be null
*/
suspend fun httpHead(url: String, headers: Headers? = null): Response {
val request = Request.Builder()
.head()
.url(url)
if (headers != null) {
request.headers(headers)
}
return httpClient.newCall(request.build()).await().ensureSuccess()
}
/**
* Do a POST http request to specific url with `multipart/form-data` payload
* @param url
* @param form payload as key=>value map
* @param headers an additional headers for request, may be null
*/
suspend fun httpPost(
url: String,
form: Map<String, String>,
headers: Headers? = null,
): Response {
val body = FormBody.Builder()
form.forEach { (k, v) ->
body.addEncoded(k, v)
}
val request = Request.Builder()
.post(body.build())
.url(url)
if (headers != null) {
request.headers(headers)
}
return httpClient.newCall(request.build()).await().ensureSuccess()
}
/**
* Do a POST http request to specific url with `multipart/form-data` payload
* @param url
* @param payload payload as `key=value` string with `&` separator
* @param headers an additional headers for request, may be null
*/
suspend fun httpPost(
url: String,
payload: String,
headers: Headers?,
): Response {
val body = FormBody.Builder()
payload.split('&').forEach {
val pos = it.indexOf('=')
if (pos != -1) {
val k = it.substring(0, pos)
val v = it.substring(pos + 1)
body.addEncoded(k, v)
}
}
val request = Request.Builder()
.post(body.build())
.url(url)
if (headers != null) {
request.headers(headers)
}
return httpClient.newCall(request.build()).await().ensureSuccess()
}
/**
* Do a GraphQL request to specific url
* @param endpoint an url
* @param query GraphQL request payload
*/
suspend fun graphQLQuery(endpoint: String, query: String): JSONObject {
val body = JSONObject()
body.put("operationName", null as Any?)
body.put("variables", JSONObject())
body.put("query", "{$query}")
val mediaType = "application/json; charset=utf-8".toMediaType()
val requestBody = body.toString().toRequestBody(mediaType)
val request = Request.Builder()
.post(requestBody)
.url(endpoint)
val json = httpClient.newCall(request.build()).await().ensureSuccess().parseJson()
json.optJSONArray("errors")?.let {
if (it.length() != 0) {
throw GraphQLException(it)
}
}
return json
}
open fun encodeBase64(data: ByteArray): String = Base64.getEncoder().encodeToString(data)
@@ -29,4 +146,21 @@ abstract class MangaLoaderContext {
abstract suspend fun evaluateJs(script: String): String?
abstract fun getConfig(source: MangaSource): MangaSourceConfig
}
private fun Response.ensureSuccess(): Response {
val exception: Exception? = when (code) { // Catch some error codes, not all
404 -> NotFoundException(message, request.url.toString())
in 500..599 -> HttpStatusException(message, code, request.url.toString())
else -> null
}
if (exception != null) {
runCatching {
close()
}.onFailure {
exception.addSuppressed(it)
}
throw exception
}
return this
}
}

245
src/main/kotlin/org/koitharu/kotatsu/parsers/MangaParser.kt
View File

@@ -3,117 +3,67 @@ package org.koitharu.kotatsu.parsers
import androidx.annotation.CallSuper
import androidx.annotation.VisibleForTesting
import okhttp3.Headers
import okhttp3.HttpUrl
import org.jsoup.nodes.Element
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.exception.ParseException
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.network.OkHttpWebClient
import org.koitharu.kotatsu.parsers.network.UserAgents
import org.koitharu.kotatsu.parsers.network.WebClient
import org.koitharu.kotatsu.parsers.util.FaviconParser
import org.koitharu.kotatsu.parsers.util.RelatedMangaFinder
import org.koitharu.kotatsu.parsers.util.domain
import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl
import java.util.*
abstract class MangaParser @InternalParsersApi constructor(
@property:InternalParsersApi val context: MangaLoaderContext,
val source: MangaSource,
) {
abstract class MangaParser @InternalParsersApi constructor(val source: MangaSource) {
protected abstract val context: MangaLoaderContext
/**
* Supported [SortOrder] variants. Must not be empty.
*
* For better performance use [EnumSet] for more than one item.
*/
abstract val availableSortOrders: Set<SortOrder>
/**
* Supported [MangaState] variants for filtering. May be empty.
*
* For better performance use [EnumSet] for more than one item.
*/
open val availableStates: Set<MangaState>
get() = emptySet()
open val availableContentRating: Set<ContentRating>
get() = emptySet()
/**
* Whether parser supports filtering by more than one tag
*/
open val isMultipleTagsSupported: Boolean = true
/**
* Whether parser supports tagsExclude field in filter
*/
open val isTagsExclusionSupported: Boolean = false
/**
* Whether parser supports searching by string query using [MangaListFilter.Search]
*/
open val isSearchSupported: Boolean = true
@Deprecated(
message = "Use availableSortOrders instead",
replaceWith = ReplaceWith("availableSortOrders"),
)
open val sortOrders: Set<SortOrder>
get() = availableSortOrders
abstract val sortOrders: Set<SortOrder>
val config by lazy { context.getConfig(source) }
open val sourceLocale: Locale
get() = source.locale?.let { Locale(it) } ?: Locale.ROOT
val isNsfwSource = source.contentType == ContentType.HENTAI
val sourceLocale: Locale?
get() = source.locale?.let { Locale(it) }
/**
* Provide default domain and available alternatives, if any.
*
* Never hardcode domain in requests, use [domain] instead.
* Never hardcode domain in requests, use [getDomain] instead.
*/
@InternalParsersApi
abstract val configKeyDomain: ConfigKey.Domain
protected abstract val configKeyDomain: ConfigKey.Domain
open val headers: Headers = Headers.Builder()
.add("User-Agent", UserAgents.CHROME_MOBILE)
.build()
@VisibleForTesting(otherwise = VisibleForTesting.PROTECTED)
internal open val headers: Headers? = null
/**
* Used as fallback if value of `sortOrder` passed to [getList] is null
*/
@VisibleForTesting(otherwise = VisibleForTesting.PROTECTED)
open val defaultSortOrder: SortOrder
protected open val defaultSortOrder: SortOrder
get() {
val supported = availableSortOrders
return SortOrder.entries.first { it in supported }
val supported = sortOrders
return SortOrder.values().first { it in supported }
}
@JvmField
protected val webClient: WebClient = OkHttpWebClient(context.httpClient, source)
/**
* Parse list of manga by specified criteria
*
* @param offset starting from 0 and used for pagination.
* Note than passed value may not be divisible by internal page size, so you should adjust it manually.
* @param query search query, may be null or empty if no search needed
* @param tags genres for filtering, values from [getAvailableTags] and [Manga.tags]. May be null or empty
* @param sortOrder one of [availableSortOrders] or null for default value
* @param tags genres for filtering, values from [getTags] and [Manga.tags]. May be null or empty
* @param sortOrder one of [sortOrders] or null for default value
*/
@JvmSynthetic
@InternalParsersApi
@Deprecated(
"Use getList with filter instead",
replaceWith = ReplaceWith("getList(offset, filter)"),
)
open suspend fun getList(
abstract suspend fun getList(
offset: Int,
query: String?,
tags: Set<MangaTag>?,
tagsExclude: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> = throw NotImplementedError("Please implement getList(offset, filter) instead")
): List<Manga>
/**
* Parse list of manga with search by text query
@@ -121,15 +71,8 @@ abstract class MangaParser @InternalParsersApi constructor(
* @param offset starting from 0 and used for pagination.
* @param query search query
*/
@Deprecated(
"Use getList with filter instead",
ReplaceWith(
"getList(offset, MangaListFilter.Search(query))",
"org.koitharu.kotatsu.parsers.model.MangaListFilter",
),
)
open suspend fun getList(offset: Int, query: String): List<Manga> {
return getList(offset, MangaListFilter.Search(query))
return getList(offset, query, null, defaultSortOrder)
}
/**
@@ -137,62 +80,11 @@ abstract class MangaParser @InternalParsersApi constructor(
*
* @param offset starting from 0 and used for pagination.
* Note than passed value may not be divisible by internal page size, so you should adjust it manually.
* @param tags genres for filtering, values from [getAvailableTags] and [Manga.tags]. May be null or empty
* @param sortOrder one of [availableSortOrders] or null for default value
* @param tags genres for filtering, values from [getTags] and [Manga.tags]. May be null or empty
* @param sortOrder one of [sortOrders] or null for default value
*/
@Deprecated(
"Use getList with filter instead",
ReplaceWith(
"getList(offset, MangaListFilter.Advanced(sortOrder, tags, null, emptySet()))",
"org.koitharu.kotatsu.parsers.model.MangaListFilter",
),
)
open suspend fun getList(
offset: Int,
tags: Set<MangaTag>?,
tagsExclude: Set<MangaTag>?,
sortOrder: SortOrder?,
): List<Manga> {
return getList(
offset,
MangaListFilter.Advanced(
sortOrder = sortOrder ?: defaultSortOrder,
tags = tags.orEmpty(),
tagsExclude = tagsExclude.orEmpty(),
locale = null,
states = emptySet(),
contentRating = emptySet(),
),
)
}
@Suppress("DEPRECATION")
open suspend fun getList(offset: Int, filter: MangaListFilter?): List<Manga> {
return when (filter) {
is MangaListFilter.Advanced -> getList(
offset = offset,
query = null,
tags = filter.tags,
tagsExclude = filter.tagsExclude,
sortOrder = filter.sortOrder,
)
is MangaListFilter.Search -> getList(
offset = offset,
query = filter.query,
tags = null,
tagsExclude = null,
sortOrder = defaultSortOrder,
)
null -> getList(
offset = offset,
query = null,
tags = null,
tagsExclude = null,
sortOrder = defaultSortOrder,
)
}
open suspend fun getList(offset: Int, tags: Set<MangaTag>?, sortOrder: SortOrder?): List<Manga> {
return getList(offset, null, tags, sortOrder ?: defaultSortOrder)
}
/**
@@ -211,29 +103,27 @@ abstract class MangaParser @InternalParsersApi constructor(
/**
* Fetch direct link to the page image.
*/
open suspend fun getPageUrl(page: MangaPage): String = page.url.toAbsoluteUrl(domain)
open suspend fun getPageUrl(page: MangaPage): String = page.url.toAbsoluteUrl(getDomain())
/**
* Fetch available tags (genres) for source
*/
abstract suspend fun getAvailableTags(): Set<MangaTag>
abstract suspend fun getTags(): Set<MangaTag>
/**
* Fetch available locales for multilingual sources
* Returns direct link to the website favicon
*/
open suspend fun getAvailableLocales(): Set<Locale> = emptySet()
@Deprecated(
message = "Use getAvailableTags instead",
replaceWith = ReplaceWith("getAvailableTags()"),
message = "Use parseFavicons() to get multiple favicons with different size",
replaceWith = ReplaceWith("parseFavicons()"),
)
suspend fun getTags(): Set<MangaTag> = getAvailableTags()
open fun getFaviconUrl() = "https://${getDomain()}/favicon.ico"
/**
* Parse favicons from the main page of the source`s website
*/
open suspend fun getFavicons(): Favicons {
return FaviconParser(webClient, domain).parseFavicons()
return FaviconParser(context, getDomain(), headers).parseFavicons()
}
@CallSuper
@@ -241,13 +131,70 @@ abstract class MangaParser @InternalParsersApi constructor(
keys.add(configKeyDomain)
}
open suspend fun getRelatedManga(seed: Manga): List<Manga> {
return RelatedMangaFinder(listOf(this)).invoke(seed)
/* Utils */
fun getDomain(): String {
return config[configKeyDomain]
}
protected fun getParser(source: MangaSource) = if (this.source == source) {
this
} else {
context.newParserInstance(source)
fun getDomain(subdomain: String): String {
val domain = getDomain()
return subdomain + "." + domain.removePrefix("www.")
}
}
fun urlBuilder(): HttpUrl.Builder {
return HttpUrl.Builder()
.scheme("https")
.host(getDomain())
}
/**
* Create a unique id for [Manga]/[MangaChapter]/[MangaPage].
* @param url must be relative url, without a domain
* @see [Manga.id]
* @see [MangaChapter.id]
* @see [MangaPage.id]
*/
@InternalParsersApi
protected fun generateUid(url: String): Long {
var h = 1125899906842597L
source.name.forEach { c ->
h = 31 * h + c.code
}
url.forEach { c ->
h = 31 * h + c.code
}
return h
}
/**
* Create a unique id for [Manga]/[MangaChapter]/[MangaPage].
* @param id an internal identifier
* @see [Manga.id]
* @see [MangaChapter.id]
* @see [MangaPage.id]
*/
@InternalParsersApi
protected fun generateUid(id: Long): Long {
var h = 1125899906842597L
source.name.forEach { c ->
h = 31 * h + c.code
}
h = 31 * h + id
return h
}
@InternalParsersApi
protected fun Element.parseFailed(message: String? = null): Nothing {
throw ParseException(message, ownerDocument()?.location() ?: baseUri(), null)
}
@InternalParsersApi
protected fun Set<MangaTag>?.oneOrThrowIfMany(): MangaTag? {
return when {
isNullOrEmpty() -> null
size == 1 -> first()
else -> throw IllegalArgumentException("Multiple genres are not supported by this source")
}
}
}

9
src/main/kotlin/org/koitharu/kotatsu/parsers/MangaSourceParser.kt
View File

@@ -1,7 +1,5 @@
package org.koitharu.kotatsu.parsers
import org.koitharu.kotatsu.parsers.model.ContentType
/**
* Annotate each [MangaParser] implementation with this annotation, used by codegen
*/
@@ -13,15 +11,10 @@ annotation class MangaSourceParser(
val name: String,
/**
* User-friendly title of manga source. In most case equals the website name.
* Avoid extra whitespaces between the words if it is not required.
*/
val title: String,
/**
* Language code (for example "en" or "ru") or blank if parser provide manga on different languages.
*/
val locale: String = "",
/**
* Type of content provided by parser. See [ContentType] for more info
*/
val type: ContentType = ContentType.MANGA,
)
)

75
src/main/kotlin/org/koitharu/kotatsu/parsers/PagedMangaParser.kt
View File

@@ -1,33 +1,27 @@
package org.koitharu.kotatsu.parsers
import androidx.annotation.RestrictTo
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.model.Manga
import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.model.MangaTag
import org.koitharu.kotatsu.parsers.model.SortOrder
import org.koitharu.kotatsu.parsers.util.Paginator
@InternalParsersApi
abstract class PagedMangaParser(
context: MangaLoaderContext,
source: MangaSource,
@RestrictTo(RestrictTo.Scope.TESTS) @JvmField internal val pageSize: Int,
pageSize: Int,
searchPageSize: Int = pageSize,
) : MangaParser(context, source) {
) : MangaParser(source) {
@JvmField
protected val paginator = Paginator(pageSize)
@JvmField
protected val searchPaginator = Paginator(searchPageSize)
final override suspend fun getList(offset: Int, filter: MangaListFilter?): List<Manga> {
return getList(
paginator = if (filter is MangaListFilter.Search) {
searchPaginator
} else {
paginator
},
offset = offset,
filter = filter,
)
override suspend fun getList(offset: Int, query: String): List<Manga> {
return getList(searchPaginator, offset, query, null, defaultSortOrder)
}
override suspend fun getList(offset: Int, tags: Set<MangaTag>?, sortOrder: SortOrder?): List<Manga> {
return getList(paginator, offset, null, tags, sortOrder ?: defaultSortOrder)
}
@InternalParsersApi
@@ -36,54 +30,21 @@ abstract class PagedMangaParser(
offset: Int,
query: String?,
tags: Set<MangaTag>?,
tagsExclude: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> = throw UnsupportedOperationException("You should use getListPage for PagedMangaParser")
open suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
tagsExclude: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> = throw NotImplementedError("Please implement getListPage(page, filter) instead")
open suspend fun getListPage(page: Int, filter: MangaListFilter?): List<Manga> {
return when (filter) {
is MangaListFilter.Advanced -> getListPage(
page = page,
query = null,
tags = filter.tags,
tagsExclude = filter.tagsExclude,
sortOrder = filter.sortOrder,
)
is MangaListFilter.Search -> getListPage(
page = page,
query = filter.query,
tags = null,
tagsExclude = null,
sortOrder = defaultSortOrder,
)
null -> getListPage(
page = page,
query = null,
tags = null,
tagsExclude = null,
sortOrder = defaultSortOrder,
)
}
}
abstract suspend fun getListPage(page: Int, query: String?, tags: Set<MangaTag>?, sortOrder: SortOrder): List<Manga>
private suspend fun getList(
paginator: Paginator,
offset: Int,
filter: MangaListFilter?,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val page = paginator.getPage(offset)
val list = getListPage(page, filter)
val list = getListPage(page, query, tags, sortOrder)
paginator.onListReceived(offset, page, list.size)
return list
}
}
}

23
src/main/kotlin/org/koitharu/kotatsu/parsers/config/ConfigKey.kt
View File

@@ -1,28 +1,13 @@
package org.koitharu.kotatsu.parsers.config
sealed class ConfigKey<T>(
@JvmField val key: String,
val key: String,
) {
abstract val defaultValue: T
class Domain(
@JvmField @JvmSuppressWildcards vararg val presetValues: String,
) : ConfigKey<String>("domain") {
init {
require(presetValues.isNotEmpty()) { "You must provide at least one domain" }
}
override val defaultValue: String
get() = presetValues.first()
}
class ShowSuspiciousContent(
override val defaultValue: Boolean,
) : ConfigKey<Boolean>("show_suspicious")
class UserAgent(
override val defaultValue: String,
) : ConfigKey<String>("user_agent")
}
val presetValues: Array<String>?,
) : ConfigKey<String>("domain")
}

5
src/main/kotlin/org/koitharu/kotatsu/parsers/exception/AuthRequiredException.kt
View File

@@ -6,7 +6,6 @@ import org.koitharu.kotatsu.parsers.model.MangaSource
/**
* Authorization is required for access to the requested content
*/
class AuthRequiredException @InternalParsersApi @JvmOverloads constructor(
class AuthRequiredException @InternalParsersApi constructor(
val source: MangaSource,
cause: Throwable? = null,
) : RuntimeException("Authorization required", cause)
) : RuntimeException("Authorization required")

7
src/main/kotlin/org/koitharu/kotatsu/parsers/exception/CloudFlareProtectedException.kt Normal file
View File

@@ -0,0 +1,7 @@
package org.koitharu.kotatsu.parsers.exception
import okio.IOException
class CloudFlareProtectedException(
val url: String,
) : IOException("Protected by CloudFlare: $url")

7
src/main/kotlin/org/koitharu/kotatsu/parsers/model/ContentRating.kt
View File

@@ -1,7 +0,0 @@
package org.koitharu.kotatsu.parsers.model
enum class ContentRating {
SAFE,
SUGGESTIVE,
ADULT
}

24
src/main/kotlin/org/koitharu/kotatsu/parsers/model/ContentType.kt
View File

@@ -1,24 +0,0 @@
package org.koitharu.kotatsu.parsers.model
enum class ContentType {
/**
* Standard manga, manhua, webtoons, etc
*/
MANGA,
/**
* Use this if the source provides mostly nsfw content.
*/
HENTAI,
/**
* Western comics
*/
COMICS,
/**
* Use this type if no other suits your needs. For example, for an indie manga
*/
OTHER,
}

9
src/main/kotlin/org/koitharu/kotatsu/parsers/model/Favicon.kt
View File

@@ -3,12 +3,11 @@ package org.koitharu.kotatsu.parsers.model
import okhttp3.HttpUrl.Companion.toHttpUrl
class Favicon internal constructor(
@JvmField val url: String,
@JvmField val size: Int,
@JvmField internal val rel: String?,
val url: String,
val size: Int,
internal val rel: String?,
) : Comparable<Favicon> {
@JvmField
val type: String = url.toHttpUrl().pathSegments.last()
.substringAfterLast('.', "").lowercase()
@@ -49,4 +48,4 @@ class Favicon internal constructor(
"mask-icon" -> -1
else -> 0
}
}
}

9
src/main/kotlin/org/koitharu/kotatsu/parsers/model/Favicons.kt
View File

@@ -2,7 +2,7 @@ package org.koitharu.kotatsu.parsers.model
class Favicons internal constructor(
favicons: Collection<Favicon>,
@JvmField val referer: String,
val referer: String,
) : Collection<Favicon> {
private val icons = favicons.sortedDescending()
@@ -18,11 +18,6 @@ class Favicons internal constructor(
override fun iterator(): Iterator<Favicon> = icons.iterator()
operator fun minus(victim: Favicon): Favicons = Favicons(
favicons = icons.filterNot { it == victim },
referer = referer,
)
/**
* Finds a favicon whose size in pixels is greater than or equal to the specified size.
* If such icon is not available returns the largest icon
@@ -47,4 +42,4 @@ class Favicons internal constructor(
}
return result
}
}
}

36
src/main/kotlin/org/koitharu/kotatsu/parsers/model/Manga.kt
View File

@@ -6,67 +6,67 @@ class Manga(
/**
* Unique identifier for manga
*/
@JvmField val id: Long,
val id: Long,
/**
* Manga title, human-readable
*/
@JvmField val title: String,
val title: String,
/**
* Alternative title (for example on other language), may be null
*/
@JvmField val altTitle: String?,
val altTitle: String?,
/**
* Relative url to manga (**without** a domain) or any other uri.
* Used principally in parsers
*/
@JvmField val url: String,
val url: String,
/**
* Absolute url to manga, must be ready to open in browser
*/
@JvmField val publicUrl: String,
val publicUrl: String,
/**
* Normalized manga rating, must be in range of 0..1 or [RATING_UNKNOWN] if rating s unknown
* @see hasRating
*/
@JvmField val rating: Float,
val rating: Float,
/**
* Indicates that manga may contain sensitive information (18+, NSFW)
*/
@JvmField val isNsfw: Boolean,
val isNsfw: Boolean,
/**
* Absolute link to the cover
* @see largeCoverUrl
*/
@JvmField val coverUrl: String,
val coverUrl: String,
/**
* Tags (genres) of the manga
*/
@JvmField val tags: Set<MangaTag>,
val tags: Set<MangaTag>,
/**
* Manga status (ongoing, finished) or null if unknown
*/
@JvmField val state: MangaState?,
val state: MangaState?,
/**
* Author of the manga, may be null
*/
@JvmField val author: String?,
val author: String?,
/**
* Large cover url (absolute), null if is no large cover
* @see coverUrl
*/
@JvmField val largeCoverUrl: String? = null,
val largeCoverUrl: String? = null,
/**
* Manga description, may be html or null
*/
@JvmField val description: String? = null,
val description: String? = null,
/**
* List of chapters
*/
@JvmField val chapters: List<MangaChapter>? = null,
val chapters: List<MangaChapter>? = null,
/**
* Manga source
*/
@JvmField val source: MangaSource,
val source: MangaSource,
) {
/**
@@ -155,8 +155,4 @@ class Manga(
result = 31 * result + source.hashCode()
return result
}
override fun toString(): String {
return "Manga($id - \"$title\" [$url] - $source)"
}
}
}

70
src/main/kotlin/org/koitharu/kotatsu/parsers/model/MangaChapter.kt
View File

@@ -4,61 +4,39 @@ class MangaChapter(
/**
* An unique id of chapter
*/
@JvmField val id: Long,
val id: Long,
/**
* User-readable name of chapter
*/
@JvmField val name: String,
val name: String,
/**
* Chapter number starting from 1, 0 if unknown
* Chapter number starting from 1
*/
@JvmField val number: Float,
/**
* Volume number starting from 1, 0 if unknown
*/
@JvmField val volume: Int,
val number: Int,
/**
* Relative url to chapter (**without** a domain) or any other uri.
* Used principally in parsers
*/
@JvmField val url: String,
val url: String,
/**
* User-readable name of scanlator (releaser) or null if unknown
*/
@JvmField val scanlator: String?,
val scanlator: String?,
/**
* Chapter upload date in milliseconds
*/
@JvmField val uploadDate: Long,
val uploadDate: Long,
/**
* User-readable name of branch.
* A branch is a group of chapters that overlap (e.g. different languages)
*/
@JvmField val branch: String?,
@JvmField val source: MangaSource,
) {
val branch: String?,
val source: MangaSource,
) : Comparable<MangaChapter> {
@Deprecated(message = "Consider using constructor with volume value")
constructor(
id: Long,
name: String,
number: Int,
url: String,
scanlator: String?,
uploadDate: Long,
branch: String?,
source: MangaSource,
) : this(
id = id,
name = name,
number = number.toFloat(),
volume = 0,
url = url,
scanlator = scanlator,
uploadDate = uploadDate,
branch = branch,
source = source,
)
override fun compareTo(other: MangaChapter): Int {
return number.compareTo(other.number)
}
override fun equals(other: Any?): Boolean {
if (this === other) return true
@@ -69,7 +47,6 @@ class MangaChapter(
if (id != other.id) return false
if (name != other.name) return false
if (number != other.number) return false
if (volume != other.volume) return false
if (url != other.url) return false
if (scanlator != other.scanlator) return false
if (uploadDate != other.uploadDate) return false
@@ -82,8 +59,7 @@ class MangaChapter(
override fun hashCode(): Int {
var result = id.hashCode()
result = 31 * result + name.hashCode()
result = 31 * result + number.hashCode()
result = 31 * result + volume
result = 31 * result + number
result = 31 * result + url.hashCode()
result = 31 * result + (scanlator?.hashCode() ?: 0)
result = 31 * result + uploadDate.hashCode()
@@ -91,20 +67,4 @@ class MangaChapter(
result = 31 * result + source.hashCode()
return result
}
override fun toString(): String {
return "MangaChapter($id - #$number [$url] - $source)"
}
internal fun copy(volume: Int, number: Float) = MangaChapter(
id = id,
name = name,
number = number,
volume = volume,
url = url,
scanlator = scanlator,
uploadDate = uploadDate,
branch = branch,
source = source,
)
}
}

93
src/main/kotlin/org/koitharu/kotatsu/parsers/model/MangaListFilter.kt
View File

@@ -1,93 +0,0 @@
package org.koitharu.kotatsu.parsers.model
import org.koitharu.kotatsu.parsers.MangaParser
import java.util.*
sealed interface MangaListFilter {
fun isEmpty(): Boolean
val sortOrder: SortOrder?
fun isValid(parser: MangaParser): Boolean = when (this) {
is Advanced -> (sortOrder in parser.availableSortOrders) &&
(tags.size <= 1 || parser.isMultipleTagsSupported) &&
(tagsExclude.isEmpty() || parser.isTagsExclusionSupported) &&
(contentRating.isEmpty() || parser.availableContentRating.containsAll(contentRating)) &&
(states.isEmpty() || parser.availableStates.containsAll(states))
is Search -> parser.isSearchSupported
}
data class Search(
@JvmField val query: String,
) : MangaListFilter {
override val sortOrder: SortOrder? = null
override fun isEmpty() = query.isBlank()
}
data class Advanced(
override val sortOrder: SortOrder,
@JvmField val tags: Set<MangaTag>,
@JvmField val tagsExclude: Set<MangaTag>,
@JvmField val locale: Locale?,
@JvmField val states: Set<MangaState>,
@JvmField val contentRating: Set<ContentRating>,
) : MangaListFilter {
override fun isEmpty(): Boolean =
tags.isEmpty() && tagsExclude.isEmpty() && locale == null && states.isEmpty() && contentRating.isEmpty()
fun newBuilder() = Builder(sortOrder)
.tags(tags)
.tagsExclude(tagsExclude)
.locale(locale)
.states(states)
.contentRatings(contentRating)
class Builder(sortOrder: SortOrder) {
private var _sortOrder: SortOrder = sortOrder
private var _tags: Set<MangaTag>? = null
private var _tagsExclude: Set<MangaTag>? = null
private var _locale: Locale? = null
private var _states: Set<MangaState>? = null
private var _contentRating: Set<ContentRating>? = null
fun sortOrder(order: SortOrder) = apply {
_sortOrder = order
}
fun tags(tags: Set<MangaTag>?) = apply {
_tags = tags
}
fun tagsExclude(tags: Set<MangaTag>?) = apply {
_tagsExclude = tags
}
fun locale(locale: Locale?) = apply {
_locale = locale
}
fun states(states: Set<MangaState>?) = apply {
_states = states
}
fun contentRatings(rating: Set<ContentRating>?) = apply {
_contentRating = rating
}
fun build() = Advanced(
sortOrder = _sortOrder,
tags = _tags.orEmpty(),
tagsExclude = _tagsExclude.orEmpty(),
locale = _locale,
states = _states.orEmpty(),
contentRating = _contentRating.orEmpty(),
)
}
}
}

25
src/main/kotlin/org/koitharu/kotatsu/parsers/model/MangaPage.kt
View File

@@ -6,19 +6,24 @@ class MangaPage(
/**
* Unique identifier for manga
*/
@JvmField val id: Long,
val id: Long,
/**
* Relative url to page (**without** a domain) or any other uri.
* Used principally in parsers.
* May contain link to image or html page.
* @see MangaParser.getPageUrl
*/
@JvmField val url: String,
val url: String,
/**
* Absolute link to the chapter or website home page.
* Used in Referer header
*/
val referer: String,
/**
* Absolute url of the small page image if exists, null otherwise
*/
@JvmField val preview: String?,
@JvmField val source: MangaSource,
val preview: String?,
val source: MangaSource,
) {
override fun equals(other: Any?): Boolean {
@@ -29,19 +34,19 @@ class MangaPage(
if (id != other.id) return false
if (url != other.url) return false
if (referer != other.referer) return false
if (preview != other.preview) return false
return source == other.source
if (source != other.source) return false
return true
}
override fun hashCode(): Int {
var result = id.hashCode()
result = 31 * result + url.hashCode()
result = 31 * result + referer.hashCode()
result = 31 * result + (preview?.hashCode() ?: 0)
result = 31 * result + source.hashCode()
return result
}
override fun toString(): String {
return "MangaPage($id [$url] - $source)"
}
}
}

4
src/main/kotlin/org/koitharu/kotatsu/parsers/model/MangaState.kt
View File

@@ -1,5 +1,5 @@
package org.koitharu.kotatsu.parsers.model
enum class MangaState {
ONGOING, FINISHED, ABANDONED, PAUSED, UPCOMING
}
ONGOING, FINISHED
}

12
src/main/kotlin/org/koitharu/kotatsu/parsers/model/MangaTag.kt
View File

@@ -6,13 +6,13 @@ class MangaTag(
/**
* User-readable tag title, should be in Title case
*/
@JvmField val title: String,
val title: String,
/**
* Identifier of a tag, must be unique among the source.
* @see MangaParser.getList
*/
@JvmField val key: String,
@JvmField val source: MangaSource,
val key: String,
val source: MangaSource,
) {
override fun equals(other: Any?): Boolean {
@@ -34,8 +34,4 @@ class MangaTag(
result = 31 * result + source.hashCode()
return result
}
override fun toString(): String {
return "MangaTag($key \"$title\" - $source)"
}
}
}

5
src/main/kotlin/org/koitharu/kotatsu/parsers/model/SortOrder.kt
View File

@@ -5,6 +5,5 @@ enum class SortOrder {
POPULARITY,
RATING,
NEWEST,
ALPHABETICAL,
ALPHABETICAL_DESC
}
ALPHABETICAL
}

133
src/main/kotlin/org/koitharu/kotatsu/parsers/network/OkHttpWebClient.kt
View File

@@ -1,133 +0,0 @@
package org.koitharu.kotatsu.parsers.network
import okhttp3.*
import okhttp3.MediaType.Companion.toMediaType
import okhttp3.RequestBody.Companion.toRequestBody
import org.json.JSONObject
import org.jsoup.HttpStatusException
import org.koitharu.kotatsu.parsers.exception.AuthRequiredException
import org.koitharu.kotatsu.parsers.exception.GraphQLException
import org.koitharu.kotatsu.parsers.exception.NotFoundException
import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.util.await
import org.koitharu.kotatsu.parsers.util.parseJson
import java.net.HttpURLConnection
class OkHttpWebClient(
private val httpClient: OkHttpClient,
private val mangaSource: MangaSource,
) : WebClient {
override suspend fun httpGet(url: HttpUrl, extraHeaders: Headers?): Response {
val request = Request.Builder()
.get()
.url(url)
.addTags()
.addExtraHeaders(extraHeaders)
return httpClient.newCall(request.build()).await().ensureSuccess()
}
override suspend fun httpHead(url: HttpUrl): Response {
val request = Request.Builder()
.head()
.url(url)
.addTags()
return httpClient.newCall(request.build()).await().ensureSuccess()
}
override suspend fun httpPost(url: HttpUrl, form: Map<String, String>, extraHeaders: Headers?): Response {
val body = FormBody.Builder()
form.forEach { (k, v) ->
body.addEncoded(k, v)
}
val request = Request.Builder()
.post(body.build())
.url(url)
.addTags()
.addExtraHeaders(extraHeaders)
return httpClient.newCall(request.build()).await().ensureSuccess()
}
override suspend fun httpPost(url: HttpUrl, payload: String, extraHeaders: Headers?): Response {
val body = FormBody.Builder()
payload.split('&').forEach {
val pos = it.indexOf('=')
if (pos != -1) {
val k = it.substring(0, pos)
val v = it.substring(pos + 1)
body.addEncoded(k, v)
}
}
val request = Request.Builder()
.post(body.build())
.url(url)
.addTags()
.addExtraHeaders(extraHeaders)
return httpClient.newCall(request.build()).await().ensureSuccess()
}
override suspend fun httpPost(url: HttpUrl, body: JSONObject, extraHeaders: Headers?): Response {
val mediaType = "application/json; charset=utf-8".toMediaType()
val requestBody = body.toString().toRequestBody(mediaType)
val request = Request.Builder()
.post(requestBody)
.url(url)
.addTags()
.addExtraHeaders(extraHeaders)
return httpClient.newCall(request.build()).await().ensureSuccess()
}
override suspend fun graphQLQuery(endpoint: String, query: String): JSONObject {
val body = JSONObject()
body.put("operationName", null as Any?)
body.put("variables", JSONObject())
body.put("query", "{$query}")
val mediaType = "application/json; charset=utf-8".toMediaType()
val requestBody = body.toString().toRequestBody(mediaType)
val request = Request.Builder()
.post(requestBody)
.url(endpoint)
.addTags()
val json = httpClient.newCall(request.build()).await().parseJson()
json.optJSONArray("errors")?.let {
if (it.length() != 0) {
throw GraphQLException(it)
}
}
return json
}
private fun Request.Builder.addTags(): Request.Builder {
tag(MangaSource::class.java, mangaSource)
return this
}
private fun Request.Builder.addExtraHeaders(headers: Headers?): Request.Builder {
if (headers != null) {
headers(headers)
}
return this
}
private fun Response.ensureSuccess(): Response {
val exception: Exception? = when (code) { // Catch some error codes, not all
HttpURLConnection.HTTP_NOT_FOUND -> NotFoundException(message, request.url.toString())
HttpURLConnection.HTTP_UNAUTHORIZED -> request.tag(MangaSource::class.java)?.let {
AuthRequiredException(it)
} ?: HttpStatusException(message, code, request.url.toString())
in 400..599 -> HttpStatusException(message, code, request.url.toString())
else -> null
}
if (exception != null) {
runCatching {
close()
}.onFailure {
exception.addSuppressed(it)
}
throw exception
}
return this
}
}

14
src/main/kotlin/org/koitharu/kotatsu/parsers/network/UserAgents.kt
View File

@@ -1,14 +0,0 @@
package org.koitharu.kotatsu.parsers.network
object UserAgents {
const val CHROME_MOBILE =
"Mozilla/5.0 (Linux; Android 13) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.5735.196 Mobile Safari/537.36"
const val CHROME_DESKTOP =
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"
const val FIREFOX_DESKTOP = "Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/116.0"
const val KOTATSU = "Kotatsu/5.3 (Android 13;;; en)"
}

116
src/main/kotlin/org/koitharu/kotatsu/parsers/network/WebClient.kt
View File

@@ -1,116 +0,0 @@
package org.koitharu.kotatsu.parsers.network
import okhttp3.Headers
import okhttp3.HttpUrl
import okhttp3.HttpUrl.Companion.toHttpUrl
import okhttp3.Response
import org.json.JSONObject
interface WebClient {
/**
* Do a GET http request to specific url
* @param url
*/
suspend fun httpGet(url: String): Response = httpGet(url.toHttpUrl())
suspend fun httpGet(url: String, extraHeaders: Headers?): Response = httpGet(url.toHttpUrl(), extraHeaders)
/**
* Do a GET http request to specific url
* @param url
*/
suspend fun httpGet(url: HttpUrl): Response = httpGet(url, null)
/**
* Do a GET http request to specific url
* @param url
* @param extraHeaders additional HTTP headers for request
*/
suspend fun httpGet(url: HttpUrl, extraHeaders: Headers?): Response
/**
* Do a HEAD http request to specific url
* @param url
*/
suspend fun httpHead(url: String): Response = httpHead(url.toHttpUrl())
/**
* Do a HEAD http request to specific url
* @param url
*/
suspend fun httpHead(url: HttpUrl): Response
/**
* Do a POST http request to specific url with `multipart/form-data` payload
* @param url
* @param form payload as key=>value map
*/
suspend fun httpPost(url: String, form: Map<String, String>): Response = httpPost(url.toHttpUrl(), form, null)
/**
* Do a POST http request to specific url with `multipart/form-data` payload
* @param url
* @param form payload as key=>value map
*/
suspend fun httpPost(url: HttpUrl, form: Map<String, String>): Response = httpPost(url, form, null)
/**
* Do a POST http request to specific url with `multipart/form-data` payload
* @param url
* @param form payload as key=>value map
* @param extraHeaders additional HTTP headers for request
*/
suspend fun httpPost(url: HttpUrl, form: Map<String, String>, extraHeaders: Headers?): Response
/**
* Do a POST http request to specific url with `multipart/form-data` payload
* @param url
* @param payload payload as `key=value` string with `&` separator
*/
suspend fun httpPost(url: String, payload: String): Response = httpPost(url.toHttpUrl(), payload, null)
/**
* Do a POST http request to specific url with `multipart/form-data` payload
* @param url
* @param payload payload as `key=value` string with `&` separator
*/
suspend fun httpPost(url: HttpUrl, payload: String): Response = httpPost(url, payload, null)
/**
* Do a POST http request to specific url with `multipart/form-data` payload
* @param url
* @param payload payload as `key=value` string with `&` separator
* @param extraHeaders additional HTTP headers for request
*/
suspend fun httpPost(url: HttpUrl, payload: String, extraHeaders: Headers?): Response
/**
* Do a POST http request to specific url with json payload
* @param url
* @param body
*/
suspend fun httpPost(url: String, body: JSONObject): Response = httpPost(url.toHttpUrl(), body, null)
/**
* Do a POST http request to specific url with json payload
* @param url
* @param body
*/
suspend fun httpPost(url: HttpUrl, body: JSONObject): Response = httpPost(url, body, null)
/**
* Do a POST http request to specific url with json payload
* @param url
* @param body
* @param extraHeaders additional HTTP headers for request
*/
suspend fun httpPost(url: HttpUrl, body: JSONObject, extraHeaders: Headers?): Response
/**
* Do a GraphQL request to specific url
* @param endpoint an url
* @param query GraphQL request payload
*/
suspend fun graphQLQuery(endpoint: String, query: String): JSONObject
}

78
src/main/kotlin/org/koitharu/kotatsu/parsers/site/be/AnibelParser.kt → src/main/kotlin/org/koitharu/kotatsu/parsers/site/AnibelParser.kt
View File

@@ -1,4 +1,4 @@
package org.koitharu.kotatsu.parsers.site.be
package org.koitharu.kotatsu.parsers.site
import androidx.collection.ArraySet
import org.json.JSONArray
@@ -8,30 +8,29 @@ import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.domain
import org.koitharu.kotatsu.parsers.util.generateUid
import org.koitharu.kotatsu.parsers.util.getDomain
import org.koitharu.kotatsu.parsers.util.json.mapJSON
import org.koitharu.kotatsu.parsers.util.json.mapJSONIndexed
import org.koitharu.kotatsu.parsers.util.json.mapJSONNotNull
import org.koitharu.kotatsu.parsers.util.json.stringIterator
import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl
import java.util.*
@MangaSourceParser("ANIBEL", "Anibel", "be")
internal class AnibelParser(context: MangaLoaderContext) : MangaParser(context, MangaSource.ANIBEL) {
internal class AnibelParser(override val context: MangaLoaderContext) : MangaParser(MangaSource.ANIBEL) {
override val configKeyDomain = ConfigKey.Domain("anibel.net")
override val configKeyDomain = ConfigKey.Domain("anibel.net", null)
override val availableSortOrders: Set<SortOrder> = EnumSet.of(
override val sortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.NEWEST,
)
override fun getFaviconUrl(): String {
return "https://cdn.${getDomain()}/favicons/favicon.png"
}
override suspend fun getList(
offset: Int,
query: String?,
tags: Set<MangaTag>?,
tagsExclude: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
if (!query.isNullOrEmpty()) {
@@ -74,12 +73,12 @@ internal class AnibelParser(context: MangaLoaderContext) : MangaParser(context,
title = title.getString("be"),
coverUrl = jo.getString("poster").removePrefix("/cdn")
.toAbsoluteUrl(getDomain("cdn")) + "?width=200&height=280",
altTitle = title.optJSONArray("alt")?.optString(0)?.takeUnless(String::isEmpty),
altTitle = title.getString("alt").takeUnless(String::isEmpty),
author = null,
isNsfw = false,
rating = jo.getDouble("rating").toFloat() / 10f,
url = href,
publicUrl = "https://${domain}/$href",
publicUrl = "https://${getDomain()}/$href",
tags = jo.getJSONArray("genres").mapToTags(),
state = when (jo.getString("status")) {
"ongoing" -> MangaState.ONGOING
@@ -112,21 +111,20 @@ internal class AnibelParser(context: MangaLoaderContext) : MangaParser(context,
""".trimIndent(),
).getJSONObject("media")
val title = details.getJSONObject("title")
val poster = details.getString("poster").removePrefix("/cdn").toAbsoluteUrl(getDomain("cdn"))
val poster = details.getString("poster").removePrefix("/cdn")
.toAbsoluteUrl(getDomain("cdn"))
val chapters = apiCall(
"""
chapters(mediaId: "${details.getString("mediaId")}", offset: 0, limit: 99999) {
docs {
id
chapter
released
}
chapters(mediaId: "${details.getString("mediaId")}") {
id
chapter
released
}
""".trimIndent(),
).getJSONObject("chapters").getJSONArray("docs")
).getJSONArray("chapters")
return manga.copy(
title = title.getString("be"),
altTitle = title.optJSONArray("alt")?.optString(0)?.takeUnless(String::isEmpty),
altTitle = title.getString("alt"),
coverUrl = "$poster?width=200&height=280",
largeCoverUrl = poster,
description = details.getJSONObject("description").getString("be"),
@@ -167,17 +165,19 @@ internal class AnibelParser(context: MangaLoaderContext) : MangaParser(context,
""".trimIndent(),
).getJSONObject("chapter")
val pages = chapterJson.getJSONArray("images")
val chapterUrl = "https://${getDomain()}/${chapter.url}"
return pages.mapJSONIndexed { i, jo ->
MangaPage(
id = generateUid("${chapter.url}/$i"),
url = jo.getString("large"),
referer = chapterUrl,
preview = jo.getString("thumbnail"),
source = source,
)
}
}
override suspend fun getAvailableTags(): Set<MangaTag> {
override suspend fun getTags(): Set<MangaTag> {
val json = apiCall(
"""
getFilters(mediaType: manga) {
@@ -192,29 +192,23 @@ internal class AnibelParser(context: MangaLoaderContext) : MangaParser(context,
private suspend fun search(query: String): List<Manga> {
val json = apiCall(
"""
search(query: "$query", limit: 60) {
mediaId
search(query: "$query", limit: 40) {
id
title {
be
en
}
poster
status
slug
mediaType
genres
url
type
}
""".trimIndent(),
)
val array = json.getJSONArray("search")
return array.mapJSONNotNull { jo ->
val type = jo.getString("mediaType").lowercase()
if (type != "manga") {
return@mapJSONNotNull null
}
val mediaId = jo.getString("mediaId")
return array.mapJSON { jo ->
val mediaId = jo.getString("id")
val title = jo.getJSONObject("title")
val href = "$type/${jo.getString("slug")}"
val href = "${jo.getString("type").lowercase()}/${jo.getString("url")}"
Manga(
id = generateUid(mediaId),
title = title.getString("be"),
@@ -225,20 +219,17 @@ internal class AnibelParser(context: MangaLoaderContext) : MangaParser(context,
isNsfw = false,
rating = RATING_UNKNOWN,
url = href,
publicUrl = "https://${domain}/$href",
tags = jo.getJSONArray("genres").mapToTags(),
state = when (jo.getString("status")) {
"ongoing" -> MangaState.ONGOING
"finished" -> MangaState.FINISHED
else -> null
},
publicUrl = "https://${getDomain()}/$href",
tags = emptySet(),
state = null,
source = source,
)
}
}
private suspend fun apiCall(request: String): JSONObject {
return webClient.graphQLQuery("https://${domain}/graphql", request).getJSONObject("data")
return context.graphQLQuery("https://api.${getDomain()}/graphql", request)
.getJSONObject("data")
}
private fun JSONArray.mapToTags(): Set<MangaTag> {
@@ -251,7 +242,6 @@ internal class AnibelParser(context: MangaLoaderContext) : MangaParser(context,
c == '-' -> {
builder.setCharAt(i, ' ')
}
capitalize -> {
builder.setCharAt(i, c.uppercaseChar())
capitalize = false
@@ -273,4 +263,4 @@ internal class AnibelParser(context: MangaLoaderContext) : MangaParser(context,
}
return result
}
}
}

194
src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/BatoToParser.kt → src/main/kotlin/org/koitharu/kotatsu/parsers/site/BatoToParser.kt
View File

@@ -1,4 +1,4 @@
package org.koitharu.kotatsu.parsers.site.all
package org.koitharu.kotatsu.parsers.site
import androidx.collection.ArraySet
import org.json.JSONArray
@@ -11,7 +11,6 @@ import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.exception.ParseException
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import org.koitharu.kotatsu.parsers.util.json.isNullOrEmpty
import java.nio.charset.StandardCharsets
import java.security.MessageDigest
import java.util.*
@@ -20,137 +19,58 @@ import javax.crypto.spec.IvParameterSpec
import javax.crypto.spec.SecretKeySpec
@MangaSourceParser("BATOTO", "Bato.To")
internal class BatoToParser(context: MangaLoaderContext) : PagedMangaParser(
context = context,
internal class BatoToParser(override val context: MangaLoaderContext) : PagedMangaParser(
source = MangaSource.BATOTO,
pageSize = 60,
searchPageSize = 20,
) {
override val availableSortOrders: Set<SortOrder> = EnumSet.of(
override val sortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.NEWEST,
SortOrder.UPDATED,
SortOrder.POPULARITY,
SortOrder.ALPHABETICAL,
)
override val availableStates: Set<MangaState> = EnumSet.allOf(MangaState::class.java)
override val isTagsExclusionSupported: Boolean = true
override val availableContentRating: Set<ContentRating> = EnumSet.of(ContentRating.SAFE)
override val configKeyDomain = ConfigKey.Domain(
"bato.to",
"batocomic.com",
"batocomic.net",
"batocomic.org",
"batotoo.com",
"batotwo.com",
"battwo.com",
"comiko.net",
"comiko.org",
"mangatoto.com",
"mangatoto.net",
"mangatoto.org",
"readtoto.com",
"readtoto.net",
"readtoto.org",
"dto.to",
"hto.to",
"mto.to",
"wto.to",
"xbato.com",
"xbato.net",
"xbato.org",
"zbato.com",
"zbato.net",
"zbato.org",
arrayOf("bato.to", "mto.to", "mangatoto.com", "battwo.com", "batotwo.com", "comiko.net", "batotoo.com"),
)
override suspend fun getListPage(page: Int, filter: MangaListFilter?): List<Manga> {
when (filter) {
is MangaListFilter.Search -> {
return search(page, filter.query)
}
is MangaListFilter.Advanced -> {
val url = buildString {
append("https://")
append(domain)
append("/browse?sort=")
when (filter.sortOrder) {
SortOrder.UPDATED -> append("update.za")
SortOrder.POPULARITY -> append("views_a.za")
SortOrder.NEWEST -> append("create.za")
SortOrder.ALPHABETICAL -> append("title.az")
else -> append("update.za")
}
filter.states.oneOrThrowIfMany()?.let {
append("&release=")
append(
when (it) {
MangaState.ONGOING -> "ongoing"
MangaState.FINISHED -> "completed"
MangaState.ABANDONED -> "cancelled"
MangaState.PAUSED -> "hiatus"
MangaState.UPCOMING -> "pending"
},
)
}
filter.locale?.let {
append("&langs=")
append(it.language)
}
append("&genres=")
if (filter.tags.isNotEmpty()) {
appendAll(filter.tags, ",") { it.key }
}
append("|")
if (filter.tagsExclude.isNotEmpty()) {
appendAll(filter.tagsExclude, ",") { it.key }
}
if (filter.contentRating.isNotEmpty()) {
filter.contentRating.oneOrThrowIfMany()?.let {
append(
when (it) {
ContentRating.SAFE -> append(",gore,bloody,violence,ecchi,adult,mature,smut,hentai")
else -> append("")
},
)
}
}
append("&page=")
append(page.toString())
}
return parseList(url, page)
}
null -> {
val url = buildString {
append("https://")
append(domain)
append("/browse?sort=update.za")
append("&page=")
append(page.toString())
}
return parseList(url, page)
}
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
if (!query.isNullOrEmpty()) {
return search(page, query)
}
@Suppress("NON_EXHAUSTIVE_WHEN_STATEMENT")
val url = buildString {
append("https://")
append(getDomain())
append("/browse?sort=")
when (sortOrder) {
SortOrder.UPDATED,
-> append("update.za")
SortOrder.POPULARITY -> append("views_a.za")
SortOrder.NEWEST -> append("create.za")
SortOrder.ALPHABETICAL -> append("title.az")
}
if (!tags.isNullOrEmpty()) {
append("&genres=")
appendAll(tags, ",") { it.key }
}
append("&page=")
append(page)
}
return parseList(url, page)
}
override suspend fun getDetails(manga: Manga): Manga {
val root = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml()
val root = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml()
.requireElementById("mainer")
val details = root.selectFirstOrThrow(".detail-set")
val attrs = details.selectFirst(".attr-main")?.select(".attr-item")?.associate {
@@ -164,29 +84,28 @@ internal class BatoToParser(context: MangaLoaderContext) : PagedMangaParser(
?.selectFirst(".limit-html")
?.html(),
tags = manga.tags + attrs["Genres:"]?.parseTags().orEmpty(),
state = when (attrs["Original work:"]?.text()) {
state = when (attrs["Release status:"]?.text()) {
"Ongoing" -> MangaState.ONGOING
"Completed" -> MangaState.FINISHED
"Cancelled" -> MangaState.ABANDONED
"Hiatus" -> MangaState.PAUSED
else -> manga.state
},
author = attrs["Authors:"]?.text()?.trim() ?: manga.author,
chapters = root.selectFirst(".episode-list")
?.selectFirst(".main")
?.children()
?.mapChapters(reversed = true) { i, div ->
?.reversed()
?.mapChapters { i, div ->
div.parseChapter(i)
}.orEmpty(),
)
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(domain)
val scripts = webClient.httpGet(fullUrl).parseHtml().select("script")
val fullUrl = chapter.url.toAbsoluteUrl(getDomain())
val scripts = context.httpGet(fullUrl).parseHtml().select("script")
for (script in scripts) {
val scriptSrc = script.html()
val p = scriptSrc.indexOf("const imgHttps =")
val p = scriptSrc.indexOf("const imgHttpLis =")
if (p == -1) continue
val start = scriptSrc.indexOf('[', p)
val end = scriptSrc.indexOf(';', start)
@@ -206,7 +125,8 @@ internal class BatoToParser(context: MangaLoaderContext) : PagedMangaParser(
val url = images.getString(i)
result += MangaPage(
id = generateUid(url),
url = if (args.length() == 0) url else url + "?" + args.getString(i),
url = url + "?" + args.getString(i),
referer = fullUrl,
preview = null,
source = source,
)
@@ -216,9 +136,9 @@ internal class BatoToParser(context: MangaLoaderContext) : PagedMangaParser(
throw ParseException("Cannot find images list", fullUrl)
}
override suspend fun getAvailableTags(): Set<MangaTag> {
val scripts = webClient.httpGet(
"https://${domain}/browse",
override suspend fun getTags(): Set<MangaTag> {
val scripts = context.httpGet(
"https://${getDomain()}/browse",
).parseHtml().selectOrThrow("script")
for (script in scripts) {
val genres = script.html().substringBetweenFirst("const _genres =", ";") ?: continue
@@ -227,7 +147,7 @@ internal class BatoToParser(context: MangaLoaderContext) : PagedMangaParser(
jo.keys().forEach { key ->
val item = jo.getJSONObject(key)
result += MangaTag(
title = item.getString("text").toTitleCase(Locale.ENGLISH),
title = item.getString("text").toTitleCase(),
key = item.getString("file"),
source = source,
)
@@ -237,26 +157,12 @@ internal class BatoToParser(context: MangaLoaderContext) : PagedMangaParser(
throw ParseException("Cannot find gernes list", scripts[0].baseUri())
}
override suspend fun getAvailableLocales(): Set<Locale> = setOf(
Locale.CHINESE, Locale.ENGLISH, Locale.US, Locale.FRENCH, Locale.GERMAN, Locale.ITALIAN, Locale.JAPANESE,
Locale("af"), Locale("ar"), Locale("az"), Locale("eu"), Locale("be"),
Locale("bn"), Locale("bs"), Locale("bg"), Locale("my"), Locale("km"),
Locale("ceb"), Locale("zh_hk"), Locale("zh_tw"), Locale("hr"), Locale("cs"),
Locale("da"), Locale("nl"), Locale("eo"), Locale("et"), Locale("fil"),
Locale("fi"), Locale("ka"), Locale("el"), Locale("ht"), Locale("he"),
Locale("hi"), Locale("hu"), Locale("id"), Locale("kk"), Locale("ko"),
Locale("lv"), Locale("ms"), Locale("ml"), Locale("mo"), Locale("mn"),
Locale("ne"), Locale("no"), Locale("fa"), Locale("pl"), Locale("pt"),
Locale("pt_br"), Locale("pt_pt"), Locale("ro"), Locale("ru"), Locale("sr"),
Locale("si"), Locale("sk"), Locale("es"), Locale("es_419"), Locale("ta"),
Locale("te"), Locale("th"), Locale("ti"), Locale("tr"), Locale("uk"),
Locale("vi"), Locale("zu"),
)
override fun getFaviconUrl(): String = "https://styles.amarkcdn.com/img/batoto/favicon.ico?v0"
private suspend fun search(page: Int, query: String): List<Manga> {
val url = buildString {
append("https://")
append(domain)
append(getDomain())
append("/search?word=")
append(query.replace(' ', '+'))
append("&page=")
@@ -271,7 +177,7 @@ internal class BatoToParser(context: MangaLoaderContext) : PagedMangaParser(
?.toIntOrNull() ?: body.parseFailed("Cannot determine current page")
private suspend fun parseList(url: String, page: Int): List<Manga> {
val body = webClient.httpGet(url).parseHtml().body()
val body = context.httpGet(url).parseHtml().body()
if (body.selectFirst(".browse-no-matches") != null) {
return emptyList()
}
@@ -401,4 +307,4 @@ internal class BatoToParser(context: MangaLoaderContext) : PagedMangaParser(
} else byteArrayOf(),
)
}
}
}

278
src/main/kotlin/org/koitharu/kotatsu/parsers/site/vi/BlogTruyenParser.kt → src/main/kotlin/org/koitharu/kotatsu/parsers/site/BlogTruyenParser.kt
View File

@@ -1,7 +1,8 @@
package org.koitharu.kotatsu.parsers.site.vi
package org.koitharu.kotatsu.parsers.site
import androidx.collection.ArrayMap
import okhttp3.Headers
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
import org.json.JSONArray
import org.jsoup.nodes.Document
import org.jsoup.nodes.Element
@@ -10,139 +11,27 @@ import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.network.UserAgents
import org.koitharu.kotatsu.parsers.util.*
import java.text.SimpleDateFormat
import java.util.*
import kotlin.collections.HashSet
@MangaSourceParser("BLOGTRUYEN", "BlogTruyen", "vi")
class BlogTruyenParser(context: MangaLoaderContext) :
PagedMangaParser(context, MangaSource.BLOGTRUYEN, pageSize = 20) {
class BlogTruyenParser(override val context: MangaLoaderContext) :
PagedMangaParser(MangaSource.BLOGTRUYEN, pageSize = 20) {
override val configKeyDomain: ConfigKey.Domain
get() = ConfigKey.Domain("blogtruyenmoi.com")
get() = ConfigKey.Domain("blogtruyen.vn", null)
override val availableSortOrders: Set<SortOrder>
override val sortOrders: Set<SortOrder>
get() = EnumSet.of(SortOrder.UPDATED)
override val headers: Headers = Headers.Builder()
.add("User-Agent", UserAgents.CHROME_DESKTOP)
.build()
override val isMultipleTagsSupported = false
private val mutex = Mutex()
private val dateFormat = SimpleDateFormat("dd/MM/yyyy HH:mm", Locale.US)
private var cacheTags = SuspendLazy(::fetchTags)
override suspend fun getListPage(page: Int, filter: MangaListFilter?): List<Manga> {
return when (filter) {
is MangaListFilter.Search -> {
val searchUrl = "https://${domain}/timkiem/nangcao/1/0/-1/-1?txt=${filter.query.urlEncoded()}&p=$page"
val searchContent = webClient.httpGet(searchUrl).parseHtml()
.selectFirst("section.list-manga-bycate > div.list")
parseMangaList(searchContent)
}
is MangaListFilter.Advanced -> {
if (filter.tags.isNotEmpty()) {
filter.tags.oneOrThrowIfMany().let {
val categoryAjax =
"https://${domain}/ajax/Category/AjaxLoadMangaByCategory?id=${it?.key}&orderBy=5&p=$page"
val listContent = webClient.httpGet(categoryAjax).parseHtml().selectFirst("div.list")
parseMangaList(listContent)
}
} else {
getNormalList(page)
}
}
null -> getNormalList(page)
}
}
private suspend fun getNormalList(page: Int): List<Manga> {
val pageLink = "https://${domain}/page-$page"
val doc = webClient.httpGet(pageLink).parseHtml()
val listElements = doc.selectFirstOrThrow("section.list-mainpage.listview")
.select("div.bg-white.storyitem")
return listElements.mapNotNull { el ->
val linkTag = el.selectFirst("div.fl-l > a") ?: return@mapNotNull null
val relativeUrl = linkTag.attrAsRelativeUrl("href")
val tags = cacheTags.tryGet().getOrNull()?.let { tagMap ->
el.select("footer > div.category > a").mapNotNullToSet { a ->
tagMap[a.text()]
}
}
Manga(
id = generateUid(relativeUrl),
title = linkTag.attr("title"),
altTitle = null,
description = el.selectFirst("p.al-j.break.line-height-15")?.text(),
url = relativeUrl,
publicUrl = relativeUrl.toAbsoluteUrl(domain),
coverUrl = linkTag.selectLast("img")?.src().orEmpty(),
source = source,
tags = tags ?: emptySet(),
isNsfw = false,
rating = RATING_UNKNOWN,
author = null,
state = null,
)
}
}
private fun parseMangaList(listElement: Element?): List<Manga> {
listElement ?: return emptyList()
return listElement.select("span.tiptip[data-tiptip]").mapNotNull {
val mangaInfo = listElement.getElementById(it.attr("data-tiptip")) ?: return@mapNotNull null
val a = it.selectFirst("a") ?: return@mapNotNull null
val relativeUrl = a.attrAsRelativeUrl("href")
Manga(
id = generateUid(relativeUrl),
title = a.text(),
altTitle = null,
description = mangaInfo.select("div.al-j.fs-12").text(),
url = relativeUrl,
publicUrl = relativeUrl.toAbsoluteUrl(domain),
coverUrl = mangaInfo.selectFirst("div > img.img")?.src().orEmpty(),
isNsfw = false,
rating = RATING_UNKNOWN,
tags = emptySet(),
author = null,
state = null,
source = source,
)
}
}
override suspend fun getAvailableTags(): Set<MangaTag> {
return cacheTags.get().values.toSet()
}
private suspend fun fetchTags(): Map<String, MangaTag> {
val doc = webClient.httpGet("/timkiem/nangcao".toAbsoluteUrl(domain)).parseHtml()
val tagItems = doc.select("li[data-id]")
val tagMap = ArrayMap<String, MangaTag>(tagItems.size)
for (tag in tagItems) {
val title = tag.text().trim()
tagMap[tag.text().trim()] = MangaTag(
title = title,
key = tag.attr("data-id"),
source = source,
)
}
return tagMap
}
private var cacheTags: ArrayMap<String, MangaTag>? = null
override suspend fun getDetails(manga: Manga): Manga {
val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml()
val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml()
val descriptionElement = doc.selectFirstOrThrow("div.description")
val statusText = descriptionElement
.selectFirst("p:contains(Trạng thái) > span.color-red")
@@ -167,28 +56,27 @@ class BlogTruyenParser(context: MangaLoaderContext) :
}
}
val tags = cacheTags.tryGet().getOrNull()?.let { tagMap ->
descriptionElement.select("p > span.category").mapNotNullToSet {
val tagName = it.selectFirst("a")?.text()?.trim() ?: return@mapNotNullToSet null
tagMap[tagName]
}
val tagMap = getOrCreateTagMap()
val tags = descriptionElement.select("p > span.category").mapNotNullToSet {
val tagName = it.selectFirst("a")?.text()?.trim() ?: return@mapNotNullToSet null
tagMap[tagName]
}
return manga.copy(
tags = tags ?: emptySet(),
tags = tags,
author = descriptionElement.selectFirst("p:contains(Tác giả) > a")?.text(),
description = doc.selectFirst(".detail .content")?.html(),
chapters = parseChapterList(doc),
largeCoverUrl = doc.selectLast("div.thumbnail > img")?.src().orEmpty(),
largeCoverUrl = doc.selectLast("div.thumbnail > img")?.attrAsAbsoluteUrlOrNull("src"),
state = state,
rating = rating ?: RATING_UNKNOWN,
isNsfw = doc.getElementById("warningCategory") != null,
isNsfw = doc.getElementById("warningCategory") != null
)
}
private fun parseChapterList(doc: Document): List<MangaChapter> {
val chapterList = doc.select("#list-chapters > p")
return chapterList.mapChapters(reversed = true) { index, element ->
return chapterList.asReversed().mapChapters { index, element ->
val titleElement = element.selectFirst("span.title > a") ?: return@mapChapters null
val name = titleElement.text()
val relativeUrl = titleElement.attrAsRelativeUrl("href")
@@ -202,6 +90,87 @@ class BlogTruyenParser(context: MangaLoaderContext) :
scanlator = null,
uploadDate = uploadDate,
branch = null,
source = source
)
}
}
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
return when {
!query.isNullOrEmpty() -> {
val searchUrl = "https://${getDomain()}/timkiem/nangcao/1/0/-1/-1?txt=${query.urlEncoded()}&p=$page"
val searchContent = context.httpGet(searchUrl).parseHtml()
.selectFirst("section.list-manga-bycate > div.list")
parseMangaList(searchContent)
}
!tags.isNullOrEmpty() -> {
val tag = tags.oneOrThrowIfMany()!!
val categoryAjax = "https://${getDomain()}/ajax/Category/AjaxLoadMangaByCategory?id=${tag.key}&orderBy=5&p=$page"
val listContent = context.httpGet(categoryAjax).parseHtml().selectFirst("div.list")
parseMangaList(listContent)
}
else -> getNormalList(page)
}
}
private suspend fun getNormalList(page: Int): List<Manga> {
val pageLink = "https://${getDomain()}/page-$page"
val doc = context.httpGet(pageLink).parseHtml()
val listElements = doc.selectFirstOrThrow("section.list-mainpage.listview")
.select("div.bg-white.storyitem")
return listElements.mapNotNull {
val linkTag = it.selectFirst("div.fl-l > a") ?: return@mapNotNull null
val relativeUrl = linkTag.attrAsRelativeUrl("href")
val tagMap = getOrCreateTagMap()
val tags = it.select("footer > div.category > a").mapNotNullToSet { a ->
tagMap[a.text()]
}
Manga(
id = generateUid(relativeUrl),
title = linkTag.attr("title"),
altTitle = null,
description = it.selectFirst("p.al-j.break.line-height-15")?.text(),
url = relativeUrl,
publicUrl = relativeUrl.toAbsoluteUrl(getDomain()),
coverUrl = linkTag.selectLast("img")?.attr("src").orEmpty(),
source = source,
tags = tags,
isNsfw = false,
rating = RATING_UNKNOWN,
author = null,
state = null,
)
}
}
private fun parseMangaList(listElement: Element?): List<Manga> {
listElement ?: return emptyList()
return listElement.select("span.tiptip[data-tiptip]").mapNotNull {
val mangaInfo = listElement.getElementById(it.attr("data-tiptip")) ?: return@mapNotNull null
val a = it.selectFirst("a") ?: return@mapNotNull null
val relativeUrl = a.attrAsRelativeUrl("href")
Manga(
id = generateUid(relativeUrl),
title = a.text(),
altTitle = null,
description = mangaInfo.select("div.al-j.fs-12").text(),
url = relativeUrl,
publicUrl = relativeUrl.toAbsoluteUrl(getDomain()),
coverUrl = mangaInfo.selectFirst("div > img.img")?.absUrl("src").orEmpty(),
isNsfw = false,
rating = RATING_UNKNOWN,
tags = emptySet(),
author = null,
state = null,
source = source,
)
}
@@ -210,16 +179,19 @@ class BlogTruyenParser(context: MangaLoaderContext) :
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
fun generateImageId(index: Int) = generateUid("${chapter.url}/$index")
val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml()
val doc = context.httpGet(chapter.url.toAbsoluteUrl(getDomain())).parseHtml()
val pages = ArrayList<MangaPage>()
val referer = chapter.url.toAbsoluteUrl(getDomain())
doc.select("#content > img").forEach { img ->
val url = img.attrAsRelativeUrl("src")
pages.add(
MangaPage(
id = generateImageId(pages.size),
url = img.src().orEmpty(),
id = generateImageId(pages.lastIndex),
url = url,
referer = referer,
preview = null,
source = source,
),
)
)
}
@@ -232,15 +204,45 @@ class BlogTruyenParser(context: MangaLoaderContext) :
val imageUrl = imageArr.getJSONObject(i).getString("url")
pages.add(
MangaPage(
id = generateImageId(pages.size),
id = generateImageId(pages.lastIndex),
url = imageUrl,
referer = referer,
preview = null,
source = source,
),
source = source
)
)
}
}
return pages
}
override suspend fun getTags(): Set<MangaTag> {
val map = getOrCreateTagMap()
val tags = HashSet<MangaTag>(map.size)
for (entry in map) {
tags.add(entry.value)
}
return tags
}
private suspend fun getOrCreateTagMap(): ArrayMap<String, MangaTag> = mutex.withLock {
cacheTags?.let { return@withLock it }
val doc = context.httpGet("/timkiem/nangcao".toAbsoluteUrl(getDomain())).parseHtml()
val tagItems = doc.select("li[data-id]")
val tagMap = ArrayMap<String, MangaTag>(tagItems.size)
for (tag in tagItems) {
val title = tag.text().trim()
tagMap[tag.text().trim()] = MangaTag(
title = title,
key = tag.attr("data-id"),
source = source
)
}
cacheTags = tagMap
tagMap
}
}

73
src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/CloneMangaParser.kt → src/main/kotlin/org/koitharu/kotatsu/parsers/site/CloneMangaParser.kt
View File

@@ -1,53 +1,47 @@
package org.koitharu.kotatsu.parsers.site.en
package org.koitharu.kotatsu.parsers.site
import org.koitharu.kotatsu.parsers.InternalParsersApi
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.*
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import java.util.*
@MangaSourceParser("CLONEMANGA", "CloneManga", "en")
internal class CloneMangaParser(context: MangaLoaderContext) : MangaParser(context, MangaSource.CLONEMANGA) {
internal class CloneMangaParser(override val context: MangaLoaderContext) : PagedMangaParser(
MangaSource.CLONEMANGA,
pageSize = 1,
) {
override val availableSortOrders: Set<SortOrder> = Collections.singleton(
override val sortOrders: Set<SortOrder> = Collections.singleton(
SortOrder.POPULARITY,
)
override val configKeyDomain = ConfigKey.Domain("manga.clone-army.org")
override val configKeyDomain = ConfigKey.Domain("manga.clone-army.org", null)
@InternalParsersApi
override suspend fun getList(offset: Int, filter: MangaListFilter?): List<Manga> {
val link = when (filter) {
is MangaListFilter.Search -> {
return emptyList()
}
is MangaListFilter.Advanced -> {
if (offset > 0) {
return emptyList()
}
"https://$domain/viewer_landing.php"
}
null -> "https://$domain/viewer_landing.php"
override fun getFaviconUrl(): String {
return "https://pbs.twimg.com/profile_images/458758466346029056/Ys93EANp_400x400.png"
}
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
if (query != null || page > 1) {
return emptyList()
}
val doc = webClient.httpGet(link).parseHtml()
val link = "https://${getDomain()}/viewer_landing.php"
val doc = context.httpGet(link).parseHtml()
val mangas = doc.getElementsByClass("comicPreviewContainer")
return mangas.mapNotNull { item ->
val background = item.selectFirstOrThrow(".comicPreview").styleValueOrNull("background")
val href = item.selectFirst("a")?.attrAsRelativeUrl("href") ?: return@mapNotNull null
val cover = background?.substring(background.indexOf("site/themes"), background.indexOf(")"))
val attr = item.getElementsByClass("comicPreview").attr("style")
val href = item.selectFirst("a")?.attrAsAbsoluteUrl("href") ?: return@mapNotNull null
val cover = attr.substring(attr.indexOf("site/themes"), attr.indexOf(")"))
Manga(
id = generateUid(href),
title = item.selectFirst("h3")?.text() ?: return@mapNotNull null,
coverUrl = "https://$domain/$cover",
coverUrl = "https://${getDomain()}/$cover",
altTitle = null,
author = "Dan Kim",
rating = RATING_UNKNOWN,
@@ -55,14 +49,14 @@ internal class CloneMangaParser(context: MangaLoaderContext) : MangaParser(conte
isNsfw = false,
tags = emptySet(),
state = null,
publicUrl = href.toAbsoluteUrl(domain),
publicUrl = href.toAbsoluteUrl(getDomain()),
source = source,
)
}
}
override suspend fun getDetails(manga: Manga): Manga {
val doc = webClient.httpGet(manga.publicUrl).parseHtml()
val doc = context.httpGet(manga.publicUrl).parseHtml()
val series = doc.location()
val numChapters = Regex(
pattern = "&page=(.*)&lang=",
@@ -81,7 +75,7 @@ internal class CloneMangaParser(context: MangaLoaderContext) : MangaParser(conte
scanlator = null,
branch = null,
uploadDate = 0L,
source = source,
source = MangaSource.DUMMY,
)
chapters.add(chapter)
}
@@ -89,7 +83,7 @@ internal class CloneMangaParser(context: MangaLoaderContext) : MangaParser(conte
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml()
val doc = context.httpGet(chapter.url.toAbsoluteUrl(getDomain())).parseHtml()
val imgUrl = doc.getElementsByClass("subsectionContainer")[0]
.selectFirst("img")
?.attrAsAbsoluteUrlOrNull("src") ?: doc.parseFailed("Something broken")
@@ -97,11 +91,14 @@ internal class CloneMangaParser(context: MangaLoaderContext) : MangaParser(conte
MangaPage(
id = generateUid(imgUrl),
url = imgUrl,
referer = imgUrl,
preview = null,
source = source,
source = MangaSource.DUMMY,
),
)
}
override suspend fun getAvailableTags(): Set<MangaTag> = emptySet()
}
override suspend fun getTags(): Set<MangaTag> {
return emptySet()
}
}

216
src/main/kotlin/org/koitharu/kotatsu/parsers/site/ComickFunParser.kt Normal file
View File

@@ -0,0 +1,216 @@
package org.koitharu.kotatsu.parsers.site
import androidx.collection.ArraySet
import androidx.collection.SparseArrayCompat
import org.json.JSONArray
import org.json.JSONObject
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import org.koitharu.kotatsu.parsers.util.json.*
import java.text.SimpleDateFormat
import java.util.*
/**
* https://api.comick.fun/docs/static/index.html
*/
private const val PAGE_SIZE = 20
private const val CHAPTERS_LIMIT = 99999
@MangaSourceParser("COMICK_FUN", "ComicK")
internal class ComickFunParser(override val context: MangaLoaderContext) : MangaParser(MangaSource.COMICK_FUN) {
override val configKeyDomain = ConfigKey.Domain("comick.fun", null)
override val sortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.POPULARITY,
SortOrder.UPDATED,
SortOrder.RATING,
)
@Volatile
private var cachedTags: SparseArrayCompat<MangaTag>? = null
override suspend fun getList(
offset: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val domain = getDomain()
val url = buildString {
append("https://api.")
append(domain)
append("/search?tachiyomi=true")
if (!query.isNullOrEmpty()) {
if (offset > 0) {
return emptyList()
}
append("&q=")
append(query.urlEncoded())
} else {
append("&limit=")
append(PAGE_SIZE)
append("&page=")
append((offset / PAGE_SIZE) + 1)
if (!tags.isNullOrEmpty()) {
append("&genres=")
appendAll(tags, "&genres=", MangaTag::key)
}
append("&sort=") // view, uploaded, rating, follow, user_follow_count
append(
when (sortOrder) {
SortOrder.POPULARITY -> "view"
SortOrder.RATING -> "rating"
else -> "uploaded"
},
)
}
}
val ja = context.httpGet(url).parseJsonArray()
val tagsMap = cachedTags ?: loadTags()
return ja.mapJSON { jo ->
val slug = jo.getString("slug")
Manga(
id = generateUid(slug),
title = jo.getString("title"),
altTitle = null,
url = slug,
publicUrl = "https://$domain/comic/$slug",
rating = jo.getDoubleOrDefault("rating", -10.0).toFloat() / 10f,
isNsfw = false,
coverUrl = jo.getString("cover_url"),
largeCoverUrl = null,
description = jo.getStringOrNull("desc"),
tags = jo.selectGenres("genres", tagsMap),
state = runCatching {
if (jo.getBoolean("translation_completed")) {
MangaState.FINISHED
} else {
MangaState.ONGOING
}
}.getOrNull(),
author = null,
source = source,
)
}
}
override suspend fun getDetails(manga: Manga): Manga {
val domain = getDomain()
val url = "https://api.$domain/comic/${manga.url}?tachiyomi=true"
val jo = context.httpGet(url).parseJson()
val comic = jo.getJSONObject("comic")
return manga.copy(
title = comic.getString("title"),
altTitle = null, // TODO
isNsfw = jo.getBoolean("matureContent") || comic.getBoolean("hentai"),
description = comic.getStringOrNull("parsed") ?: comic.getString("desc"),
tags = manga.tags + jo.getJSONArray("genres").mapJSONToSet {
MangaTag(
title = it.getString("name"),
key = it.getString("slug"),
source = source,
)
},
author = jo.getJSONArray("artists").optJSONObject(0)?.getString("name"),
chapters = getChapters(comic.getLong("id")),
)
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val jo = context.httpGet(
"https://api.${getDomain()}/chapter/${chapter.url}?tachiyomi=true",
).parseJson().getJSONObject("chapter")
val referer = "https://${getDomain()}/"
return jo.getJSONArray("images").mapJSON {
val url = it.getString("url")
MangaPage(
id = generateUid(url),
url = url,
referer = referer,
preview = null,
source = source,
)
}
}
override suspend fun getTags(): Set<MangaTag> {
val sparseArray = cachedTags ?: loadTags()
val set = ArraySet<MangaTag>(sparseArray.size())
for (i in 0 until sparseArray.size()) {
set.add(sparseArray.valueAt(i))
}
return set
}
private suspend fun loadTags(): SparseArrayCompat<MangaTag> {
val ja = context.httpGet("https://api.${getDomain()}/genre").parseJsonArray()
val tags = SparseArrayCompat<MangaTag>(ja.length())
for (jo in ja.JSONIterator()) {
tags.append(
jo.getInt("id"),
MangaTag(
title = jo.getString("name"),
key = jo.getString("slug"),
source = source,
),
)
}
cachedTags = tags
return tags
}
private suspend fun getChapters(id: Long): List<MangaChapter> {
val ja = context.httpGet(
url = "https://api.${getDomain()}/comic/$id/chapter?tachiyomi=true&limit=$CHAPTERS_LIMIT",
).parseJson().getJSONArray("chapters")
val dateFormat = SimpleDateFormat("yyyy-MM-dd")
val counters = HashMap<Locale, Int>()
return ja.mapReversed { jo ->
val locale = Locale.forLanguageTag(jo.getString("lang"))
var number = counters[locale] ?: 0
number++
counters[locale] = number
MangaChapter(
id = generateUid(jo.getLong("id")),
name = buildString {
jo.getStringOrNull("vol")?.let { append("Vol ").append(it).append(' ') }
jo.getStringOrNull("chap")?.let { append("Chap ").append(it) }
jo.getStringOrNull("title")?.let { append(": ").append(it) }
},
number = number,
url = jo.getString("hid"),
scanlator = jo.optJSONArray("group_name")?.optString(0),
uploadDate = dateFormat.tryParse(jo.getString("created_at").substringBefore('T')),
branch = locale.getDisplayName(locale).toTitleCase(locale),
source = source,
)
}
}
private inline fun <R> JSONArray.mapReversed(block: (JSONObject) -> R): List<R> {
val len = length()
val destination = ArrayList<R>(len)
for (i in (0 until len).reversed()) {
val jo = getJSONObject(i)
destination.add(block(jo))
}
return destination
}
private fun JSONObject.selectGenres(name: String, tags: SparseArrayCompat<MangaTag>): Set<MangaTag> {
val array = optJSONArray(name) ?: return emptySet()
val res = ArraySet<MangaTag>(array.length())
for (i in 0 until array.length()) {
val id = array.getInt(i)
val tag = tags.get(id) ?: continue
res.add(tag)
}
return res
}
}

85
src/main/kotlin/org/koitharu/kotatsu/parsers/site/ru/DesuMeParser.kt → src/main/kotlin/org/koitharu/kotatsu/parsers/site/DesuMeParser.kt
View File

@@ -1,49 +1,39 @@
package org.koitharu.kotatsu.parsers.site.ru
package org.koitharu.kotatsu.parsers.site
import androidx.collection.ArrayMap
import okhttp3.Headers
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.exception.ParseException
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.network.UserAgents
import org.koitharu.kotatsu.parsers.util.*
import org.koitharu.kotatsu.parsers.util.json.mapJSON
import org.koitharu.kotatsu.parsers.util.json.mapJSONIndexed
import org.koitharu.kotatsu.parsers.util.json.mapJSONToSet
import java.util.*
@MangaSourceParser("DESUME", "Desu.Me", "ru")
internal class DesuMeParser(context: MangaLoaderContext) : PagedMangaParser(context, MangaSource.DESUME, 20) {
@MangaSourceParser("DESUME", "Desu.me", "ru")
internal class DesuMeParser(override val context: MangaLoaderContext) : PagedMangaParser(MangaSource.DESUME, 20) {
override val configKeyDomain = ConfigKey.Domain("desu.me", "desu.win")
override val configKeyDomain = ConfigKey.Domain("desu.me", null)
override val availableSortOrders: Set<SortOrder> = EnumSet.of(
override val sortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.UPDATED,
SortOrder.POPULARITY,
SortOrder.NEWEST,
SortOrder.ALPHABETICAL,
)
override val headers: Headers = Headers.Builder()
.add("User-Agent", UserAgents.KOTATSU)
.build()
private val tagsCache = SuspendLazy(::fetchTags)
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
tagsExclude: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
if (query != null && page != searchPaginator.firstPage) {
return emptyList()
}
val domain = domain
val domain = getDomain()
val url = buildString {
append("https://")
append(domain)
@@ -60,16 +50,14 @@ internal class DesuMeParser(context: MangaLoaderContext) : PagedMangaParser(cont
append(query)
}
}
val json = webClient.httpGet(url).parseJson().getJSONArray("response")
val json = context.httpGet(url).parseJson().getJSONArray("response")
?: throw ParseException("Invalid response", url)
val total = json.length()
val list = ArrayList<Manga>(total)
val tagsMap = tagsCache.tryGet().getOrNull()
for (i in 0 until total) {
val jo = json.getJSONObject(i)
val cover = jo.getJSONObject("image")
val id = jo.getLong("id")
val genres = jo.getString("genres").split(',')
list += Manga(
url = "/manga/api/$id",
publicUrl = jo.getString("url"),
@@ -85,13 +73,7 @@ internal class DesuMeParser(context: MangaLoaderContext) : PagedMangaParser(cont
rating = jo.getDouble("score").toFloat().coerceIn(0f, 1f),
id = generateUid(id),
isNsfw = false,
tags = if (!tagsMap.isNullOrEmpty()) {
genres.mapNotNullToSet { g ->
tagsMap[g.trim().toTitleCase()]
}
} else {
emptySet()
},
tags = emptySet(),
author = null,
description = jo.getString("description"),
)
@@ -100,8 +82,8 @@ internal class DesuMeParser(context: MangaLoaderContext) : PagedMangaParser(cont
}
override suspend fun getDetails(manga: Manga): Manga {
val url = manga.url.toAbsoluteUrl(domain)
val json = webClient.httpGet(url).parseJson().getJSONObject("response")
val url = manga.url.toAbsoluteUrl(getDomain())
val json = context.httpGet(url).parseJson().getJSONObject("response")
?: throw ParseException("Invalid response", url)
val baseChapterUrl = manga.url + "/chapter/"
val chaptersList = json.getJSONObject("chapters").getJSONArray("list")
@@ -135,13 +117,14 @@ internal class DesuMeParser(context: MangaLoaderContext) : PagedMangaParser(cont
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(domain)
val json = webClient.httpGet(fullUrl)
val fullUrl = chapter.url.toAbsoluteUrl(getDomain())
val json = context.httpGet(fullUrl)
.parseJson()
.getJSONObject("response") ?: throw ParseException("Invalid response", fullUrl)
return json.getJSONObject("pages").getJSONArray("list").mapJSON { jo ->
MangaPage(
id = generateUid(jo.getLong("id")),
referer = fullUrl,
preview = null,
source = chapter.source,
url = jo.getString("img"),
@@ -149,8 +132,22 @@ internal class DesuMeParser(context: MangaLoaderContext) : PagedMangaParser(cont
}
}
override suspend fun getAvailableTags(): Set<MangaTag> {
return tagsCache.get().values.toSet()
override suspend fun getTags(): Set<MangaTag> {
val doc = context.httpGet("https://${getDomain()}/manga/").parseHtml()
val root = doc.body().requireElementById("animeFilter")
.selectFirstOrThrow(".catalog-genres")
return root.select("li").mapToSet {
val input = it.selectFirstOrThrow("input")
MangaTag(
source = source,
key = input.attr("data-genre-slug").ifEmpty {
it.parseFailed("data-genre-slug is empty")
},
title = input.attr("data-genre-name").toTitleCase().ifEmpty {
it.parseFailed("data-genre-name is empty")
},
)
}
}
private fun getSortKey(sortOrder: SortOrder) =
@@ -161,26 +158,4 @@ internal class DesuMeParser(context: MangaLoaderContext) : PagedMangaParser(cont
SortOrder.NEWEST -> "id"
else -> "updated"
}
private suspend fun fetchTags(): Map<String, MangaTag> {
val doc = webClient.httpGet("https://$domain/manga/").parseHtml()
val root = doc.body().requireElementById("animeFilter")
.selectFirstOrThrow(".catalog-genres")
val li = root.select("li")
val result = ArrayMap<String, MangaTag>(li.size)
li.forEach {
val input = it.selectFirstOrThrow("input")
val tag = MangaTag(
source = source,
key = input.attr("data-genre-slug").ifEmpty {
it.parseFailed("data-genre-slug is empty")
},
title = input.attr("data-genre-name").toTitleCase().ifEmpty {
it.parseFailed("data-genre-name is empty")
},
)
result[tag.title] = tag
}
return result
}
}
}

279
src/main/kotlin/org/koitharu/kotatsu/parsers/site/ExHentaiParser.kt Normal file
View File

@@ -0,0 +1,279 @@
package org.koitharu.kotatsu.parsers.site
import org.jsoup.nodes.Element
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParserAuthProvider
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.exception.AuthRequiredException
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import java.util.*
import kotlin.math.pow
private const val DOMAIN_UNAUTHORIZED = "e-hentai.org"
private const val DOMAIN_AUTHORIZED = "exhentai.org"
@MangaSourceParser("EXHENTAI", "ExHentai")
internal class ExHentaiParser(
override val context: MangaLoaderContext,
) : PagedMangaParser(MangaSource.EXHENTAI, pageSize = 25), MangaParserAuthProvider {
override val sortOrders: Set<SortOrder> = Collections.singleton(
SortOrder.NEWEST,
)
override val configKeyDomain: ConfigKey.Domain
get() = ConfigKey.Domain(if (isAuthorized) DOMAIN_AUTHORIZED else DOMAIN_UNAUTHORIZED, null)
override val authUrl: String
get() = "https://${getDomain()}/bounce_login.php"
private val ratingPattern = Regex("-?[0-9]+px")
private val authCookies = arrayOf("ipb_member_id", "ipb_pass_hash")
private var updateDm = false
override val isAuthorized: Boolean
get() {
val authorized = isAuthorized(DOMAIN_UNAUTHORIZED)
if (authorized) {
if (!isAuthorized(DOMAIN_AUTHORIZED)) {
context.cookieJar.copyCookies(
DOMAIN_UNAUTHORIZED,
DOMAIN_AUTHORIZED,
authCookies,
)
context.cookieJar.insertCookies(DOMAIN_AUTHORIZED, "yay=louder")
}
return true
}
return false
}
init {
context.cookieJar.insertCookies(DOMAIN_AUTHORIZED, "nw=1", "sl=dm_2")
context.cookieJar.insertCookies(DOMAIN_UNAUTHORIZED, "nw=1", "sl=dm_2")
}
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
var search = query?.urlEncoded().orEmpty()
val url = buildString {
append("https://")
append(getDomain())
append("/?page=")
append(page)
if (!tags.isNullOrEmpty()) {
var fCats = 0
for (tag in tags) {
tag.key.toIntOrNull()?.let { fCats = fCats or it } ?: run {
search += tag.key + " "
}
}
if (fCats != 0) {
append("&f_cats=")
append(1023 - fCats)
}
}
if (search.isNotEmpty()) {
append("&f_search=")
append(search.trim().replace(' ', '+'))
}
// by unknown reason cookie "sl=dm_2" is ignored, so, we should request it again
if (updateDm) {
append("&inline_set=dm_e")
}
}
val body = context.httpGet(url).parseHtml().body()
val root = body.selectFirst("table.itg")
?.selectFirst("tbody")
?: if (updateDm) {
body.parseFailed("Cannot find root")
} else {
updateDm = true
return getListPage(page, query, tags, sortOrder)
}
updateDm = false
return root.children().mapNotNull { tr ->
if (tr.childrenSize() != 2) return@mapNotNull null
val (td1, td2) = tr.children()
val glink = td2.selectFirstOrThrow("div.glink")
val a = glink.parents().select("a").first() ?: glink.parseFailed("link not found")
val href = a.attrAsRelativeUrl("href")
val tagsDiv = glink.nextElementSibling() ?: glink.parseFailed("tags div not found")
val mainTag = td2.selectFirst("div.cn")?.let { div ->
MangaTag(
title = div.text().toTitleCase(),
key = tagIdByClass(div.classNames()) ?: return@let null,
source = source,
)
}
Manga(
id = generateUid(href),
title = glink.text().cleanupTitle(),
altTitle = null,
url = href,
publicUrl = a.absUrl("href"),
rating = td2.selectFirst("div.ir")?.parseRating() ?: RATING_UNKNOWN,
isNsfw = true,
coverUrl = td1.selectFirst("img")?.absUrl("src").orEmpty(),
tags = setOfNotNull(mainTag),
state = null,
author = tagsDiv.getElementsContainingOwnText("artist:").first()
?.nextElementSibling()?.text(),
source = source,
)
}
}
override suspend fun getDetails(manga: Manga): Manga {
val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml()
val root = doc.body().selectFirstOrThrow("div.gm")
val cover = root.getElementById("gd1")?.children()?.first()
val title = root.getElementById("gd2")
val taglist = root.getElementById("taglist")
val tabs = doc.body().selectFirst("table.ptt")?.selectFirst("tr")
return manga.copy(
title = title?.getElementById("gn")?.text()?.cleanupTitle() ?: manga.title,
altTitle = title?.getElementById("gj")?.text()?.cleanupTitle() ?: manga.altTitle,
publicUrl = doc.baseUri().ifEmpty { manga.publicUrl },
rating = root.getElementById("rating_label")?.text()
?.substringAfterLast(' ')
?.toFloatOrNull()
?.div(5f) ?: manga.rating,
largeCoverUrl = cover?.styleValueOrNull("background")?.cssUrl(),
description = taglist?.select("tr")?.joinToString("<br>") { tr ->
val (tc, td) = tr.children()
val subtags = td.select("a").joinToString { it.html() }
"<b>${tc.html()}</b> $subtags"
},
chapters = tabs?.select("a")?.findLast { a ->
a.text().toIntOrNull() != null
}?.let { a ->
val count = a.text().toInt()
val chapters = ChaptersListBuilder(count)
for (i in 1..count) {
val url = "${manga.url}?p=${i - 1}"
chapters += MangaChapter(
id = generateUid(url),
name = "${manga.title} #$i",
number = i,
url = url,
uploadDate = 0L,
source = source,
scanlator = null,
branch = null,
)
}
chapters.toList()
},
)
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val doc = context.httpGet(chapter.url.toAbsoluteUrl(getDomain())).parseHtml()
val root = doc.body().requireElementById("gdt")
return root.select("a").map { a ->
val url = a.attrAsRelativeUrl("href")
MangaPage(
id = generateUid(url),
url = url,
referer = a.absUrl("href"),
preview = null,
source = source,
)
}
}
override suspend fun getPageUrl(page: MangaPage): String {
val doc = context.httpGet(page.url.toAbsoluteUrl(getDomain())).parseHtml()
return doc.body().requireElementById("img").attrAsAbsoluteUrl("src")
}
override suspend fun getTags(): Set<MangaTag> {
val doc = context.httpGet("https://${getDomain()}").parseHtml()
val root = doc.body().requireElementById("searchbox").selectFirstOrThrow("table")
return root.select("div.cs").mapNotNullToSet { div ->
val id = div.id().substringAfterLast('_').toIntOrNull()
?: return@mapNotNullToSet null
MangaTag(
title = div.text().toTitleCase(),
key = id.toString(),
source = source,
)
}
}
override suspend fun getUsername(): String {
val doc = context.httpGet("https://forums.$DOMAIN_UNAUTHORIZED/").parseHtml().body()
val username = doc.getElementById("userlinks")
?.getElementsByAttributeValueContaining("href", "showuser=")
?.firstOrNull()
?.ownText()
?: if (doc.getElementById("userlinksguest") != null) {
throw AuthRequiredException(source)
} else {
doc.parseFailed()
}
return username
}
private fun isAuthorized(domain: String): Boolean {
val cookies = context.cookieJar.getCookies(domain).mapToSet { x -> x.name }
return authCookies.all { it in cookies }
}
private fun Element.parseRating(): Float {
return runCatching {
val style = requireNotNull(attr("style"))
val (v1, v2) = ratingPattern.find(style)!!.destructured
var p1 = v1.dropLast(2).toInt()
val p2 = v2.dropLast(2).toInt()
if (p2 != -1) {
p1 += 8
}
(80 - p1) / 80f
}.getOrDefault(RATING_UNKNOWN)
}
private fun String.cleanupTitle(): String {
val result = StringBuilder(length)
var skip = false
for (c in this) {
when {
c == '[' -> skip = true
c == ']' -> skip = false
c.isWhitespace() && result.isEmpty() -> continue
!skip -> result.append(c)
}
}
while (result.lastOrNull()?.isWhitespace() == true) {
result.deleteCharAt(result.lastIndex)
}
return result.toString()
}
private fun String.cssUrl(): String? {
val fromIndex = indexOf("url(")
if (fromIndex == -1) {
return null
}
val toIndex = indexOf(')', startIndex = fromIndex)
return if (toIndex == -1) {
null
} else {
substring(fromIndex + 4, toIndex).trim()
}
}
private fun tagIdByClass(classNames: Collection<String>): String? {
val className = classNames.find { x -> x.startsWith("ct") } ?: return null
val num = className.drop(2).toIntOrNull(16) ?: return null
return 2.0.pow(num).toInt().toString()
}
}

241
src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/MangaDexParser.kt → src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaDexParser.kt
View File

@@ -1,4 +1,4 @@
package org.koitharu.kotatsu.parsers.site.all
package org.koitharu.kotatsu.parsers.site
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.async
@@ -20,26 +20,29 @@ private const val PAGE_SIZE = 20
private const val CHAPTERS_FIRST_PAGE_SIZE = 120
private const val CHAPTERS_MAX_PAGE_SIZE = 500
private const val CHAPTERS_PARALLELISM = 3
private const val CHAPTERS_MAX_COUNT = 10_000 // strange api behavior, looks like a bug
private const val CONTENT_RATING =
"contentRating[]=safe&contentRating[]=suggestive&contentRating[]=erotica&contentRating[]=pornographic"
private const val LOCALE_FALLBACK = "en"
@MangaSourceParser("MANGADEX", "MangaDex")
internal class MangaDexParser(context: MangaLoaderContext) : MangaParser(context, MangaSource.MANGADEX) {
internal class MangaDexParser(override val context: MangaLoaderContext) : MangaParser(MangaSource.MANGADEX) {
override val configKeyDomain = ConfigKey.Domain("mangadex.org")
override val configKeyDomain = ConfigKey.Domain("mangadex.org", null)
override val availableSortOrders: EnumSet<SortOrder> = EnumSet.allOf(SortOrder::class.java)
override val sortOrders: EnumSet<SortOrder> = EnumSet.of(
SortOrder.UPDATED,
SortOrder.ALPHABETICAL,
SortOrder.NEWEST,
SortOrder.POPULARITY,
)
override val availableContentRating: Set<ContentRating> = EnumSet.allOf(ContentRating::class.java)
override val availableStates: Set<MangaState> =
EnumSet.of(MangaState.ONGOING, MangaState.FINISHED, MangaState.PAUSED, MangaState.ABANDONED)
override val isTagsExclusionSupported: Boolean = true
override suspend fun getList(offset: Int, filter: MangaListFilter?): List<Manga> {
val domain = domain
override suspend fun getList(
offset: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val domain = getDomain()
val url = buildString {
append("https://api.")
append(domain)
@@ -47,67 +50,32 @@ internal class MangaDexParser(context: MangaLoaderContext) : MangaParser(context
append(PAGE_SIZE)
append("&offset=")
append(offset)
append("&includes[]=cover_art&includes[]=author&includes[]=artist")
when (filter) {
is MangaListFilter.Search -> {
append("&title=")
append(filter.query)
}
is MangaListFilter.Advanced -> {
filter.tags.forEach {
append("&includedTags[]=")
append(it.key)
}
filter.tagsExclude.forEach {
append("&excludedTags[]=")
append(it.key)
}
if (filter.contentRating.isNotEmpty()) {
filter.contentRating.forEach {
when (it) {
ContentRating.SAFE -> append("&contentRating[]=safe")
ContentRating.SUGGESTIVE -> append("&contentRating[]=suggestive&contentRating[]=erotica")
ContentRating.ADULT -> append("&contentRating[]=pornographic")
}
}
}
append("&order")
append(
when (filter.sortOrder) {
SortOrder.UPDATED -> "[latestUploadedChapter]=desc"
SortOrder.RATING -> "[rating]=desc"
SortOrder.ALPHABETICAL -> "[title]=asc"
SortOrder.ALPHABETICAL_DESC -> "[title]=desc"
SortOrder.NEWEST -> "[createdAt]=desc"
SortOrder.POPULARITY -> "[followedCount]=desc"
},
)
filter.states.forEach {
append("&status[]=")
when (it) {
MangaState.ONGOING -> append("ongoing")
MangaState.FINISHED -> append("completed")
MangaState.ABANDONED -> append("cancelled")
MangaState.PAUSED -> append("hiatus")
else -> append("")
}
}
filter.locale?.let {
append("&availableTranslatedLanguage[]=")
append(it.language)
}
}
null -> {
append("&order[latestUploadedChapter]=desc")
}
append("&includes[]=cover_art&includes[]=author&includes[]=artist&")
tags?.forEach { tag ->
append("includedTags[]=")
append(tag.key)
append('&')
}
if (!query.isNullOrEmpty()) {
append("title=")
append(query.urlEncoded())
append('&')
}
append(CONTENT_RATING)
append("&order")
append(
when (sortOrder) {
SortOrder.UPDATED,
-> "[latestUploadedChapter]=desc"
SortOrder.ALPHABETICAL -> "[title]=asc"
SortOrder.NEWEST -> "[createdAt]=desc"
SortOrder.POPULARITY -> "[followedCount]=desc"
else -> "[followedCount]=desc"
},
)
}
val json = webClient.httpGet(url).parseJson().getJSONArray("data")
val json = context.httpGet(url).parseJson().getJSONArray("data")
return json.mapJSON { jo ->
val id = jo.getString("id")
val attrs = jo.getJSONObject("attributes")
@@ -127,10 +95,7 @@ internal class MangaDexParser(context: MangaLoaderContext) : MangaParser(context
url = id,
publicUrl = "https://$domain/title/$id",
rating = RATING_UNKNOWN,
isNsfw = when (attrs.getStringOrNull("contentRating")) {
"erotica", "pornographic" -> true
else -> false
},
isNsfw = attrs.getStringOrNull("contentRating") == "erotica",
coverUrl = cover?.plus(".256.jpg").orEmpty(),
largeCoverUrl = cover,
description = attrs.optJSONObject("description")?.selectByLocale(),
@@ -144,11 +109,9 @@ internal class MangaDexParser(context: MangaLoaderContext) : MangaParser(context
source = source,
)
},
state = when (attrs.getStringOrNull("status")) {
state = when (jo.getStringOrNull("status")) {
"ongoing" -> MangaState.ONGOING
"completed" -> MangaState.FINISHED
"hiatus" -> MangaState.PAUSED
"cancelled" -> MangaState.ABANDONED
else -> null
},
author = (relations["author"] ?: relations["artist"])
@@ -160,64 +123,80 @@ internal class MangaDexParser(context: MangaLoaderContext) : MangaParser(context
}
override suspend fun getDetails(manga: Manga): Manga = coroutineScope {
val domain = domain
val domain = getDomain()
val mangaId = manga.url.removePrefix("/")
val attrsDeferred = async {
webClient.httpGet(
context.httpGet(
"https://api.$domain/manga/${mangaId}?includes[]=artist&includes[]=author&includes[]=cover_art",
).parseJson().getJSONObject("data").getJSONObject("attributes")
}
val feedDeferred = async { loadChapters(mangaId) }
val mangaAttrs = attrsDeferred.await()
val feed = feedDeferred.await()
// 2022-01-02T00:27:11+00:00
val dateFormat = SimpleDateFormat(
"yyyy-MM-dd'T'HH:mm:ss'+00:00'",
Locale.ROOT,
)
manga.copy(
description = mangaAttrs.optJSONObject("description")?.selectByLocale()
description = mangaAttrs.getJSONObject("description").selectByLocale()
?: manga.description,
chapters = mapChapters(feed),
chapters = feed.mapChapters { _, jo ->
val id = jo.getString("id")
val attrs = jo.getJSONObject("attributes")
if (!attrs.isNull("externalUrl")) {
return@mapChapters null
}
val locale = attrs.getStringOrNull("translatedLanguage")?.let { Locale.forLanguageTag(it) }
val relations = jo.getJSONArray("relationships").associateByKey("type")
val number = attrs.getIntOrDefault("chapter", 0)
MangaChapter(
id = generateUid(id),
name = attrs.getStringOrNull("title")?.takeUnless(String::isEmpty)
?: "Chapter #$number",
number = number,
url = id,
scanlator = relations["scanlation_group"]?.getStringOrNull("name"),
uploadDate = dateFormat.tryParse(attrs.getString("publishAt")),
branch = locale?.getDisplayName(locale)?.toTitleCase(locale),
source = source,
)
},
)
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val domain = domain
val chapterJson = webClient.httpGet("https://api.$domain/at-home/server/${chapter.url}?forcePort443=false")
val domain = getDomain()
val chapterJson = context.httpGet("https://api.$domain/at-home/server/${chapter.url}?forcePort443=false")
.parseJson()
.getJSONObject("chapter")
val pages = chapterJson.getJSONArray("data")
val prefix = "https://uploads.$domain/data/${chapterJson.getString("hash")}/"
val referer = "https://$domain/"
return List(pages.length()) { i ->
val url = prefix + pages.getString(i)
MangaPage(
id = generateUid(url),
url = url,
referer = referer,
preview = null, // TODO prefix + dataSaver.getString(i),
source = source,
)
}
}
override suspend fun getAvailableTags(): Set<MangaTag> {
val tags = webClient.httpGet("https://api.${domain}/manga/tag").parseJson()
override suspend fun getTags(): Set<MangaTag> {
val tags = context.httpGet("https://api.${getDomain()}/manga/tag").parseJson()
.getJSONArray("data")
return tags.mapJSONToSet { jo ->
MangaTag(
title = jo.getJSONObject("attributes").getJSONObject("name")
.firstStringValue()
.toTitleCase(Locale.ENGLISH),
title = jo.getJSONObject("attributes").getJSONObject("name").firstStringValue().toTitleCase(),
key = jo.getString("id"),
source = source,
)
}
}
override suspend fun getAvailableLocales(): Set<Locale> {
val head = webClient.httpGet("https://$domain/").parseHtml().head()
return head.getElementsByAttributeValue("property", "og:locale:alternate")
.mapNotNullToSet { meta ->
val raw = meta.attrOrNull("content") ?: return@mapNotNullToSet null
Locale(raw.substringBefore('_'), raw.substringAfter('_', ""))
}
}
private fun JSONObject.firstStringValue() = values().next() as String
private fun JSONObject.selectByLocale(): String? {
@@ -235,7 +214,7 @@ internal class MangaDexParser(context: MangaLoaderContext) : MangaParser(context
return firstPage.data
}
val tail = coroutineScope {
val leftCount = firstPage.total.coerceAtMost(CHAPTERS_MAX_COUNT) - firstPage.size
val leftCount = firstPage.total - firstPage.size
val pages = (leftCount / CHAPTERS_MAX_PAGE_SIZE.toFloat()).toIntUp()
val dispatcher = Dispatchers.Default.limitedParallelism(CHAPTERS_PARALLELISM)
List(pages) { page ->
@@ -252,24 +231,20 @@ internal class MangaDexParser(context: MangaLoaderContext) : MangaParser(context
}
private suspend fun loadChapters(mangaId: String, offset: Int, limit: Int): Chapters {
val limitedLimit = when {
offset >= CHAPTERS_MAX_COUNT -> return Chapters(emptyList(), CHAPTERS_MAX_COUNT)
offset + limit > CHAPTERS_MAX_COUNT -> CHAPTERS_MAX_COUNT - offset
else -> limit
}
val url = buildString {
append("https://api.")
append(domain)
append(getDomain())
append("/manga/")
append(mangaId)
append("/feed")
append("?limit=")
append(limitedLimit)
append(limit)
append("&includes[]=scanlation_group&order[volume]=asc&order[chapter]=asc&offset=")
append(offset)
append("&contentRating[]=safe&contentRating[]=suggestive&contentRating[]=erotica&contentRating[]=pornographic")
append('&')
append(CONTENT_RATING)
}
val json = webClient.httpGet(url).parseJson()
val json = context.httpGet(url).parseJson()
if (json.getString("result") == "ok") {
return Chapters(
data = json.optJSONArray("data")?.toJSONList().orEmpty(),
@@ -283,48 +258,6 @@ internal class MangaDexParser(context: MangaLoaderContext) : MangaParser(context
}
}
private fun mapChapters(list: List<JSONObject>): List<MangaChapter> {
// 2022-01-02T00:27:11+00:00
val dateFormat = SimpleDateFormat(
"yyyy-MM-dd'T'HH:mm:ss'+00:00'",
Locale.ROOT,
)
val chaptersBuilder = ChaptersListBuilder(list.size)
val branchedChapters = HashMap<String?, HashMap<Float, MangaChapter>>()
for (jo in list) {
val id = jo.getString("id")
val attrs = jo.getJSONObject("attributes")
if (!attrs.isNull("externalUrl")) {
continue
}
val number = jo.getJSONObject("attributes").getFloatOrDefault("chapter", 0f)
val locale = attrs.getStringOrNull("translatedLanguage")?.let { Locale.forLanguageTag(it) }
val lc = locale?.getDisplayName(locale)?.toTitleCase(locale)
val relations = jo.getJSONArray("relationships").associateByKey("type")
val team = relations["scanlation_group"]?.getJSONObject("attributes")?.getStringOrNull("name")
?.takeUnless { it.isBlank() }
val branch = (list.indices).firstNotNullOf { i ->
val b = if (i == 0) lc else "$lc ($i)"
if (branchedChapters[b]?.get(number) == null) b else null
}
val chapter = MangaChapter(
id = generateUid(id),
name = attrs.getStringOrNull("title")?.takeUnless(String::isEmpty)
?: "Chapter #${number.toString().removeSuffix(".0")}",
number = if (number <= 0f) (branchedChapters[branch]?.size?.plus(1) ?: 0) else number.toInt(),
url = id,
scanlator = team,
uploadDate = dateFormat.tryParse(attrs.getString("publishAt")),
branch = branch,
source = source,
)
if (chaptersBuilder.add(chapter)) {
branchedChapters.getOrPut(branch, ::HashMap)[number] = chapter
}
}
return chaptersBuilder.toList()
}
private class Chapters(
val data: List<JSONObject>,
val total: Int,
@@ -333,4 +266,4 @@ internal class MangaDexParser(context: MangaLoaderContext) : MangaParser(context
val size: Int
get() = data.size
}
}
}

96
src/main/kotlin/org/koitharu/kotatsu/parsers/site/uk/MangaInUaParser.kt → src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaInUaParser.kt
View File

@@ -1,6 +1,5 @@
package org.koitharu.kotatsu.parsers.site.uk
package org.koitharu.kotatsu.parsers.site
import org.koitharu.kotatsu.parsers.ErrorMessages
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
@@ -13,42 +12,40 @@ import java.util.*
private const val DEF_BRANCH_NAME = "Основний переклад"
@MangaSourceParser("MANGAINUA", "MANGA/in/UA", "uk")
class MangaInUaParser(context: MangaLoaderContext) : PagedMangaParser(
context = context,
class MangaInUaParser(override val context: MangaLoaderContext) : PagedMangaParser(
source = MangaSource.MANGAINUA,
pageSize = 24,
searchPageSize = 10,
) {
override val availableSortOrders: Set<SortOrder> = setOf(SortOrder.UPDATED)
override val sortOrders: Set<SortOrder>
get() = Collections.singleton(SortOrder.UPDATED)
override val configKeyDomain: ConfigKey.Domain = ConfigKey.Domain("manga.in.ua")
override val isMultipleTagsSupported: Boolean = false
private val userHashRegex by lazy {
Regex("site_login_hash\\s*=\\s*\'([^\']+)\'", RegexOption.IGNORE_CASE)
}
override val configKeyDomain: ConfigKey.Domain = ConfigKey.Domain("manga.in.ua", null)
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
tagsExclude: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val url = when {
!query.isNullOrEmpty() -> ("/index.php?do=search&subaction=search&search_start=$page&full_search=1&story=$query&titleonly=3").toAbsoluteUrl(
domain,
)
!query.isNullOrEmpty() -> (
"/index.php?do=search" +
"&subaction=search" +
"&search_start=$page" +
"&full_search=1" +
"&story=$query" +
"&titleonly=3"
).toAbsoluteUrl(getDomain())
tags.isNullOrEmpty() -> "/mangas/page/$page".toAbsoluteUrl(domain)
tags.isNullOrEmpty() -> "/mangas/page/$page".toAbsoluteUrl(getDomain())
tags.size == 1 -> "${tags.first().key}/page/$page"
tags.size > 1 -> throw IllegalArgumentException(ErrorMessages.FILTER_MULTIPLE_GENRES_NOT_SUPPORTED)
else -> "/mangas/page/$page".toAbsoluteUrl(domain)
tags.size > 1 -> throw IllegalArgumentException("This source supports only 1 genre")
else -> "/mangas/page/$page".toAbsoluteUrl(getDomain())
}
val doc = webClient.httpGet(url).parseHtml()
val container = doc.body().requireElementById("site-content")
val doc = context.httpGet(url).parseHtml()
val container = doc.body().requireElementById("dle-content")
val items = container.select("div.col-6")
return items.mapNotNull { item ->
val href = item.selectFirst("a")?.attrAsRelativeUrl("href") ?: return@mapNotNull null
@@ -60,45 +57,33 @@ class MangaInUaParser(context: MangaLoaderContext) : PagedMangaParser(
}.orEmpty(),
altTitle = null,
author = null,
rating = item.selectFirst("div.card__short-rate--num")?.text()?.toFloatOrNull()?.div(10F)
?: RATING_UNKNOWN,
rating = item.selectFirst("div.card__short-rate--num")
?.text()
?.toFloatOrNull()
?.div(10F) ?: RATING_UNKNOWN,
url = href,
isNsfw = item.selectFirst("ul.card__list")?.select("li")?.lastOrNull()?.text() == "18+",
tags = runCatching {
item.selectFirst("div.card__category")?.select("a")?.mapToSet {
MangaTag(
title = it.ownText(),
key = it.attrOrThrow("href").removeSuffix("/"),
key = it.attr("href").removeSuffix("/"),
source = source,
)
}
}.getOrNull().orEmpty(),
state = null,
publicUrl = href.toAbsoluteUrl(container.host ?: domain),
publicUrl = href.toAbsoluteUrl(container.host ?: getDomain()),
source = source,
)
}
}
override suspend fun getDetails(manga: Manga): Manga {
val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml()
val root = doc.body().requireElementById("site-content")
val linkToComics = root.requireElementById("linkstocomics")
val userHash = doc.select("script").firstNotNullOf { script ->
userHashRegex.find(script.html())?.groupValues?.getOrNull(1)
}
val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml()
val root = doc.body().requireElementById("dle-content")
val dateFormat = SimpleDateFormat("dd.MM.yyyy", Locale.US)
val chaptersDoc = webClient.httpPost(
"https://$domain/engine/ajax/controller.php?mod=load_chapters",
mapOf(
"action" to "show",
"news_id" to linkToComics.attrOrThrow("data-news_id"),
"news_category" to linkToComics.attrOrThrow("data-news_category"),
"this_link" to "",
"user_hash" to userHash,
),
).parseHtml()
val chapterNodes = chaptersDoc.select(".ltcitems")
val chapterNodes = root.selectFirstOrThrow(".linkstocomics").select(".ltcitems")
var prevChapterName: String? = null
var i = 0
return manga.copy(
@@ -106,7 +91,8 @@ class MangaInUaParser(context: MangaLoaderContext) : PagedMangaParser(
largeCoverUrl = root.selectFirst("div.item__full-sidebar--poster")?.selectFirst("img")
?.attrAsAbsoluteUrlOrNull("src"),
chapters = chapterNodes.mapChapters { _, item ->
val href = item.selectFirst("a")?.attrAsRelativeUrlOrNull("href") ?: return@mapChapters null
val href = item?.selectFirst("a")?.attrAsRelativeUrlOrNull("href")
?: return@mapChapters null
val isAlternative = item.styleValueOrNull("background") != null
val name = item.selectFirst("a")?.text().orEmpty()
if (!isAlternative) i++
@@ -134,16 +120,9 @@ class MangaInUaParser(context: MangaLoaderContext) : PagedMangaParser(
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(domain)
val doc = webClient.httpGet(fullUrl).parseHtml()
val userHash = doc.select("script").firstNotNullOf { script ->
userHashRegex.find(script.html())?.groupValues?.getOrNull(1)
}
val ajaxUrl = urlBuilder().addPathSegment("engine").addPathSegment("ajax").addPathSegment("controller.php")
.addEncodedQueryParameter("mod", "load_chapters_image")
.addQueryParameter("news_id", doc.requireElementById("comics").attrOrThrow("data-news_id"))
.addEncodedQueryParameter("action", "show").addQueryParameter("user_hash", userHash).build()
val root = webClient.httpGet(ajaxUrl).parseHtml().root()
val fullUrl = chapter.url.toAbsoluteUrl(getDomain())
val doc = context.httpGet(fullUrl).parseHtml()
val root = doc.body().requireElementById("comics").selectFirstOrThrow("ul.xfieldimagegallery")
return root.select("li").map { ul ->
val img = ul.selectFirstOrThrow("img")
val url = img.attrAsAbsoluteUrl("data-src")
@@ -151,22 +130,23 @@ class MangaInUaParser(context: MangaLoaderContext) : PagedMangaParser(
id = generateUid(url),
url = url,
preview = null,
referer = fullUrl,
source = source,
)
}
}
override suspend fun getAvailableTags(): Set<MangaTag> {
val domain = domain
val doc = webClient.httpGet("https://$domain/mangas").parseHtml()
override suspend fun getTags(): Set<MangaTag> {
val domain = getDomain()
val doc = context.httpGet("https://$domain/mangas").parseHtml()
val root = doc.body().requireElementById("menu_1").selectFirstOrThrow("div.menu__wrapper")
return root.select("li").mapNotNullToSet { li ->
val a = li.selectFirst("a") ?: return@mapNotNullToSet null
MangaTag(
title = a.ownText(),
key = a.attrOrThrow("href").removeSuffix("/"),
key = a.attr("href").removeSuffix("/"),
source = source,
)
}
}
}
}

187
src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaOwlParser.kt Normal file
View File

@@ -0,0 +1,187 @@
package org.koitharu.kotatsu.parsers.site
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import java.text.SimpleDateFormat
import java.util.*
@Deprecated("Permanently shutdown")
@MangaSourceParser("MANGAOWL", "MangaOwl", "en")
internal class MangaOwlParser(override val context: MangaLoaderContext) : MangaParser(MangaSource.MANGAOWL) {
override val configKeyDomain = ConfigKey.Domain("mangaowls.com", null)
override val sortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.POPULARITY,
SortOrder.NEWEST,
SortOrder.UPDATED,
)
private val regexNsfw = Regex("(yaoi)|(yuri)|(smut)|(mature)|(adult)", RegexOption.IGNORE_CASE)
override suspend fun getList(
offset: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val page = (offset / 36f).toIntUp().inc()
val link = buildString {
append("https://")
append(getDomain())
when {
!query.isNullOrEmpty() -> {
append("/search/$page?search=")
append(query.urlEncoded())
}
!tags.isNullOrEmpty() -> {
for (tag in tags) {
append(tag.key)
}
append("/$page?type=${getAlternativeSortKey(sortOrder)}")
}
else -> {
append("/${getSortKey(sortOrder)}/$page")
}
}
}
val doc = context.httpGet(link).parseHtml()
val slides = doc.body().selectOrThrow("ul.slides")
val items = slides.select("div.col-md-2")
return items.mapNotNull { item ->
val href = item.selectFirst("h6 a")?.attrAsRelativeUrlOrNull("href") ?: return@mapNotNull null
Manga(
id = generateUid(href),
title = item.selectFirst("h6 a")?.text() ?: return@mapNotNull null,
coverUrl = item.select("div.img-responsive").attr("abs:data-background-image"),
altTitle = null,
author = null,
rating = runCatching {
item.selectFirst("div.block-stars")
?.text()
?.toFloatOrNull()
?.div(10f)
}.getOrNull() ?: RATING_UNKNOWN,
url = href,
isNsfw = false,
tags = emptySet(),
state = null,
publicUrl = href.toAbsoluteUrl(getDomain()),
source = source,
)
}
}
override suspend fun getDetails(manga: Manga): Manga {
val doc = context.httpGet(manga.publicUrl).parseHtml()
val info = doc.body().selectFirstOrThrow("div.single_detail")
val table = doc.body().selectFirstOrThrow("div.single-grid-right")
val dateFormat = SimpleDateFormat("MM/dd/yyyy", Locale.US)
val trRegex = "window\\['tr'] = '([^']*)';".toRegex(RegexOption.IGNORE_CASE)
val trElement = doc.getElementsByTag("script").find { trRegex.find(it.data()) != null }
?: doc.parseFailed("Oops, tr not found")
val tr = trRegex.find(trElement.data())!!.groups[1]!!.value
val s = context.encodeBase64(getDomain().toByteArray())
var isNsfw = manga.isNsfw
val parsedTags = info.select("div.col-xs-12.col-md-8.single-right-grid-right > p > a[href*=genres]")
.mapNotNullToSet {
val a = it.selectFirst("a") ?: return@mapNotNullToSet null
val name = a.text()
if (!isNsfw && isNsfwGenre(name)) {
isNsfw = true
}
MangaTag(
title = name.toTitleCase(),
key = a.attr("href"),
source = source,
)
}
return manga.copy(
description = info.selectFirst(".description")?.html(),
largeCoverUrl = info.select("img").first()?.let { img ->
if (img.hasAttr("data-src")) img.attr("abs:data-src") else img.attr("abs:src")
},
isNsfw = isNsfw,
author = info.selectFirst("p.fexi_header_para a.author_link")?.text(),
state = parseStatus(info.select("p.fexi_header_para:contains(status)").first()?.ownText()),
tags = manga.tags + parsedTags,
chapters = table.select("div.table.table-chapter-list").select("li.list-group-item.chapter_list")
.asReversed().mapChapters { i, li ->
val a = li.select("a")
val href = a.attr("data-href").ifEmpty {
li.parseFailed("Link is missing")
}
MangaChapter(
id = generateUid(href),
name = a.select("label").text(),
number = i + 1,
url = "$href?tr=$tr&s=$s",
scanlator = null,
branch = null,
uploadDate = dateFormat.tryParse(li.selectFirst("small:last-of-type")?.text()),
source = MangaSource.MANGAOWL,
)
},
)
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(getDomain())
val doc = context.httpGet(fullUrl).parseHtml()
val root = doc.body().selectOrThrow("div.item img.owl-lazy")
return root.map { div ->
val url = div?.attrAsRelativeUrlOrNull("data-src") ?: doc.parseFailed("Page image not found")
MangaPage(
id = generateUid(url),
url = url,
preview = null,
referer = url,
source = MangaSource.MANGAOWL,
)
}
}
private fun parseStatus(status: String?) = when {
status == null -> null
status.contains("Ongoing") -> MangaState.ONGOING
status.contains("Completed") -> MangaState.FINISHED
else -> null
}
override suspend fun getTags(): Set<MangaTag> {
val doc = context.httpGet("https://${getDomain()}/").parseHtml()
val root = doc.body().select("ul.dropdown-menu.multi-column.columns-3").select("li")
return root.mapToSet { p ->
val a = p.selectFirstOrThrow("a")
MangaTag(
title = a.text().toTitleCase(),
key = a.attr("href"),
source = source,
)
}
}
private fun getSortKey(sortOrder: SortOrder) =
when (sortOrder) {
SortOrder.POPULARITY -> "popular"
SortOrder.NEWEST -> "new_release"
SortOrder.UPDATED -> "lastest"
else -> "lastest"
}
private fun getAlternativeSortKey(sortOrder: SortOrder) =
when (sortOrder) {
SortOrder.POPULARITY -> "0"
SortOrder.NEWEST -> "2"
SortOrder.UPDATED -> "3"
else -> "3"
}
private fun isNsfwGenre(name: String): Boolean = regexNsfw.containsMatchIn(name)
}

124
src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/MangaTownParser.kt → src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaTownParser.kt
View File

@@ -1,4 +1,4 @@
package org.koitharu.kotatsu.parsers.site.en
package org.koitharu.kotatsu.parsers.site
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser
@@ -11,83 +11,49 @@ import java.text.SimpleDateFormat
import java.util.*
@MangaSourceParser("MANGATOWN", "MangaTown", "en")
internal class MangaTownParser(context: MangaLoaderContext) : MangaParser(context, MangaSource.MANGATOWN) {
internal class MangaTownParser(override val context: MangaLoaderContext) : MangaParser(MangaSource.MANGATOWN) {
override val configKeyDomain = ConfigKey.Domain("www.mangatown.com")
override val configKeyDomain = ConfigKey.Domain("www.mangatown.com", null)
override val availableSortOrders: Set<SortOrder> = EnumSet.of(
override val sortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.ALPHABETICAL,
SortOrder.RATING,
SortOrder.POPULARITY,
SortOrder.UPDATED,
)
override val availableStates: Set<MangaState> = EnumSet.of(
MangaState.ONGOING,
MangaState.FINISHED,
)
private val regexTag = Regex("[^\\-]+-[^\\-]+-[^\\-]+-[^\\-]+-[^\\-]+-[^\\-]+")
override val isMultipleTagsSupported = false
override suspend fun getList(offset: Int, filter: MangaListFilter?): List<Manga> {
override suspend fun getList(
offset: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val sortKey = when (sortOrder) {
SortOrder.ALPHABETICAL -> "?name.az"
SortOrder.RATING -> "?rating.za"
SortOrder.UPDATED -> "?last_chapter_time.za"
else -> ""
}
val page = (offset / 30) + 1
val url = buildString {
append("https://")
append(domain)
when (filter) {
is MangaListFilter.Search -> {
append("/search?name=")
append(filter.query.urlEncoded())
append("&page=")
append(page.toString())
val url = when {
!query.isNullOrEmpty() -> {
if (offset != 0) {
return emptyList()
}
"/search?name=${query.urlEncoded()}".toAbsoluteUrl(getDomain())
}
is MangaListFilter.Advanced -> {
append("/directory/")
append("0-")
if (filter.tags.isNotEmpty()) {
filter.tags.oneOrThrowIfMany()?.let {
append(it.key)
}
} else {
append("0")
}
append("-0-")
if (filter.states.isNotEmpty()) {
filter.states.oneOrThrowIfMany()?.let {
append(
when (it) {
MangaState.ONGOING -> "ongoing"
MangaState.FINISHED -> "completed"
else -> "0"
},
)
}
} else {
append("0")
}
append("-0-0/")
append(page.toString())
append(".htm")
append(
when (filter.sortOrder) {
SortOrder.POPULARITY -> ""
SortOrder.UPDATED -> "?last_chapter_time.za"
SortOrder.ALPHABETICAL -> "?name.az"
SortOrder.RATING -> "?rating.za"
else -> "?last_chapter_time.za"
},
)
}
null -> append("/directory/$page.htm?last_chapter_time.za")
tags.isNullOrEmpty() -> "/directory/$page.htm$sortKey".toAbsoluteUrl(getDomain())
tags.size == 1 -> "/directory/${tags.first().key}/$page.htm$sortKey".toAbsoluteUrl(getDomain())
else -> tags.joinToString(
prefix = "/search?page=$page".toAbsoluteUrl(getDomain()),
) { tag ->
"&genres[${tag.key}]=1"
}
}
val doc = webClient.httpGet(url).parseHtml()
val doc = context.httpGet(url).parseHtml()
val root = doc.body().selectFirstOrThrow("ul.manga_pic_list")
return root.select("li").mapNotNull { li ->
val a = li.selectFirst("a.manga_cover")
@@ -115,19 +81,19 @@ internal class MangaTownParser(context: MangaLoaderContext) : MangaParser(contex
tags = li.selectFirst("p.keyWord")?.select("a")?.mapNotNullToSet tags@{ x ->
MangaTag(
title = x.attr("title").toTitleCase(),
key = x.attr("href").substringAfter("/directory/0-").substringBefore("-0-"),
key = x.attr("href").parseTagKey() ?: return@tags null,
source = MangaSource.MANGATOWN,
)
}.orEmpty(),
url = href,
isNsfw = false,
publicUrl = href.toAbsoluteUrl(a.host ?: domain),
publicUrl = href.toAbsoluteUrl(a.host ?: getDomain()),
)
}
}
override suspend fun getDetails(manga: Manga): Manga {
val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml()
val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml()
val root = doc.body().selectFirstOrThrow("section.main")
.selectFirstOrThrow("div.article_content")
val info = root.selectFirst("div.detail_info")?.selectFirst("ul")
@@ -140,7 +106,7 @@ internal class MangaTownParser(context: MangaLoaderContext) : MangaParser(contex
}?.select("a")?.mapNotNull { a ->
MangaTag(
title = a.attr("title").toTitleCase(),
key = a.attr("href").substringAfter("/directory/0-").substringBefore("-0-"),
key = a.attr("href").parseTagKey() ?: return@mapNotNull null,
source = MangaSource.MANGATOWN,
)
}.orEmpty(),
@@ -169,8 +135,8 @@ internal class MangaTownParser(context: MangaLoaderContext) : MangaParser(contex
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(domain)
val doc = webClient.httpGet(fullUrl).parseHtml()
val fullUrl = chapter.url.toAbsoluteUrl(getDomain())
val doc = context.httpGet(fullUrl).parseHtml()
val root = doc.body().selectFirstOrThrow("div.page_select")
return root.selectFirstOrThrow("select").selectOrThrow("option").mapNotNull {
val href = it.attrAsRelativeUrlOrNull("value")
@@ -181,25 +147,29 @@ internal class MangaTownParser(context: MangaLoaderContext) : MangaParser(contex
id = generateUid(href),
url = href,
preview = null,
referer = fullUrl,
source = MangaSource.MANGATOWN,
)
}
}
override suspend fun getPageUrl(page: MangaPage): String {
val doc = webClient.httpGet(page.url.toAbsoluteUrl(domain)).parseHtml()
val doc = context.httpGet(page.url.toAbsoluteUrl(getDomain())).parseHtml()
return doc.requireElementById("image").attrAsAbsoluteUrl("src")
}
override suspend fun getAvailableTags(): Set<MangaTag> {
val doc = webClient.httpGet("/directory/".toAbsoluteUrl(domain)).parseHtml()
override suspend fun getTags(): Set<MangaTag> {
val doc = context.httpGet("/directory/".toAbsoluteUrl(getDomain())).parseHtml()
val root = doc.body().selectFirst("aside.right")
?.getElementsContainingOwnText("Genres")
?.first()
?.nextElementSibling() ?: doc.parseFailed("Root not found")
return root.select("li").mapNotNullToSet { li ->
val a = li.selectFirst("a") ?: return@mapNotNullToSet null
val key = a.attr("href").substringAfter("/directory/0-").substringBefore("-0-")
val key = a.attr("href").parseTagKey()
if (key.isNullOrEmpty()) {
return@mapNotNullToSet null
}
MangaTag(
source = MangaSource.MANGATOWN,
key = key,
@@ -218,7 +188,7 @@ internal class MangaTownParser(context: MangaLoaderContext) : MangaParser(contex
}
private suspend fun bypassLicensedChapters(manga: Manga): List<MangaChapter> {
val doc = webClient.httpGet(manga.url.toAbsoluteUrl(getDomain("m"))).parseHtml()
val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain("m"))).parseHtml()
val list = doc.body().selectFirst("ul.detail-ch-list") ?: return emptyList()
val dateFormat = SimpleDateFormat("MMM dd,yyyy", Locale.US)
return list.select("li").asReversed().mapIndexedNotNull { i, li ->
@@ -242,4 +212,6 @@ internal class MangaTownParser(context: MangaLoaderContext) : MangaParser(contex
)
}
}
}
private fun String.parseTagKey() = split('/').findLast { regexTag matches it }
}

188
src/main/kotlin/org/koitharu/kotatsu/parsers/site/NHentaiParser.kt Normal file
View File

@@ -0,0 +1,188 @@
package org.koitharu.kotatsu.parsers.site
import androidx.collection.ArraySet
import kotlinx.coroutines.async
import kotlinx.coroutines.awaitAll
import kotlinx.coroutines.coroutineScope
import org.jsoup.nodes.Element
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import java.text.SimpleDateFormat
import java.util.*
@MangaSourceParser("NHENTAI", "N-Hentai")
class NHentaiParser(override val context: MangaLoaderContext) : PagedMangaParser(MangaSource.NHENTAI, pageSize = 25) {
override val configKeyDomain: ConfigKey.Domain
get() = ConfigKey.Domain("nhentai.net", null)
override val sortOrders: Set<SortOrder>
get() = EnumSet.of(SortOrder.NEWEST, SortOrder.POPULARITY)
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
if (query.isNullOrEmpty() && tags != null && tags.size > 1) {
return getListPage(page, buildQuery(tags), emptySet(), sortOrder)
}
val domain = getDomain()
val url = buildString {
append("https://")
append(domain)
if (!query.isNullOrEmpty()) {
append("/search/?q=")
append(query.urlEncoded())
append("&page=")
append(page)
if (sortOrder == SortOrder.POPULARITY) {
append("&sort=popular")
}
} else {
append('/')
if (!tags.isNullOrEmpty()) {
val tag = tags.single()
append("tag/")
append(tag.key)
append('/')
if (sortOrder == SortOrder.POPULARITY) {
append("popular")
}
append("?page=")
append(page)
} else {
if (sortOrder == SortOrder.POPULARITY) {
append("?sort=popular&page=")
} else {
append("?page=")
}
append(page)
}
}
}
val root = context.httpGet(url).parseHtml().body().requireElementById("content")
.selectLastOrThrow("div.index-container")
val regexBrackets = Regex("\\[[^]]+]|\\([^)]+\\)")
val regexSpaces = Regex("\\s+")
return root.select(".gallery").map { div ->
val a = div.selectFirstOrThrow("a.cover")
val href = a.attrAsRelativeUrl("href")
val img = div.selectFirstOrThrow("img")
val title = div.selectFirstOrThrow(".caption").text()
Manga(
id = generateUid(href),
title = title.replace(regexBrackets, "")
.replace(regexSpaces, " ")
.trim(),
altTitle = null,
url = href,
publicUrl = href.toAbsoluteUrl(domain),
rating = RATING_UNKNOWN,
isNsfw = true,
coverUrl = img.attrAsAbsoluteUrlOrNull("data-src")
?: img.attrAsAbsoluteUrl("src"),
tags = setOf(),
state = null,
author = null,
largeCoverUrl = null,
description = null,
chapters = listOf(),
source = source,
)
}
}
override suspend fun getDetails(manga: Manga): Manga {
val root = context.httpGet(
url = manga.url.toAbsoluteUrl(getDomain()),
).parseHtml().body().requireElementById("bigcontainer")
val img = root.requireElementById("cover").selectFirstOrThrow("img")
val tagContainers = root.requireElementById("tags").select(".tag-container")
val dateFormat = SimpleDateFormat(
"yyyy-MM-dd'T'HH:mm:ss.SSSSSS'+00:00'",
Locale.ROOT,
)
return manga.copy(
tags = tagContainers.find { x -> x.ownText() == "Tags:" }?.parseTags() ?: manga.tags,
author = tagContainers.find { x -> x.ownText() == "Artists:" }
?.selectFirst("span.name")?.text()?.toCamelCase(),
largeCoverUrl = img.attrAsAbsoluteUrlOrNull("data-src")
?: img.attrAsAbsoluteUrl("src"),
description = null,
chapters = listOf(
MangaChapter(
id = manga.id,
name = manga.title,
number = 1,
url = manga.url,
scanlator = null,
uploadDate = dateFormat.tryParse(
tagContainers.find { x -> x.ownText() == "Uploaded:" }
?.selectFirst("time")
?.attr("datetime"),
),
branch = null,
source = source,
),
),
)
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val url = chapter.url.toAbsoluteUrl(getDomain())
val root = context.httpGet(url).parseHtml().requireElementById("thumbnail-container")
return root.select(".thumb-container").map { div ->
val a = div.selectFirstOrThrow("a")
val img = div.selectFirstOrThrow("img")
val href = a.attrAsRelativeUrl("href")
MangaPage(
id = generateUid(href),
url = href,
referer = url,
preview = img.attrAsAbsoluteUrlOrNull("data-src")
?: img.attrAsAbsoluteUrl("src"),
source = source,
)
}
}
override suspend fun getPageUrl(page: MangaPage): String {
val root = context.httpGet(page.url.toAbsoluteUrl(getDomain())).parseHtml().body()
.requireElementById("image-container")
return root.selectFirstOrThrow("img").attrAsAbsoluteUrl("src")
}
override suspend fun getTags(): Set<MangaTag> {
return coroutineScope {
// parse first 3 pages of tags
(1..3).map { page ->
async { getTags(page) }
}
}.awaitAll().flattenTo(ArraySet(360))
}
private suspend fun getTags(page: Int): Set<MangaTag> {
val root = context.httpGet("https://${getDomain()}/tags/popular?page=$page").parseHtml().body()
.getElementById("tag-container")
return root?.parseTags().orEmpty()
}
private fun Element.parseTags() = select("a.tag").mapToSet { a ->
val href = a.attr("href").removeSuffix('/')
MangaTag(
title = a.selectFirstOrThrow(".name").text().toTitleCase(),
key = href.substringAfterLast('/'),
source = source,
)
}
private fun buildQuery(tags: Collection<MangaTag>) = tags.joinToString(separator = " ") { tag ->
"tag:\"${tag.key}\""
}
}

221
src/main/kotlin/org/koitharu/kotatsu/parsers/site/NetTruyenParser.kt Normal file
View File

@@ -0,0 +1,221 @@
package org.koitharu.kotatsu.parsers.site
import androidx.collection.ArrayMap
import androidx.collection.ArraySet
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
import org.koitharu.kotatsu.parsers.*
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.exception.NotFoundException
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import java.text.SimpleDateFormat
import java.util.*
@MangaSourceParser("NETTRUYEN", "NetTruyen", "vi")
class NetTruyenParser(override val context: MangaLoaderContext) :
PagedMangaParser(MangaSource.NETTRUYEN, pageSize = 36) {
override val configKeyDomain: ConfigKey.Domain
get() = ConfigKey.Domain("www.nettruyenme.com", null)
override val sortOrders: Set<SortOrder>
get() = EnumSet.of(SortOrder.UPDATED, SortOrder.POPULARITY, SortOrder.NEWEST, SortOrder.RATING)
private val mutex = Mutex()
private val dateFormat = SimpleDateFormat("dd/MM/yy", Locale.US)
private var tagCache: ArrayMap<String, MangaTag>? = null
override suspend fun getDetails(manga: Manga): Manga {
val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml()
val rating = doc.selectFirst("span[itemprop=ratingValue]")
?.ownText()
?.toFloatOrNull() ?: 0f
val chapterElements = doc.getElementById("nt_listchapter")?.select("ul > li") ?: doc.parseFailed()
val chapters = chapterElements.asReversed().mapChapters { index, element ->
val a = element.selectFirst("div.chapter > a") ?: return@mapChapters null
val relativeUrl = a.attrAsRelativeUrlOrNull("href") ?: return@mapChapters null
val timeText = element.selectFirst("div.col-xs-4.text-center.no-wrap.small")?.text()
MangaChapter(
id = generateUid(relativeUrl),
name = a.text(),
number = index + 1,
url = relativeUrl,
scanlator = null,
uploadDate = parseChapterTime(timeText),
branch = null,
source = source,
)
}
return manga.copy(
rating = rating / 5,
chapters = chapters,
description = doc.selectFirst("div.detail-content > p")?.html(),
isNsfw = doc.selectFirst("div.alert.alert-danger > strong:contains(Cảnh báo độ tuổi)") != null,
)
}
// 20 giây trước
// 52 phút trước
// 6 giờ trước
// 2 ngày trước
// 19:09 30/07
// 23/12/21
private fun parseChapterTime(timeText: String?): Long {
if (timeText.isNullOrEmpty()) {
return 0L
}
val timeWords = arrayOf("giây", "phút", "giờ", "ngày")
val calendar = Calendar.getInstance()
val timeArr = timeText.split(' ')
if (WordSet(*timeWords).anyWordIn(timeText)) {
val timeSuffix = timeArr.getOrNull(1)
val timeDiff = timeArr.getOrNull(0)?.toIntOrNull() ?: return 0L
when (timeSuffix) {
timeWords[0] -> calendar.add(Calendar.SECOND, -timeDiff)
timeWords[1] -> calendar.add(Calendar.MINUTE, -timeDiff)
timeWords[2] -> calendar.add(Calendar.HOUR, -timeDiff)
timeWords[3] -> calendar.add(Calendar.DATE, -timeDiff)
else -> return 0L
}
} else {
val relativeDate = timeArr.lastOrNull() ?: return 0L
val dateString = when (relativeDate.split('/').size) {
2 -> {
val currentYear = calendar.get(Calendar.YEAR).toString().takeLast(2)
"$relativeDate/$currentYear"
}
3 -> relativeDate
else -> return 0L
}
calendar.timeInMillis = dateFormat.tryParse(dateString)
}
return calendar.time.time
}
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val isSearching = !query.isNullOrEmpty()
val url = buildString {
append("https://")
append(getDomain())
if (isSearching) {
append("/tim-truyen?keyword=")
append(query!!.urlEncoded())
append("&page=")
append(page)
} else {
val tagQuery = tags.orEmpty().joinToString(",") { it.key }
append("/tim-truyen-nang-cao?genres=$tagQuery")
append("&notgenres=&gender=-1&status=-1&minchapter=1&sort=${getSortOrderKey(sortOrder)}")
append("&page=$page")
}
}
val response = if (isSearching) {
val result = runCatching { context.httpGet(url) }
val exception = result.exceptionOrNull()
if (exception is NotFoundException) {
return emptyList()
}
result.getOrThrow()
} else {
context.httpGet(url)
}
val itemsElements = response.parseHtml()
.select("div.ModuleContent > div.items")
.select("div.item")
return itemsElements.mapNotNull { item ->
val tooltipElement = item.selectFirst("div.box_tootip") ?: return@mapNotNull null
val absUrl = item.selectFirst("div.image > a")?.attrAsAbsoluteUrlOrNull("href") ?: return@mapNotNull null
val slug = absUrl.substringAfterLast('/')
val mangaState = when (tooltipElement.selectFirst("div.message_main > p:contains(Tình trạng)")?.ownText()) {
"Đang tiến hành" -> MangaState.ONGOING
"Hoàn thành" -> MangaState.FINISHED
else -> null
}
val tagMap = getOrCreateTagMap()
val tagsElement = tooltipElement.selectFirst("div.message_main > p:contains(Thể loại)")?.ownText().orEmpty()
val mangaTags = tagsElement.split(',').mapNotNullToSet { tagMap[it.trim()] }
Manga(
id = generateUid(slug),
title = tooltipElement.selectFirst("div.title")?.text().orEmpty(),
altTitle = null,
url = absUrl.toRelativeUrl(getDomain()),
publicUrl = absUrl,
rating = RATING_UNKNOWN,
isNsfw = false,
coverUrl = item.selectFirst("div.image a img")?.absUrl("data-original").orEmpty(),
largeCoverUrl = null,
tags = mangaTags,
state = mangaState,
author = tooltipElement.selectFirst("div.message_main > p:contains(Tác giả)")?.ownText(),
description = tooltipElement.selectFirst("div.box_text")?.text(),
chapters = null,
source = source,
)
}
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val pageElements = context.httpGet(chapter.url.toAbsoluteUrl(getDomain())).parseHtml()
.select("div.reading-detail.box_doc > div img")
return pageElements.map { element ->
val url = element.attrAsAbsoluteUrl("data-original")
MangaPage(
id = generateUid(url),
url = url,
referer = getDomain(),
preview = null,
source = source,
)
}
}
override suspend fun getTags(): Set<MangaTag> {
val map = getOrCreateTagMap()
val tagSet = ArraySet<MangaTag>(map.size)
for (entry in map) {
tagSet.add(entry.value)
}
return tagSet
}
private suspend fun getOrCreateTagMap(): ArrayMap<String, MangaTag> = mutex.withLock {
tagCache?.let { return@withLock it }
val doc = context.httpGet("/tim-truyen-nang-cao".toAbsoluteUrl(getDomain())).parseHtml()
val tagItems = doc.select("div.genre-item")
val result = ArrayMap<String, MangaTag>(tagItems.size)
for (item in tagItems) {
val title = item.text().trim()
val key = item.select("span[data-id]").attr("data-id")
result[title] = MangaTag(title = title, key = key, source = source)
}
tagCache = result
result
}
private fun getSortOrderKey(sortOrder: SortOrder) = when (sortOrder) {
SortOrder.UPDATED -> 0
SortOrder.POPULARITY -> 10
SortOrder.NEWEST -> 15
SortOrder.RATING -> 20
else -> throw IllegalArgumentException("Sort order ${sortOrder.name} not supported")
}
}

80
src/main/kotlin/org/koitharu/kotatsu/parsers/site/ja/NicovideoSeigaParser.kt → src/main/kotlin/org/koitharu/kotatsu/parsers/site/NicovideoSeigaParser.kt
View File

@@ -1,26 +1,20 @@
package org.koitharu.kotatsu.parsers.site.ja
package org.koitharu.kotatsu.parsers.site
import okhttp3.Headers
import org.koitharu.kotatsu.parsers.*
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.exception.AuthRequiredException
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.network.UserAgents
import org.koitharu.kotatsu.parsers.util.*
import java.util.*
private const val STATUS_ONGOING = "連載"
private const val STATUS_FINISHED = "完結"
@MangaSourceParser("NICOVIDEO_SEIGA", "NicoVideo Seiga", "ja")
class NicovideoSeigaParser(context: MangaLoaderContext) :
MangaParser(context, MangaSource.NICOVIDEO_SEIGA),
@MangaSourceParser("NICOVIDEO_SEIGA", "Nicovideo Seiga", "ja")
class NicovideoSeigaParser(override val context: MangaLoaderContext) :
MangaParser(MangaSource.NICOVIDEO_SEIGA),
MangaParserAuthProvider {
override val headers: Headers = Headers.Builder()
.add("User-Agent", UserAgents.CHROME_DESKTOP)
.build()
override val authUrl: String
get() = "https://${getDomain("account")}/login?site=seiga"
@@ -30,47 +24,36 @@ class NicovideoSeigaParser(context: MangaLoaderContext) :
}
override suspend fun getUsername(): String {
val body = webClient.httpGet("https://${getDomain("app")}/my/apps").parseHtml().body()
val body = context.httpGet("https://${getDomain("app")}/my/apps").parseHtml().body()
return body.selectFirst("#userinfo > div > div > strong")?.text() ?: throw AuthRequiredException(source)
}
override val availableSortOrders: Set<SortOrder> = EnumSet.of(
override val sortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.UPDATED,
SortOrder.POPULARITY,
)
override val isMultipleTagsSupported = false
override val configKeyDomain: ConfigKey.Domain = ConfigKey.Domain("nicovideo.jp")
override val configKeyDomain: ConfigKey.Domain = ConfigKey.Domain("nicovideo.jp", null)
@InternalParsersApi
override suspend fun getList(offset: Int, filter: MangaListFilter?): List<Manga> {
override suspend fun getList(
offset: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val page = (offset / 20f).toIntUp().inc()
val domain = getDomain("seiga")
val url =
when (filter) {
is MangaListFilter.Search -> {
return if (offset == 0) getSearchList(filter.query, page) else emptyList()
}
val url = when {
!query.isNullOrEmpty() -> return if (offset == 0) getSearchList(query, page) else emptyList()
tags.isNullOrEmpty() -> "https://$domain/manga/list?page=$page&sort=${getSortKey(sortOrder)}"
tags.size == 1 -> "https://$domain/manga/list?category=${tags.first().key}&page=$page" +
"&sort=${getSortKey(sortOrder)}"
is MangaListFilter.Advanced -> {
if (filter.tags.isNotEmpty()) {
filter.tags.oneOrThrowIfMany().let {
"https://$domain/manga/list?category=${it?.key}&page=$page&sort=${getSortKey(filter.sortOrder)}"
}
} else {
"https://$domain/manga/list?page=$page&sort=${getSortKey(filter.sortOrder)}"
}
}
null -> "https://$domain/manga/list?page=$page"
}
val doc = webClient.httpGet(url).parseHtml()
tags.size > 1 -> throw IllegalArgumentException("This source supports only 1 category")
else -> "https://$domain/manga/list?page=$page&sort=${getSortKey(sortOrder)}"
}
val doc = context.httpGet(url).parseHtml()
val comicList = doc.body().select("#comic_list > ul > li") ?: doc.parseFailed("Container not found")
val items = comicList.select("div > .description > div > div")
return items.mapNotNull { item ->
@@ -105,8 +88,8 @@ class NicovideoSeigaParser(context: MangaLoaderContext) :
}
override suspend fun getDetails(manga: Manga): Manga {
val doc = webClient.httpGet(manga.url.toAbsoluteUrl(getDomain("seiga"))).parseHtml()
val contents = doc.body().requireElementById("contents")
val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain("seiga"))).parseHtml()
val contents = doc.body().selectFirstOrThrow("#contents")
val statusText = contents
.select("div.mg_work_detail > div > div:nth-child(2) > div.tip.content_status.status_series > span")
.text()
@@ -139,7 +122,7 @@ class NicovideoSeigaParser(context: MangaLoaderContext) :
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(getDomain("seiga"))
val doc = webClient.httpGet(fullUrl).parseHtml()
val doc = context.httpGet(fullUrl).parseHtml()
if (!doc.select("#login_manga").isEmpty())
throw AuthRequiredException(source)
val root = doc.body().select("#page_contents > li")
@@ -149,19 +132,20 @@ class NicovideoSeigaParser(context: MangaLoaderContext) :
id = generateUid(url),
url = url,
preview = null,
referer = fullUrl,
source = source,
)
}
}
override suspend fun getAvailableTags(): Set<MangaTag> {
val doc = webClient.httpGet("https://${getDomain("seiga")}/manga/list").parseHtml()
val root = doc.body().selectOrThrow("#mg_category_list > ul > li").drop(1)
override suspend fun getTags(): Set<MangaTag> {
val doc = context.httpGet("https://${getDomain("seiga")}/manga/list").parseHtml()
val root = doc.body().selectOrThrow("#mg_category_list > ul > li")
return root.mapToSet { li ->
val a = li.selectFirstOrThrow("a")
MangaTag(
title = a.text(),
key = a.attrAsRelativeUrl("href").substringAfter("category=").substringBefore("&"),
key = a.attrAsRelativeUrlOrNull("href").orEmpty(),
source = source,
)
}
@@ -169,7 +153,7 @@ class NicovideoSeigaParser(context: MangaLoaderContext) :
private suspend fun getSearchList(query: String, page: Int): List<Manga> {
val domain = getDomain("seiga")
val doc = webClient.httpGet("https://$domain/manga/search/?q=$query&page=$page&sort=score").parseHtml()
val doc = context.httpGet("https://$domain/manga/search/?q=$query&page=$page&sort=score").parseHtml()
val root = doc.body().select(".search_result__item")
return root.mapNotNull { item ->
val href = item.selectFirst(".search_result__item__thumbnail > a")
@@ -198,4 +182,4 @@ class NicovideoSeigaParser(context: MangaLoaderContext) :
SortOrder.UPDATED -> "manga_updated"
else -> "manga_view"
}
}
}

182
src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/NineMangaParser.kt → src/main/kotlin/org/koitharu/kotatsu/parsers/site/NineMangaParser.kt
View File

@@ -1,11 +1,7 @@
package org.koitharu.kotatsu.parsers.site.all
package org.koitharu.kotatsu.parsers.site
import androidx.collection.ArrayMap
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
import okhttp3.Headers
import okhttp3.HttpUrl.Companion.toHttpUrl
import okhttp3.Interceptor
import okhttp3.Response
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
@@ -16,92 +12,64 @@ import java.text.SimpleDateFormat
import java.util.*
internal abstract class NineMangaParser(
context: MangaLoaderContext,
final override val context: MangaLoaderContext,
source: MangaSource,
defaultDomain: String,
) : PagedMangaParser(context, source, pageSize = 26), Interceptor {
) : PagedMangaParser(source, pageSize = 26) {
override val configKeyDomain = ConfigKey.Domain(defaultDomain)
override val configKeyDomain = ConfigKey.Domain(defaultDomain, null)
init {
context.cookieJar.insertCookies(domain, "ninemanga_template_desk=yes")
context.cookieJar.insertCookies(getDomain(), "ninemanga_template_desk=yes")
}
override val headers = super.headers.newBuilder()
override val headers = Headers.Builder()
.add("Accept-Language", "en-US;q=0.7,en;q=0.3")
.build()
override val availableSortOrders: Set<SortOrder> = Collections.singleton(
override val sortOrders: Set<SortOrder> = Collections.singleton(
SortOrder.POPULARITY,
)
override val availableStates: Set<MangaState> = EnumSet.of(
MangaState.ONGOING,
MangaState.FINISHED,
)
override val isTagsExclusionSupported: Boolean = true
override fun intercept(chain: Interceptor.Chain): Response {
val request = chain.request()
val newRequest = if (request.url.host == domain) {
request.newBuilder().removeHeader("Referer").build()
} else {
request
}
return chain.proceed(newRequest)
}
override suspend fun getListPage(page: Int, filter: MangaListFilter?): List<Manga> {
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val url = buildString {
append("https://")
append(domain)
when (filter) {
is MangaListFilter.Search -> {
append(getDomain())
when {
!query.isNullOrEmpty() -> {
append("/search/?name_sel=&wd=")
append(filter.query.urlEncoded())
append(query.urlEncoded())
append("&page=")
append(page)
append(".html")
}
is MangaListFilter.Advanced -> {
if (filter.tags.isNotEmpty() || filter.tagsExclude.isNotEmpty() || filter.states.isNotEmpty()) {
append("/search/?category_id=")
append(filter.tags.joinToString(separator = ",") { it.key })
append("&out_category_id=")
append(filter.tagsExclude.joinToString(separator = ",") { it.key })
filter.states.oneOrThrowIfMany()?.let {
append("&completed_series=")
when (it) {
MangaState.ONGOING -> append("no")
MangaState.FINISHED -> append("yes")
else -> append("either")
}
}
append("&page=")
} else {
append("/category/index_")
!tags.isNullOrEmpty() -> {
append("/search/?category_id=")
for (tag in tags) {
append(tag.key)
append(',')
}
append(page.toString())
append(".html")
append("&page=")
}
null -> {
else -> {
append("/category/index_")
append(page)
append(".html")
}
}
append(page)
append(".html")
}
val doc = webClient.httpGet(url).parseHtml()
val root = doc.body().selectFirst("ul.direlist") ?: doc.parseFailed("Cannot find root")
val doc = context.httpGet(url, headers).parseHtml()
val root = doc.body().selectFirst("ul.direlist")
?: doc.parseFailed("Cannot find root")
val baseHost = root.baseUri().toHttpUrl().host
return root.select("li").map { node ->
val href = node.selectFirst("a")?.absUrl("href") ?: node.parseFailed("Link not found")
val href = node.selectFirst("a")?.absUrl("href")
?: node.parseFailed("Link not found")
val relUrl = href.toRelativeUrl(baseHost)
val dd = node.selectFirst("dd")
Manga(
@@ -123,25 +91,30 @@ internal abstract class NineMangaParser(
}
override suspend fun getDetails(manga: Manga): Manga {
val doc = webClient.httpGet(
manga.url.toAbsoluteUrl(domain) + "?waring=1",
val doc = context.httpGet(
manga.url.toAbsoluteUrl(getDomain()) + "?waring=1",
headers,
).parseHtml()
val root = doc.body().selectFirstOrThrow("div.manga")
val infoRoot = root.selectFirstOrThrow("div.bookintro")
val tagMap = getOrCreateTagMap()
val selectTag = infoRoot.getElementsByAttributeValue("itemprop", "genre").first()?.select("a")
val tags = selectTag?.mapNotNullToSet { tagMap[it.text()] }
return manga.copy(
tags = tags.orEmpty(),
tags = infoRoot.getElementsByAttributeValue("itemprop", "genre").first()
?.select("a")?.mapToSet { a ->
MangaTag(
title = a.text().toTitleCase(),
key = a.attr("href").substringBetween("/", "."),
source = source,
)
}.orEmpty(),
author = infoRoot.getElementsByAttributeValue("itemprop", "author").first()?.text(),
state = parseStatus(infoRoot.select("li a.red").text()),
description = infoRoot.getElementsByAttributeValue("itemprop", "description").first()?.html()
?.substringAfter("</b>"),
description = infoRoot.getElementsByAttributeValue("itemprop", "description").first()
?.html()?.substringAfter("</b>"),
chapters = root.selectFirst("div.chapterbox")?.select("ul.sub_vol_ul > li")
?.mapChapters(reversed = true) { i, li ->
?.asReversed()?.mapChapters { i, li ->
val a = li.selectFirst("a.chapter_list_a")
val href =
a?.attrAsRelativeUrlOrNull("href")?.replace("%20", " ") ?: li.parseFailed("Link not found")
val href = a?.attrAsRelativeUrlOrNull("href")
?.replace("%20", " ") ?: li.parseFailed("Link not found")
MangaChapter(
id = generateUid(href),
name = a.text(),
@@ -157,12 +130,13 @@ internal abstract class NineMangaParser(
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml()
val doc = context.httpGet(chapter.url.toAbsoluteUrl(getDomain()), headers).parseHtml()
return doc.body().getElementById("page")?.select("option")?.map { option ->
val url = option.attr("value")
MangaPage(
id = generateUid(url),
url = url,
referer = chapter.url.toAbsoluteUrl(getDomain()),
preview = null,
source = source,
)
@@ -170,58 +144,30 @@ internal abstract class NineMangaParser(
}
override suspend fun getPageUrl(page: MangaPage): String {
val doc = webClient.httpGet(page.url.toAbsoluteUrl(domain)).parseHtml()
val doc = context.httpGet(page.url.toAbsoluteUrl(getDomain()), headers).parseHtml()
val root = doc.body()
return root.selectFirst("a.pic_download")?.absUrl("href") ?: doc.parseFailed("Page image not found")
return root.selectFirst("a.pic_download")?.absUrl("href")
?: doc.parseFailed("Page image not found")
}
private var tagCache: ArrayMap<String, MangaTag>? = null
private val mutex = Mutex()
override suspend fun getAvailableTags(): Set<MangaTag> {
return getOrCreateTagMap().values.toSet()
}
private suspend fun getOrCreateTagMap(): Map<String, MangaTag> = mutex.withLock {
tagCache?.let { return@withLock it }
val tagMap = ArrayMap<String, MangaTag>()
val tagElements = webClient.httpGet("https://${domain}/search/?type=high").parseHtml().select("li.cate_list")
for (el in tagElements) {
if (el.text().isEmpty()) continue
val cateId = el.attr("cate_id")
val a = el.selectFirstOrThrow("a")
tagMap[el.text()] = MangaTag(
title = a.text().toTitleCase(sourceLocale),
override suspend fun getTags(): Set<MangaTag> {
val doc = context.httpGet("https://${getDomain()}/search/?type=high", headers)
.parseHtml()
val root = doc.body().getElementById("search_form")
return root?.select("li.cate_list")?.mapNotNullToSet { li ->
val cateId = li.attr("cate_id") ?: return@mapNotNullToSet null
val a = li.selectFirst("a") ?: return@mapNotNullToSet null
MangaTag(
title = a.text().toTitleCase(),
key = cateId,
source = source,
)
}
tagCache = tagMap
return@withLock tagMap
} ?: doc.parseFailed("Root not found")
}
private fun parseStatus(status: String) = when {
// en
status.contains("Ongoing") -> MangaState.ONGOING
status.contains("Completed") -> MangaState.FINISHED
//es
status.contains("En curso") -> MangaState.ONGOING
status.contains("Completado") -> MangaState.FINISHED
//ru
status.contains("постоянный") -> MangaState.ONGOING
status.contains("завершенный") -> MangaState.FINISHED
//de
status.contains("Laufende") -> MangaState.ONGOING
status.contains("Abgeschlossen") -> MangaState.FINISHED
//pt
status.contains("Completo") -> MangaState.ONGOING
status.contains("Em tradução") -> MangaState.FINISHED
//it
status.contains("In corso") -> MangaState.ONGOING
status.contains("Completato") -> MangaState.FINISHED
//fr
status.contains("En cours") -> MangaState.ONGOING
status.contains("Complété") -> MangaState.FINISHED
else -> null
}
@@ -311,4 +257,4 @@ internal abstract class NineMangaParser(
MangaSource.NINEMANGA_FR,
"fr.ninemanga.com",
)
}
}

228
src/main/kotlin/org/koitharu/kotatsu/parsers/site/NudeMoonParser.kt Normal file
View File

@@ -0,0 +1,228 @@
package org.koitharu.kotatsu.parsers.site
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.MangaParserAuthProvider
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.exception.AuthRequiredException
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import java.text.SimpleDateFormat
import java.util.*
private const val MAX_THUMB_INDEX = 19
@MangaSourceParser("NUDEMOON", "Nude-Moon", "ru")
internal class NudeMoonParser(
override val context: MangaLoaderContext,
) : MangaParser(MangaSource.NUDEMOON), MangaParserAuthProvider {
override val configKeyDomain = ConfigKey.Domain("nude-moon.net", null)
override val authUrl: String
get() = "https://${getDomain()}/index.php"
override val isAuthorized: Boolean
get() {
return context.cookieJar.getCookies(getDomain()).any {
it.name == "fusion_user"
}
}
override val sortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.NEWEST,
SortOrder.POPULARITY,
SortOrder.RATING,
)
init {
context.cookieJar.insertCookies(
getDomain(),
"NMfYa=1;",
"nm_mobile=0;",
)
}
override suspend fun getList(
offset: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val domain = getDomain()
val url = when {
!query.isNullOrEmpty() -> "https://$domain/search?stext=${query.urlEncoded()}&rowstart=$offset"
!tags.isNullOrEmpty() -> tags.joinToString(
separator = "_",
prefix = "https://$domain/tags/",
postfix = "&rowstart=$offset",
transform = { it.key.urlEncoded() },
)
else -> "https://$domain/all_manga?${getSortKey(sortOrder)}&rowstart=$offset"
}
val doc = context.httpGet(url).parseHtml()
val root = doc.body().run {
selectFirst("td.main-bg") ?: selectFirst("td.main-body")
} ?: doc.parseFailed("Cannot find root")
return root.select("table.news_pic2").mapNotNull { row ->
val a = row.selectFirst("td.bg_style1")?.selectFirst("a")
?: return@mapNotNull null
val href = a.attrAsRelativeUrl("href")
val title = a.selectFirst("h2")?.text().orEmpty()
val info = row.selectFirst("td[width=100%]") ?: return@mapNotNull null
Manga(
id = generateUid(href),
url = href,
title = title.substringAfter(" / "),
altTitle = title.substringBefore(" / ", "")
.takeUnless { it.isBlank() },
author = info.getElementsContainingOwnText("Автор:").firstOrNull()
?.nextElementSibling()?.ownText(),
coverUrl = row.selectFirst("img.news_pic2")?.absUrl("data-src")
.orEmpty(),
tags = row.selectFirst("span.tag-links")?.select("a")
?.mapToSet {
MangaTag(
title = it.text().toTitleCase(),
key = it.attr("href").substringAfterLast('/'),
source = source,
)
}.orEmpty(),
source = source,
publicUrl = a.absUrl("href"),
rating = RATING_UNKNOWN,
isNsfw = true,
description = row.selectFirst("div.description")?.html(),
state = null,
)
}
}
override suspend fun getDetails(manga: Manga): Manga {
val body = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml().body()
val root = body.selectFirst("table.shoutbox")
?: body.parseFailed("Cannot find root")
val info = root.select("div.tbl2")
val lastInfo = info.last()
return manga.copy(
largeCoverUrl = body.selectFirst("img.news_pic2")?.absUrl("src"),
description = info.select("div.blockquote").lastOrNull()?.html() ?: manga.description,
tags = info.select("span.tag-links").firstOrNull()?.select("a")?.mapToSet {
MangaTag(
title = it.text().toTitleCase(),
key = it.attr("href").substringAfterLast('/'),
source = source,
)
}?.plus(manga.tags) ?: manga.tags,
author = lastInfo?.getElementsByAttributeValueContaining("href", "mangaka/")?.text()
?: manga.author,
chapters = listOf(
MangaChapter(
id = manga.id,
url = getReadLink(manga.url),
source = source,
number = 1,
name = manga.title,
scanlator = lastInfo?.getElementsByAttributeValueContaining("href", "perevod/")?.text(),
uploadDate = lastInfo?.getElementsContainingOwnText("Дата:")
?.firstOrNull()
?.html()
?.parseDate() ?: 0L,
branch = null,
),
),
)
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(getDomain())
val doc = context.httpGet(fullUrl).parseHtml()
val mangaId = chapter.url.substringAfterLast('/').substringBefore('-').toIntOrNull()
val script = doc.select("script").firstNotNullOfOrNull {
it.html().takeIf { x -> x.contains(" images = new ") }
} ?: if (isAuthorized) {
doc.parseFailed("Cannot find pages list")
} else {
throw AuthRequiredException(source)
}
val pagesRegex = Regex("images\\[(\\d+)].src\\s*=\\s*'([^']+)'", RegexOption.MULTILINE)
return pagesRegex.findAll(script).map { match ->
val i = match.groupValues[1].toInt()
val url = match.groupValues[2]
MangaPage(
id = generateUid(url),
url = url,
referer = fullUrl,
preview = if (i <= MAX_THUMB_INDEX && mangaId != null) {
val part2 = url.substringBeforeLast('/')
val part3 = url.substringAfterLast('/')
val part1 = part2.substringBeforeLast('/')
"$part1/thumb/$mangaId/thumb_$part3"
} else {
null
},
source = source,
)
}.toList()
}
override suspend fun getTags(): Set<MangaTag> {
val domain = getDomain()
val doc = context.httpGet("https://$domain/all_manga").parseHtml()
val root = doc.body().getElementsContainingOwnText("Поиск манги по тегам")
.firstOrNull()?.parents()?.find { it.tag().normalName() == "tbody" }
?.selectFirst("td.textbox")?.selectFirst("td.small")
?: doc.parseFailed("Tags root not found")
return root.select("a").mapToSet {
MangaTag(
title = it.text().toTitleCase(),
key = it.attr("href").substringAfterLast('/')
.removeSuffix("+"),
source = source,
)
}
}
override suspend fun getUsername(): String {
val body = context.httpGet("https://${getDomain()}/").parseHtml()
.body()
return body
.getElementsContainingOwnText("Профиль")
.firstOrNull()
?.attr("href")
?.substringAfterLast('/')
?: run {
throw if (body.selectFirst("form[name=\"loginform\"]") != null) {
AuthRequiredException(source)
} else {
body.parseFailed("Cannot find username")
}
}
}
override fun getFaviconUrl(): String {
return "https://${getDomain()}/favicon.jpg"
}
private fun getSortKey(sortOrder: SortOrder) =
when (sortOrder) {
SortOrder.POPULARITY -> "views"
SortOrder.NEWEST -> "date"
SortOrder.RATING -> "like"
else -> "like"
}
private fun String.parseDate(): Long {
val dateString = substringBetweenFirst("Дата:", "<")?.trim() ?: return 0
val dateFormat = SimpleDateFormat("d MMMM yyyy", Locale("ru"))
return dateFormat.tryParse(dateString)
}
private fun getReadLink(url: String): String {
val prefix = url.substringBefore('-', "")
val suffix = url.substringAfter('-').trimStart('-')
return "$prefix-online-$suffix"
}
}

124
src/main/kotlin/org/koitharu/kotatsu/parsers/site/ru/RemangaParser.kt → src/main/kotlin/org/koitharu/kotatsu/parsers/site/RemangaParser.kt
View File

@@ -1,9 +1,7 @@
package org.koitharu.kotatsu.parsers.site.ru
package org.koitharu.kotatsu.parsers.site
import okhttp3.Headers
import okhttp3.Interceptor
import okhttp3.Response
import okhttp3.internal.closeQuietly
import org.json.JSONArray
import org.json.JSONException
import org.json.JSONObject
@@ -12,12 +10,14 @@ import org.koitharu.kotatsu.parsers.MangaParserAuthProvider
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.exception.AuthRequiredException
import org.koitharu.kotatsu.parsers.exception.ContentUnavailableException
import org.koitharu.kotatsu.parsers.exception.ParseException
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.network.UserAgents
import org.koitharu.kotatsu.parsers.util.*
import org.koitharu.kotatsu.parsers.util.json.*
import java.net.HttpURLConnection
import java.net.URLDecoder
import java.text.DateFormat
import java.text.SimpleDateFormat
import java.util.*
@@ -25,29 +25,17 @@ import java.util.*
private const val PAGE_SIZE = 30
private const val STATUS_ONGOING = 1
private const val STATUS_FINISHED = 0
private const val TOO_MANY_REQUESTS = 429
@MangaSourceParser("REMANGA", "ReManga", "ru")
@MangaSourceParser("REMANGA", "Remanga", "ru")
internal class RemangaParser(
context: MangaLoaderContext,
) : PagedMangaParser(context, MangaSource.REMANGA, PAGE_SIZE), MangaParserAuthProvider, Interceptor {
private val userAgentKey = ConfigKey.UserAgent(UserAgents.CHROME_MOBILE)
private val baseHeaders: Headers
get() = Headers.Builder()
.add("User-Agent", config[userAgentKey])
.build()
override val headers
get() = getApiHeaders()
override val configKeyDomain = ConfigKey.Domain("remanga.org", "реманга.орг")
override val context: MangaLoaderContext,
) : PagedMangaParser(MangaSource.REMANGA, PAGE_SIZE), MangaParserAuthProvider {
override val configKeyDomain = ConfigKey.Domain("remanga.org", null)
override val authUrl: String
get() = "https://${domain}"
get() = "https://${getDomain()}/user/login"
override val availableSortOrders: Set<SortOrder> = EnumSet.of(
override val sortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.UPDATED,
SortOrder.POPULARITY,
SortOrder.RATING,
@@ -56,32 +44,21 @@ internal class RemangaParser(
override val isAuthorized: Boolean
get() {
return context.cookieJar.getCookies(domain).any {
return context.cookieJar.getCookies(getDomain()).any {
it.name == "user"
}
}
private val regexLastUrlPath = Regex("/[^/]+/?$")
override fun intercept(chain: Interceptor.Chain): Response {
val response = chain.proceed(chain.request())
if (response.code == TOO_MANY_REQUESTS) {
response.closeQuietly()
Thread.sleep(1000)
return chain.proceed(chain.request().newBuilder().build())
}
return response
}
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
tagsExclude: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
copyCookies()
val domain = domain
val domain = getDomain()
val urlBuilder = StringBuilder()
.append("https://api.")
.append(domain)
@@ -101,7 +78,7 @@ internal class RemangaParser(
.append(page)
.append("&count=")
.append(PAGE_SIZE)
val content = webClient.httpGet(urlBuilder.toString()).parseJson()
val content = context.httpGet(urlBuilder.toString(), getApiHeaders()).parseJson()
.getJSONArray("content")
return content.mapJSON { jo ->
val url = "/manga/${jo.getString("dir")}"
@@ -132,12 +109,13 @@ internal class RemangaParser(
override suspend fun getDetails(manga: Manga): Manga {
copyCookies()
val domain = domain
val domain = getDomain()
val slug = manga.url.find(regexLastUrlPath)
?: throw ParseException("Cannot obtain slug from ${manga.url}", manga.publicUrl)
val data = webClient.httpGet(
val data = context.httpGet(
url = "https://api.$domain/api/titles$slug/",
).parseJson()
headers = getApiHeaders(),
).handle401().parseJson()
val content = try {
data.getJSONObject("content")
} catch (e: JSONException) {
@@ -162,10 +140,7 @@ internal class RemangaParser(
)
},
chapters = chapters.mapChapters { i, jo ->
if (
jo.getBooleanOrDefault("is_paid", false) &&
!jo.getBooleanOrDefault("is_bought", false)
) {
if (jo.getBooleanOrDefault("is_paid", false)) {
return@mapChapters null
}
val id = jo.getLong("id")
@@ -174,7 +149,7 @@ internal class RemangaParser(
MangaChapter(
id = generateUid(id),
url = "/api/titles/chapters/$id/",
number = jo.getIntOrDefault("index", chapters.size - i),
number = chapters.size - i,
name = buildString {
append("Том ")
append(jo.optString("tome", "0"))
@@ -196,7 +171,9 @@ internal class RemangaParser(
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val content = webClient.httpGet(chapter.url.toAbsoluteUrl(getDomain("api")))
val referer = "https://${getDomain()}/"
val content = context.httpGet(chapter.url.toAbsoluteUrl(getDomain("api")), getApiHeaders())
.handle401()
.parseJson()
.getJSONObject("content")
val pages = content.optJSONArray("pages")
@@ -214,17 +191,17 @@ internal class RemangaParser(
val result = ArrayList<MangaPage>(pages.length())
for (i in 0 until pages.length()) {
when (val item = pages.get(i)) {
is JSONObject -> result += parsePage(item)
is JSONArray -> item.mapJSONTo(result) { parsePage(it) }
is JSONObject -> result += parsePage(item, referer)
is JSONArray -> item.mapJSONTo(result) { parsePage(it, referer) }
else -> throw ParseException("Unknown json item $item", chapter.url)
}
}
return result
}
override suspend fun getAvailableTags(): Set<MangaTag> {
val domain = domain
val content = webClient.httpGet("https://api.$domain/api/forms/titles/?get=genres")
override suspend fun getTags(): Set<MangaTag> {
val domain = getDomain()
val content = context.httpGet("https://api.$domain/api/forms/titles/?get=genres", getApiHeaders())
.parseJson().getJSONObject("content").getJSONArray("genres")
return content.mapJSONToSet { jo ->
MangaTag(
@@ -236,28 +213,24 @@ internal class RemangaParser(
}
override suspend fun getUsername(): String {
val jo = webClient.httpGet(
url = "https://api.${domain}/api/users/current/",
).parseJson()
val jo = context.httpGet(
url = "https://api.${getDomain()}/api/users/current/",
headers = getApiHeaders(),
).handle401().parseJson()
return jo.getJSONObject("content").getString("username")
}
override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) {
super.onCreateConfig(keys)
keys.add(userAgentKey)
}
private fun getApiHeaders(): Headers {
val userCookie = context.cookieJar.getCookies(domain).find {
private fun getApiHeaders(): Headers? {
val userCookie = context.cookieJar.getCookies(getDomain()).find {
it.name == "user"
} ?: return baseHeaders
val jo = JSONObject(userCookie.value.urlDecode())
val accessToken = jo.getStringOrNull("access_token") ?: return baseHeaders
return baseHeaders.newBuilder().add("authorization", "bearer $accessToken").build()
} ?: return null
val jo = JSONObject(URLDecoder.decode(userCookie.value, Charsets.UTF_8.name()))
val accessToken = jo.getStringOrNull("access_token") ?: return null
return Headers.headersOf("authorization", "bearer $accessToken")
}
private fun copyCookies() {
val domain = domain
val domain = getDomain()
context.cookieJar.copyCookies(domain, "api.$domain")
}
@@ -269,11 +242,11 @@ internal class RemangaParser(
else -> "-chapter_date"
}
private fun parsePage(jo: JSONObject) = MangaPage(
// id = generateUid(jo.getLong("id")), 19.01.2024 page id is gone
id = generateUid(jo.getString("link")),
private fun parsePage(jo: JSONObject, referer: String) = MangaPage(
id = generateUid(jo.getLong("id")),
url = jo.getString("link"),
preview = null,
referer = referer,
source = source,
)
@@ -281,9 +254,10 @@ internal class RemangaParser(
val result = ArrayList<JSONObject>(100)
var page = 1
while (true) {
val content = webClient.httpGet(
url = "https://api.$domain/api/titles/chapters/?branch_id=$branchId&page=$page&count=500",
).parseJson().getJSONArray("content")
val content = context.httpGet(
url = "https://api.$domain/api/titles/chapters/?branch_id=$branchId&page=$page&count=100",
headers = getApiHeaders(),
).handle401().parseJson().getJSONArray("content")
val len = content.length()
if (len == 0) {
break
@@ -296,4 +270,10 @@ internal class RemangaParser(
}
return result
}
}
private fun Response.handle401() = apply {
if (code == HttpURLConnection.HTTP_UNAUTHORIZED) {
throw AuthRequiredException(source)
}
}
}

177
src/main/kotlin/org/koitharu/kotatsu/parsers/site/UnionMangasParser.kt Normal file
View File

@@ -0,0 +1,177 @@
package org.koitharu.kotatsu.parsers.site
import org.jsoup.nodes.Element
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import org.koitharu.kotatsu.parsers.util.json.getStringOrNull
import org.koitharu.kotatsu.parsers.util.json.mapJSON
import java.text.SimpleDateFormat
import java.util.*
@MangaSourceParser("UNION_MANGAS", "Union Mangás", "pt")
class UnionMangasParser(override val context: MangaLoaderContext) : PagedMangaParser(MangaSource.UNION_MANGAS, 40) {
override val sortOrders = EnumSet.of(
SortOrder.ALPHABETICAL,
SortOrder.POPULARITY,
)
override val configKeyDomain = ConfigKey.Domain("unionleitor.top", emptyArray())
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
if (!query.isNullOrEmpty()) {
return if (page == searchPaginator.firstPage) {
search(query)
} else {
emptyList()
}
}
val tag = tags.oneOrThrowIfMany()
val url = urlBuilder()
.addPathSegment("lista-mangas")
.addPathSegment(
when {
tag != null -> tag.key
sortOrder == SortOrder.ALPHABETICAL -> "a-z"
else -> "visualizacoes"
},
).addPathSegment(page.toString())
val doc = context.httpGet(url.build()).parseHtml()
val root = doc.selectFirstOrThrow("div.tamanho-bloco-perfil")
return root.select(".lista-mangas-novos").map { div ->
val a = div.selectFirstOrThrow("a")
val img = div.selectFirstOrThrow("img")
val href = a.attrAsRelativeUrl("href")
Manga(
id = generateUid(href),
url = href,
publicUrl = a.attrAsAbsoluteUrl("href"),
title = div.selectLastOrThrow("a").text(),
coverUrl = img.attrAsAbsoluteUrl("src"),
altTitle = null,
rating = RATING_UNKNOWN,
tags = emptySet(),
description = div.selectLast("div")?.ownText(),
state = null,
author = null,
isNsfw = false,
source = source,
)
}
}
override suspend fun getDetails(manga: Manga): Manga {
val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml()
val root = doc.selectFirstOrThrow(".perfil-manga")
val dateFormat = SimpleDateFormat("dd/MM/yyyy", Locale.ROOT)
return manga.copy(
rating = root.select("h2")
.find { it.ownText().startsWith('#') }
?.ownText()?.drop(1)?.toFloatOrNull()?.div(10f) ?: manga.rating,
largeCoverUrl = root.selectFirst("img.img-thumbnail")?.attrAsAbsoluteUrlOrNull("src"),
description = root.selectFirst(".panel-default")?.selectFirst(".panel-body")?.html(),
author = root.tableValue("Autor")?.ownText(),
altTitle = root.tableValue("Título(s) Alternativo(s)")?.ownText(),
state = when (root.tableValue("Status")?.selectLast(".label")?.text()) {
"Completo" -> MangaState.FINISHED
"Ativo" -> MangaState.ONGOING
else -> null
},
tags = root.tableValue("Gênero(s)")?.select("a")?.mapToSet {
it.toMangaTag()
} ?: manga.tags,
isNsfw = root.selectFirst(".alert-danger")?.html()?.contains("18 anos") == true,
chapters = root.select("div.row.capitulos").asReversed().mapChapters { i, div ->
val a = div.selectFirstOrThrow("a")
val href = a.attrAsRelativeUrl("href")
val title = a.text()
MangaChapter(
id = generateUid(href),
name = title,
number = i + 1,
url = href,
scanlator = div.selectLast("a")?.text()?.takeUnless { it == title },
uploadDate = dateFormat.tryParse(
a.nextElementSibling()?.text()?.removeSurrounding("(", ")"),
),
branch = null,
source = source,
)
},
)
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(getDomain())
val doc = context.httpGet(fullUrl).parseHtml()
val root = doc.body().selectFirstOrThrow("article")
return root.selectOrThrow("img[pag]").mapNotNull { img ->
val href = img.attrAsRelativeUrl("src")
if (href.startsWith("/images/banner")) {
return@mapNotNull null
}
MangaPage(
id = generateUid(href),
url = href,
referer = fullUrl,
preview = null,
source = source,
)
}
}
override suspend fun getTags(): Set<MangaTag> {
val doc = context.httpGet(urlBuilder().addPathSegment("lista-mangas").build()).parseHtml()
val ul = doc.body().selectFirstOrThrow(".nav-tabs").selectFirstOrThrow("ul.dropdown-menu")
return ul.select("li").mapToSet { li ->
li.selectFirstOrThrow("a").toMangaTag()
}
}
private suspend fun search(query: String): List<Manga> {
val domain = getDomain()
val json = context.httpGet(
urlBuilder()
.addPathSegments("assets/busca.php")
.addQueryParameter("nomeManga", query)
.build(),
).parseJson()
return json.getJSONArray("items").mapJSON { jo ->
val href = "/pagina-manga/" + jo.getString("url")
Manga(
id = generateUid(href),
url = href,
publicUrl = href.toAbsoluteUrl(domain),
title = jo.getString("titulo"),
rating = RATING_UNKNOWN,
tags = emptySet(),
author = jo.getStringOrNull("autor"),
coverUrl = jo.getString("imagem"),
state = null,
isNsfw = false,
altTitle = null,
source = source,
)
}
}
private fun Element.tableValue(title: String): Element? {
return select("h4.media-heading")
.find { it.selectFirst("label.subtit-manga")?.text()?.contains(title, ignoreCase = true) == true }
}
private fun Element.toMangaTag() = MangaTag(
title = text().toTitleCase(sourceLocale ?: Locale.ROOT),
key = attr("href").removeSuffix('/').substringAfterLast('/'),
source = source,
)
}

262
src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/ComickFunParser.kt
View File

@@ -1,262 +0,0 @@
package org.koitharu.kotatsu.parsers.site.all
import androidx.collection.ArraySet
import androidx.collection.SparseArrayCompat
import org.json.JSONArray
import org.json.JSONObject
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import org.koitharu.kotatsu.parsers.util.json.*
import java.text.SimpleDateFormat
import java.util.*
/**
* cc
*/
private const val CHAPTERS_LIMIT = 99999
@MangaSourceParser("COMICK_FUN", "ComicK")
internal class ComickFunParser(context: MangaLoaderContext) : PagedMangaParser(context, MangaSource.COMICK_FUN, 20) {
override val configKeyDomain = ConfigKey.Domain("comick.cc")
override val availableSortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.POPULARITY,
SortOrder.UPDATED,
SortOrder.RATING,
)
override val availableStates: Set<MangaState> =
EnumSet.of(MangaState.ONGOING, MangaState.FINISHED, MangaState.PAUSED, MangaState.ABANDONED)
private val tagsArray = SuspendLazy(::loadTags)
override suspend fun getListPage(page: Int, filter: MangaListFilter?): List<Manga> {
val domain = domain
val url = urlBuilder()
.host("api.$domain")
.addPathSegment("v1.0")
.addPathSegment("search")
.addQueryParameter("type", "comic")
.addQueryParameter("tachiyomi", "true")
.addQueryParameter("limit", pageSize.toString())
.addQueryParameter("page", page.toString())
when (filter) {
is MangaListFilter.Search -> {
url.addQueryParameter("q", filter.query)
}
null -> {
url.addQueryParameter("sort", "view")
}
is MangaListFilter.Advanced -> {
filter.tags.forEach { tag ->
url.addQueryParameter("genres", tag.key)
}
url.addQueryParameter(
"sort",
when (filter.sortOrder) {
SortOrder.POPULARITY -> "view"
SortOrder.RATING -> "rating"
else -> "uploaded"
},
)
filter.states.oneOrThrowIfMany()?.let {
url.addQueryParameter(
"status",
when (it) {
MangaState.ONGOING -> "1"
MangaState.FINISHED -> "2"
MangaState.ABANDONED -> "3"
MangaState.PAUSED -> "4"
else -> ""
},
)
}
}
}
val ja = webClient.httpGet(url.build()).parseJsonArray()
val tagsMap = tagsArray.get()
return ja.mapJSON { jo ->
val slug = jo.getString("slug")
Manga(
id = generateUid(slug),
title = jo.getString("title"),
altTitle = null,
url = slug,
publicUrl = "https://$domain/comic/$slug",
rating = jo.getDoubleOrDefault("rating", -10.0).toFloat() / 10f,
isNsfw = false,
coverUrl = jo.getString("cover_url"),
largeCoverUrl = null,
description = jo.getStringOrNull("desc"),
tags = jo.selectGenres(tagsMap),
state = when (jo.getIntOrDefault("status", 0)) {
1 -> MangaState.ONGOING
2 -> MangaState.FINISHED
3 -> MangaState.ABANDONED
4 -> MangaState.PAUSED
else -> null
},
author = null,
source = source,
)
}
}
override suspend fun getDetails(manga: Manga): Manga {
val domain = domain
val url = "https://api.$domain/comic/${manga.url}?tachiyomi=true"
val jo = webClient.httpGet(url).parseJson()
val comic = jo.getJSONObject("comic")
var alt = ""
comic.getJSONArray("md_titles").mapJSON { alt += it.getString("title") + " - " }
return manga.copy(
altTitle = alt.ifEmpty { comic.getStringOrNull("title") },
isNsfw = jo.getBoolean("matureContent") || comic.getBoolean("hentai"),
description = comic.getStringOrNull("parsed") ?: comic.getStringOrNull("desc"),
tags = manga.tags + comic.getJSONArray("md_comic_md_genres").mapJSONToSet {
val g = it.getJSONObject("md_genres")
MangaTag(
title = g.getString("name"),
key = g.getString("slug"),
source = source,
)
},
author = jo.getJSONArray("artists").optJSONObject(0)?.getString("name"),
chapters = getChapters(comic.getString("hid")),
)
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val jo = webClient.httpGet(
"https://api.${domain}/chapter/${chapter.url}?tachiyomi=true",
).parseJson().getJSONObject("chapter")
return jo.getJSONArray("images").mapJSON {
val url = it.getString("url")
MangaPage(
id = generateUid(url),
url = url,
preview = null,
source = source,
)
}
}
override suspend fun getAvailableTags(): Set<MangaTag> {
val sparseArray = tagsArray.get()
val set = ArraySet<MangaTag>(sparseArray.size())
for (i in 0 until sparseArray.size()) {
set.add(sparseArray.valueAt(i))
}
return set
}
private suspend fun loadTags(): SparseArrayCompat<MangaTag> {
val ja = webClient.httpGet("https://api.${domain}/genre").parseJsonArray()
val tags = SparseArrayCompat<MangaTag>(ja.length())
for (jo in ja.JSONIterator()) {
tags.append(
jo.getInt("id"),
MangaTag(
title = jo.getString("name").toTitleCase(Locale.ENGLISH),
key = jo.getString("slug"),
source = source,
),
)
}
return tags
}
private suspend fun getChapters(hid: String): List<MangaChapter> {
val ja = webClient.httpGet(
url = "https://api.${domain}/comic/$hid/chapters?limit=$CHAPTERS_LIMIT",
).parseJson().getJSONArray("chapters")
val dateFormat = SimpleDateFormat("yyyy-MM-dd")
val counters = HashMap<String?, Int>()
return ja.toJSONList().reversed().mapChapters { _, jo ->
val vol = jo.getStringOrNull("vol")
val chap = jo.getStringOrNull("chap")
val locale = Locale.forLanguageTag(jo.getString("lang"))
val group = jo.optJSONArray("group_name")?.joinToString(", ")
val branch = buildString {
append(locale.getDisplayName(locale).toTitleCase(locale))
if (!group.isNullOrEmpty()) {
append(" (")
append(group)
append(')')
}
}
MangaChapter(
id = generateUid(jo.getLong("id")),
name = buildString {
vol?.let { append("Vol ").append(it).append(' ') }
chap?.let { append("Chap ").append(it) }
jo.getStringOrNull("title")?.let { append(": ").append(it) }
},
number = counters.incrementAndGet(branch),
url = jo.getString("hid"),
scanlator = jo.optJSONArray("group_name")?.asIterable<String>()?.joinToString()
?.takeUnless { it.isBlank() },
uploadDate = dateFormat.tryParse(jo.getString("created_at").substringBefore('T')),
branch = branch,
source = source,
)
}
/*val chaptersBuilder = ChaptersListBuilder(list.size)
val branchedChapters = HashMap<String?, HashMap<Pair<String?, String?>, MangaChapter>>()
for (jo in list) {
val vol = jo.getStringOrNull("vol")
val chap = jo.getStringOrNull("chap")
val volChap = vol to chap
val locale = Locale.forLanguageTag(jo.getString("lang"))
val lc = locale.getDisplayName(locale).toTitleCase(locale)
val branch = (list.indices).firstNotNullOf { i ->
val b = if (i == 0) lc else "$lc ($i)"
if (branchedChapters[b]?.get(volChap) == null) b else null
}
val chapter = MangaChapter(
id = generateUid(jo.getLong("id")),
name = buildString {
vol?.let { append("Vol ").append(it).append(' ') }
chap?.let { append("Chap ").append(it) }
jo.getStringOrNull("title")?.let { append(": ").append(it) }
},
number = branchedChapters[branch]?.size?.plus(1) ?: 1,
url = jo.getString("hid"),
scanlator = jo.optJSONArray("group_name")?.asIterable<String>()?.joinToString()
?.takeUnless { it.isBlank() },
uploadDate = dateFormat.tryParse(jo.getString("created_at").substringBefore('T')),
branch = branch,
source = source,
)
if (chaptersBuilder.add(chapter)) {
branchedChapters.getOrPut(branch, ::HashMap)[volChap] = chapter
}
}
return chaptersBuilder.toList()*/
}
private fun JSONObject.selectGenres(tags: SparseArrayCompat<MangaTag>): Set<MangaTag> {
val array = optJSONArray("genres") ?: return emptySet()
val res = ArraySet<MangaTag>(array.length())
for (i in 0 until array.length()) {
val id = array.getInt(i)
val tag = tags[id] ?: continue
res.add(tag)
}
return res
}
private fun JSONArray.joinToString(separator: String): String {
return (0 until length()).joinToString(separator) { i -> getString(i) }
}
}

420
src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/ExHentaiParser.kt
View File

@@ -1,420 +0,0 @@
package org.koitharu.kotatsu.parsers.site.all
import androidx.collection.ArrayMap
import androidx.collection.ArraySet
import androidx.collection.SparseArrayCompat
import androidx.collection.set
import okhttp3.HttpUrl.Companion.toHttpUrlOrNull
import org.jsoup.internal.StringUtil
import org.jsoup.nodes.Element
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParserAuthProvider
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.exception.AuthRequiredException
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import java.util.*
import kotlin.math.pow
private const val DOMAIN_UNAUTHORIZED = "e-hentai.org"
private const val DOMAIN_AUTHORIZED = "exhentai.org"
@MangaSourceParser("EXHENTAI", "ExHentai", type = ContentType.HENTAI)
internal class ExHentaiParser(
context: MangaLoaderContext,
) : PagedMangaParser(context, MangaSource.EXHENTAI, pageSize = 25), MangaParserAuthProvider {
override val availableSortOrders: Set<SortOrder> = setOf(SortOrder.NEWEST)
override val isTagsExclusionSupported: Boolean = true
override val configKeyDomain: ConfigKey.Domain
get() = ConfigKey.Domain(
if (isAuthorized) DOMAIN_AUTHORIZED else DOMAIN_UNAUTHORIZED,
if (isAuthorized) DOMAIN_UNAUTHORIZED else DOMAIN_AUTHORIZED,
)
override val authUrl: String
get() = "https://${domain}/bounce_login.php"
private val ratingPattern = Regex("-?[0-9]+px")
private val authCookies = arrayOf("ipb_member_id", "ipb_pass_hash")
private var updateDm = false
private val nextPages = SparseArrayCompat<Long>()
private val suspiciousContentKey = ConfigKey.ShowSuspiciousContent(false)
private val tagsMap = SuspendLazy(::fetchTags)
override val isAuthorized: Boolean
get() {
val authorized = isAuthorized(DOMAIN_UNAUTHORIZED)
if (authorized) {
if (!isAuthorized(DOMAIN_AUTHORIZED)) {
context.cookieJar.copyCookies(
DOMAIN_UNAUTHORIZED,
DOMAIN_AUTHORIZED,
authCookies,
)
context.cookieJar.insertCookies(DOMAIN_AUTHORIZED, "yay=louder")
}
return true
}
return false
}
init {
context.cookieJar.insertCookies(DOMAIN_AUTHORIZED, "nw=1", "sl=dm_2")
context.cookieJar.insertCookies(DOMAIN_UNAUTHORIZED, "nw=1", "sl=dm_2")
paginator.firstPage = 0
}
override suspend fun getListPage(page: Int, filter: MangaListFilter?): List<Manga> {
val next = nextPages.get(page, 0L)
if (page > 0 && next == 0L) {
assert(false) { "Page timestamp not found" }
return emptyList()
}
var search = ""
val url = buildString {
append("https://")
append(domain)
append("/?next=")
append(next)
when (filter) {
is MangaListFilter.Search -> {
search += filter.query.urlEncoded()
append("&f_search=")
append(search.trim().replace(' ', '+'))
}
is MangaListFilter.Advanced -> {
filter.toSearchQuery()?.let { sq ->
append("&f_search=")
append(sq.urlEncoded())
}
val catsOn = filter.tags.mapNotNullToSet { it.key.toIntOrNull() }
val catsOff = filter.tagsExclude.mapNotNullToSet { it.key.toIntOrNull() }
if (catsOff.size >= 10) {
return emptyList()
}
var fCats = catsOn.fold(0, Int::or)
if (fCats != 0) {
fCats = 1023 - fCats
}
fCats = catsOff.fold(fCats, Int::or)
if (fCats != 0) {
append("&f_cats=")
append(fCats)
}
}
null -> {}
}
// by unknown reason cookie "sl=dm_2" is ignored, so, we should request it again
if (updateDm) {
append("&inline_set=dm_e")
}
append("&advsearch=1")
if (config[suspiciousContentKey]) {
append("&f_sh=on")
}
}
val body = webClient.httpGet(url).parseHtml().body()
val root = body.selectFirst("table.itg")
?.selectFirst("tbody")
?: if (updateDm) {
body.parseFailed("Cannot find root")
} else {
updateDm = true
return getListPage(page, filter)
}
updateDm = false
nextPages[page + 1] = getNextTimestamp(body)
return root.children().mapNotNull { tr ->
if (tr.childrenSize() != 2) return@mapNotNull null
val (td1, td2) = tr.children()
val gLink = td2.selectFirstOrThrow("div.glink")
val a = gLink.parents().select("a").first() ?: gLink.parseFailed("link not found")
val href = a.attrAsRelativeUrl("href")
val tagsDiv = gLink.nextElementSibling() ?: gLink.parseFailed("tags div not found")
val mainTag = td2.selectFirst("div.cn")?.let { div ->
MangaTag(
title = div.text().toTitleCase(),
key = tagIdByClass(div.classNames()) ?: return@let null,
source = source,
)
}
Manga(
id = generateUid(href),
title = gLink.text().cleanupTitle(),
altTitle = null,
url = href,
publicUrl = a.absUrl("href"),
rating = td2.selectFirst("div.ir")?.parseRating() ?: RATING_UNKNOWN,
isNsfw = true,
coverUrl = td1.selectFirst("img")?.absUrl("src").orEmpty(),
tags = setOfNotNull(mainTag),
state = null,
author = tagsDiv.getElementsContainingOwnText("artist:").first()
?.nextElementSibling()?.text(),
source = source,
)
}
}
override suspend fun getDetails(manga: Manga): Manga {
val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml()
val root = doc.body().selectFirstOrThrow("div.gm")
val cover = root.getElementById("gd1")?.children()?.first()
val title = root.getElementById("gd2")
val tagList = root.getElementById("taglist")
val tabs = doc.body().selectFirst("table.ptt")?.selectFirst("tr")
val lang = root.getElementById("gd3")
?.selectFirst("tr:contains(Language)")
?.selectFirst(".gdt2")?.ownTextOrNull()
val tagMap = tagsMap.get()
val tags = ArraySet<MangaTag>()
tagList?.selectFirst("tr:contains(female:)")?.select("a")?.mapNotNullTo(tags) { tagMap[it.text()] }
tagList?.selectFirst("tr:contains(male:)")?.select("a")?.mapNotNullTo(tags) { tagMap[it.text()] }
return manga.copy(
title = title?.getElementById("gn")?.text()?.cleanupTitle() ?: manga.title,
altTitle = title?.getElementById("gj")?.text()?.cleanupTitle() ?: manga.altTitle,
publicUrl = doc.baseUri().ifEmpty { manga.publicUrl },
rating = root.getElementById("rating_label")?.text()
?.substringAfterLast(' ')
?.toFloatOrNull()
?.div(5f) ?: manga.rating,
largeCoverUrl = cover?.styleValueOrNull("background")?.cssUrl(),
tags = tags,
description = tagList?.select("tr")?.joinToString("<br>") { tr ->
val (tc, td) = tr.children()
val subTags = td.select("a").joinToString { it.html() }
"<b>${tc.html()}</b> $subTags"
},
chapters = tabs?.select("a")?.findLast { a ->
a.text().toIntOrNull() != null
}?.let { a ->
val count = a.text().toInt()
val chapters = ChaptersListBuilder(count)
for (i in 1..count) {
val url = "${manga.url}?p=${i - 1}"
chapters += MangaChapter(
id = generateUid(url),
name = "${manga.title} #$i",
number = i,
url = url,
uploadDate = 0L,
source = source,
scanlator = null,
branch = lang,
)
}
chapters.toList()
},
)
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml()
val root = doc.body().requireElementById("gdt")
return root.select("a").map { a ->
val url = a.attrAsRelativeUrl("href")
MangaPage(
id = generateUid(url),
url = url,
preview = null,
source = source,
)
}
}
override suspend fun getPageUrl(page: MangaPage): String {
val doc = webClient.httpGet(page.url.toAbsoluteUrl(domain)).parseHtml()
return doc.body().requireElementById("img").attrAsAbsoluteUrl("src")
}
private val tags =
"ahegao,anal,angel,apron,bandages,bbw,bdsm,beauty mark,big areolae,big ass,big breasts,big clit,big lips," +
"big nipples,bikini,blackmail,bloomers,blowjob,bodysuit,bondage,breast expansion,bukkake,bunny girl,business suit," +
"catgirl,centaur,cheating,chinese dress,christmas,collar,corset,cosplaying,cowgirl,crossdressing,cunnilingus," +
"dark skin,daughter,deepthroat,defloration,demon girl,double penetration,dougi,dragon,drunk,elf,exhibitionism,farting," +
"females only,femdom,filming,fingering,fishnets,footjob,fox girl,furry,futanari,garter belt,ghost,giantess," +
"glasses,gloves,goblin,gothic lolita,growth,guro,gyaru,hair buns,hairy,hairy armpits,handjob,harem,hidden sex," +
"horns,huge breasts,humiliation,impregnation,incest,inverted nipples,kemonomimi,kimono,kissing,lactation," +
"latex,leg lock,leotard,lingerie,lizard girl,maid,masked face,masturbation,midget,miko,milf,mind break," +
"mind control,monster girl,mother,muscle,nakadashi,netorare,nose hook,nun,nurse,oil,paizuri,panda girl," +
"pantyhose,piercing,pixie cut,policewoman,ponytail,pregnant,rape,rimjob,robot,scat,lolicon,schoolgirl uniform," +
"sex toys,shemale,sister,small breasts,smell,sole dickgirl,sole female,squirting,stockings,sundress,sweating," +
"swimsuit,swinging,tail,tall girl,teacher,tentacles,thigh high boots,tomboy,transformation,twins,twintails," +
"unusual pupils,urination,vore,vtuber,widow,wings,witch,wolf girl,x-ray,yuri,zombie,sole male,males only,yaoi," +
"tomgirl,tall man,oni,shotacon,prostate massage,policeman,males only,huge penis,fox boy,feminization,dog boy,dickgirl on male,big penis"
override suspend fun getAvailableTags(): Set<MangaTag> {
return tagsMap.get().values.toSet()
}
private suspend fun fetchTags(): Map<String, MangaTag> {
val tagMap = ArrayMap<String, MangaTag>()
val tagElements = tags.split(",")
for (el in tagElements) {
if (el.isEmpty()) continue
tagMap[el] = MangaTag(
title = el.toTitleCase(Locale.ENGLISH),
key = el,
source = source,
)
}
val doc = webClient.httpGet("https://${domain}").parseHtml()
val root = doc.body().requireElementById("searchbox").selectFirstOrThrow("table")
root.select("div.cs").mapNotNullToSet { div ->
val id = div.id().substringAfterLast('_').toIntOrNull() ?: return@mapNotNullToSet null
val name = div.text().toTitleCase(Locale.ENGLISH)
tagMap[name] = MangaTag(
title = "Kind: $name",
key = id.toString(),
source = source,
)
}
return tagMap
}
override suspend fun getAvailableLocales(): Set<Locale> = setOf(
Locale.JAPANESE,
Locale.ENGLISH,
Locale.CHINESE,
Locale("nl"),
Locale.FRENCH,
Locale.GERMAN,
Locale("hu"),
Locale.ITALIAN,
Locale("kr"),
Locale("pl"),
Locale("pt"),
Locale("ru"),
Locale("es"),
Locale("th"),
Locale("vi"),
)
private fun Locale.toLanguagePath() = when (language) {
else -> getDisplayLanguage(Locale.ENGLISH).lowercase()
}
override suspend fun getUsername(): String {
val doc = webClient.httpGet("https://forums.$DOMAIN_UNAUTHORIZED/").parseHtml().body()
val username = doc.getElementById("userlinks")
?.getElementsByAttributeValueContaining("href", "showuser=")
?.firstOrNull()
?.ownText()
?: if (doc.getElementById("userlinksguest") != null) {
throw AuthRequiredException(source)
} else {
doc.parseFailed()
}
return username
}
override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) {
super.onCreateConfig(keys)
keys.add(suspiciousContentKey)
}
private fun isAuthorized(domain: String): Boolean {
val cookies = context.cookieJar.getCookies(domain).mapToSet { x -> x.name }
return authCookies.all { it in cookies }
}
private fun Element.parseRating(): Float {
return runCatching {
val style = requireNotNull(attr("style"))
val (v1, v2) = ratingPattern.find(style)!!.destructured
var p1 = v1.dropLast(2).toInt()
val p2 = v2.dropLast(2).toInt()
if (p2 != -1) {
p1 += 8
}
(80 - p1) / 80f
}.getOrDefault(RATING_UNKNOWN)
}
private fun String.cleanupTitle(): String {
val result = StringBuilder(length)
var skip = false
for (c in this) {
when {
c == '[' -> skip = true
c == ']' -> skip = false
c.isWhitespace() && result.isEmpty() -> continue
!skip -> result.append(c)
}
}
while (result.lastOrNull()?.isWhitespace() == true) {
result.deleteCharAt(result.lastIndex)
}
return result.toString()
}
private fun String.cssUrl(): String? {
val fromIndex = indexOf("url(")
if (fromIndex == -1) {
return null
}
val toIndex = indexOf(')', startIndex = fromIndex)
return if (toIndex == -1) {
null
} else {
substring(fromIndex + 4, toIndex).trim()
}
}
private fun tagIdByClass(classNames: Collection<String>): String? {
val className = classNames.find { x -> x.startsWith("ct") } ?: return null
val num = className.drop(2).toIntOrNull(16) ?: return null
return 2.0.pow(num).toInt().toString()
}
private fun getNextTimestamp(root: Element): Long {
return root.getElementById("unext")
?.attrAsAbsoluteUrlOrNull("href")
?.toHttpUrlOrNull()
?.queryParameter("next")
?.toLongOrNull() ?: 1
}
private fun MangaListFilter.Advanced.toSearchQuery(): String? {
val joiner = StringUtil.StringJoiner(" ")
for (tag in tags) {
if (tag.key.isNumeric()) {
continue
}
joiner.add("tag:\"")
joiner.append(tag.key)
joiner.append("\"$")
}
for (tag in tagsExclude) {
if (tag.key.isNumeric()) {
continue
}
joiner.add("-tag:\"")
joiner.append(tag.key)
joiner.append("\"$")
}
locale?.let { lc ->
joiner.add("language:\"")
joiner.append(lc.toLanguagePath())
joiner.append("\"$")
}
return joiner.complete().takeUnless { it.isEmpty() }
}
}

706
src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/HitomiLaParser.kt
View File

@@ -1,706 +0,0 @@
package org.koitharu.kotatsu.parsers.site.all
import androidx.collection.ArraySet
import kotlinx.coroutines.async
import kotlinx.coroutines.awaitAll
import kotlinx.coroutines.coroutineScope
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
import okhttp3.Headers
import org.json.JSONArray
import org.json.JSONObject
import org.jsoup.Jsoup
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import org.koitharu.kotatsu.parsers.util.json.getStringOrNull
import org.koitharu.kotatsu.parsers.util.json.mapJSON
import java.nio.ByteBuffer
import java.nio.ByteOrder
import java.security.MessageDigest
import java.text.SimpleDateFormat
import java.util.*
import kotlin.math.min
@OptIn(ExperimentalUnsignedTypes::class)
@MangaSourceParser("HITOMILA", "Hitomi.La", type = ContentType.HENTAI)
class HitomiLaParser(context: MangaLoaderContext) : MangaParser(context, MangaSource.HITOMILA) {
override val configKeyDomain = ConfigKey.Domain("hitomi.la")
private val ltnBaseUrl get() = "https://${getDomain("ltn")}"
override val availableSortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.NEWEST,
SortOrder.POPULARITY,
)
private val localeMap: Map<Locale, String> = mapOf(
Locale("id") to "indonesian",
Locale("jv") to "javanese",
Locale("ca") to "catalan",
Locale("ceb") to "cebuano",
Locale("cs") to "czech",
Locale("da") to "danish",
Locale("de") to "german",
Locale("et") to "estonian",
Locale.ENGLISH to "english",
Locale("es") to "spanish",
Locale("eo") to "esperanto",
Locale("fr") to "french",
Locale("it") to "italian",
Locale("hi") to "hindi",
Locale("hu") to "hungarian",
Locale("pl") to "polish",
Locale("pt") to "portuguese",
Locale("vi") to "vietnamese",
Locale("tr") to "turkish",
Locale("ru") to "russian",
Locale("uk") to "ukrainian",
Locale("ar") to "arabic",
Locale.KOREAN to "korean",
Locale.CHINESE to "chinese",
Locale.JAPANESE to "japanese",
)
private fun Locale?.getSiteLang(): String = when (this) {
null -> "all"
else -> localeMap[this] ?: "all"
}
override suspend fun getAvailableLocales(): Set<Locale> = localeMap.keys
override suspend fun getAvailableTags(): Set<MangaTag> = coroutineScope {
('a'..'z').map { alphabet ->
async {
val doc = webClient.httpGet("https://$domain/alltags-$alphabet.html").parseHtml()
doc.select(".posts > li").mapNotNull { element ->
val num =
element.ownText().let {
Regex("""\((\d+)\)""").find(it)?.groupValues?.get(1)?.toIntOrNull() ?: 0
}
if (num > 100) {
val url = element.selectFirst("a")
val href =
url?.attrAsRelativeUrl("href")
?: return@mapNotNull null
MangaTag(
title = url.ownText().toTagTitle(),
key = href.tagUrlToTag(),
source = source,
)
} else {
null
}
}
}
}.awaitAll().flatten().toSet()
}
private var cachedSearchIds: List<Int> = emptyList()
override suspend fun getList(
offset: Int,
filter: MangaListFilter?,
): List<Manga> = when (filter) {
is MangaListFilter.Advanced -> {
if (filter.tags.isEmpty()) {
when (filter.sortOrder) {
SortOrder.POPULARITY -> {
getGalleryIDsFromNozomi(
"popular",
"today",
filter.locale.getSiteLang(),
offset.nextOffsetRange(),
)
}
else -> {
getGalleryIDsFromNozomi(null, "index", filter.locale.getSiteLang(), offset.nextOffsetRange())
}
}
} else {
if (offset == 0) {
cachedSearchIds =
hitomiSearch(
filter.tags.joinToString(" ") { it.key },
filter.sortOrder == SortOrder.POPULARITY,
filter.locale.getSiteLang(),
).toList()
}
cachedSearchIds.subList(offset, min(offset + 25, cachedSearchIds.size))
}
}
is MangaListFilter.Search -> {
if (offset == 0) {
cachedSearchIds = hitomiSearch(filter.query, filter.sortOrder == SortOrder.POPULARITY).toList()
}
cachedSearchIds.subList(offset, min(offset + 25, cachedSearchIds.size))
}
else -> getGalleryIDsFromNozomi(null, "popular", "all", offset.nextOffsetRange())
}.toMangaList()
private fun Int.nextOffsetRange(): LongRange {
val bytes = this * 4L
return bytes.until(bytes + 100L)
}
private suspend fun hitomiSearch(
query: String,
sortByPopularity: Boolean = false,
language: String = "all",
): Set<Int> =
coroutineScope {
val terms = query
.trim()
.replace(Regex("""^\?"""), "")
.lowercase()
.split(Regex("\\s+"))
.map {
it.replace('_', ' ')
}
val positiveTerms = LinkedList<String>()
val negativeTerms = LinkedList<String>()
for (term in terms) {
if (term.startsWith("-")) {
negativeTerms.push(term.removePrefix("-"))
} else if (term.isNotBlank()) {
positiveTerms.push(term)
}
}
val positiveResults = positiveTerms.map {
async {
runCatchingCancellable {
getGalleryIDsForQuery(it, language)
}.getOrDefault(emptySet())
}
}
val negativeResults = negativeTerms.map {
async {
runCatchingCancellable {
getGalleryIDsForQuery(it, language)
}.getOrDefault(emptySet())
}
}
val results = when {
sortByPopularity -> getGalleryIDsFromNozomi(null, "popular", language)
positiveTerms.isEmpty() -> getGalleryIDsFromNozomi(null, "index", language)
else -> emptySet()
}.toMutableSet()
fun filterPositive(newResults: Set<Int>) {
when {
results.isEmpty() -> results.addAll(newResults)
else -> results.retainAll(newResults)
}
}
fun filterNegative(newResults: Set<Int>) {
results.removeAll(newResults)
}
// positive results
positiveResults.forEach {
filterPositive(it.await())
}
// negative results
negativeResults.forEach {
filterNegative(it.await())
}
results
}
// search.js
private suspend fun getGalleryIDsForQuery(
query: String,
language: String = "all",
): Set<Int> {
query.replace("_", " ").let {
if (it.indexOf(':') > -1) {
val sides = it.split(":")
val ns = sides[0]
var tag = sides[1]
var area: String? = ns
var lang = language
when (ns) {
"female", "male" -> {
area = "tag"
tag = it
}
"language" -> {
area = null
lang = tag
tag = "index"
}
}
return getGalleryIDsFromNozomi(area, tag, lang)
}
val key = hashTerm(it)
val node = getGalleryNodeAtAddress(0)
val data = bSearch(key, node) ?: return emptySet()
return getGalleryIDsFromData(data)
}
}
private suspend fun getGalleryIDsFromData(data: Pair<Long, Int>): Set<Int> {
val url = "$ltnBaseUrl/galleriesindex/galleries.${galleriesIndexVersion.get()}.data"
val (offset, length) = data
require(length in 1..100000000) {
"Length $length is too long"
}
val inbuf = getRangedResponse(url, offset.until(offset + length))
val galleryIDs = mutableSetOf<Int>()
val buffer =
ByteBuffer
.wrap(inbuf)
.order(ByteOrder.BIG_ENDIAN)
val numberOfGalleryIDs = buffer.int
val expectedLength = numberOfGalleryIDs * 4 + 4
require(numberOfGalleryIDs in 1..10000000) {
"number_of_galleryids $numberOfGalleryIDs is too long"
}
require(inbuf.size == expectedLength) {
"inbuf.byteLength ${inbuf.size} != expected_length $expectedLength"
}
for (i in 0.until(numberOfGalleryIDs))
galleryIDs.add(buffer.int)
return galleryIDs
}
private suspend fun bSearch(
key: UByteArray,
node: Node,
): Pair<Long, Int>? {
fun compareArrayBuffers(
dv1: UByteArray,
dv2: UByteArray,
): Int {
val top = min(dv1.size, dv2.size)
for (i in 0.until(top)) {
if (dv1[i] < dv2[i]) {
return -1
} else if (dv1[i] > dv2[i]) {
return 1
}
}
return 0
}
fun locateKey(
key: UByteArray,
node: Node,
): Pair<Boolean, Int> {
for (i in node.keys.indices) {
val cmpResult = compareArrayBuffers(key, node.keys[i])
if (cmpResult <= 0) {
return Pair(cmpResult == 0, i)
}
}
return Pair(false, node.keys.size)
}
fun isLeaf(node: Node): Boolean {
for (subnode in node.subNodeAddresses)
if (subnode != 0L) {
return false
}
return true
}
if (node.keys.isEmpty()) {
return null
}
val (there, where) = locateKey(key, node)
if (there) {
return node.datas[where]
} else if (isLeaf(node)) {
return null
}
val nextNode = getGalleryNodeAtAddress(node.subNodeAddresses[where])
return bSearch(key, nextNode)
}
private suspend fun getGalleryIDsFromNozomi(
area: String?,
tag: String,
language: String,
range: LongRange? = null,
): Set<Int> {
val nozomiAddress = when (area) {
null -> "$ltnBaseUrl/$tag-$language.nozomi"
else -> "$ltnBaseUrl/$area/$tag-$language.nozomi"
}
val bytes = getRangedResponse(nozomiAddress, range)
val nozomi = mutableSetOf<Int>()
val arrayBuffer = ByteBuffer
.wrap(bytes)
.order(ByteOrder.BIG_ENDIAN)
while (arrayBuffer.hasRemaining())
nozomi.add(arrayBuffer.int)
return nozomi
}
private val galleriesIndexVersion = SuspendLazy {
webClient.httpGet("$ltnBaseUrl/galleriesindex/version?_=${System.currentTimeMillis()}").parseRaw()
}
private data class Node(
val keys: List<UByteArray>,
val datas: List<Pair<Long, Int>>,
val subNodeAddresses: List<Long>,
)
private fun decodeNode(data: ByteArray): Node {
val buffer = ByteBuffer
.wrap(data)
.order(ByteOrder.BIG_ENDIAN)
val uData = data.toUByteArray()
val numberOfKeys = buffer.int
val keys = ArrayList<UByteArray>()
for (i in 0.until(numberOfKeys)) {
val keySize = buffer.int
if (keySize == 0 || keySize > 32) {
throw Exception("fatal: !keySize || keySize > 32")
}
keys.add(uData.sliceArray(buffer.position().until(buffer.position() + keySize)))
buffer.position(buffer.position() + keySize)
}
val numberOfDatas = buffer.int
val datas = ArrayList<Pair<Long, Int>>()
for (i in 0.until(numberOfDatas)) {
val offset = buffer.long
val length = buffer.int
datas.add(Pair(offset, length))
}
val numberOfSubNodeAddresses = 16 + 1
val subNodeAddresses = ArrayList<Long>()
for (i in 0.until(numberOfSubNodeAddresses)) {
val subNodeAddress = buffer.long
subNodeAddresses.add(subNodeAddress)
}
return Node(keys, datas, subNodeAddresses)
}
private suspend fun getGalleryNodeAtAddress(address: Long): Node {
val url = "$ltnBaseUrl/galleriesindex/galleries.${galleriesIndexVersion.get()}.index"
val nodedata = getRangedResponse(url, address.until(address + 464))
return decodeNode(nodedata)
}
private suspend fun getRangedResponse(
url: String,
range: LongRange? = null,
): ByteArray {
val rangeHeaders = when (range) {
null -> Headers.headersOf()
else -> Headers.headersOf("Range", "bytes=${range.first}-${range.last}")
}
return webClient.httpGet(url, rangeHeaders).parseBytes()
}
private fun hashTerm(term: String): UByteArray {
return sha256(term.toByteArray()).copyOfRange(0, 4).toUByteArray()
}
private fun sha256(data: ByteArray): ByteArray {
return MessageDigest.getInstance("SHA-256").digest(data)
}
private suspend fun Collection<Int>.toMangaList(): List<Manga> = coroutineScope {
map { id ->
async {
runCatchingCancellable {
val doc = webClient.httpGet("$ltnBaseUrl/galleryblock/$id.html").let {
val baseUri = it.request.url.toString()
val html = it.parseRaw()
Jsoup.parse(rewriteTnPaths(html), baseUri)
}
Manga(
id = generateUid(id.toString()),
title = doc.selectFirstOrThrow("h1").text(),
url = id.toString(),
coverUrl =
"https:" +
doc.selectFirstOrThrow("picture > source")
.attr("data-srcset")
.substringBefore(" "),
publicUrl =
doc.selectFirstOrThrow("h1 > a")
.attrAsRelativeUrl("href")
.toAbsoluteUrl(domain),
author = null,
tags = emptySet(),
isNsfw = true,
rating = RATING_UNKNOWN,
altTitle = null,
state = null,
source = source,
)
}.getOrNull()
}
}.awaitAll().filterNotNull()
}
override suspend fun getDetails(manga: Manga): Manga {
val json = webClient.httpGet("$ltnBaseUrl/galleries/${manga.url}.js")
.parseRaw()
.substringAfter("var galleryinfo = ")
.let(::JSONObject)
return manga.copy(
title = json.getString("title"),
largeCoverUrl =
json.getJSONArray("files").getJSONObject(0).let {
val hash = it.getString("hash")
val commonId = commonImageId()
val imageId = imageIdFromHash(hash)
val subDomain = 'a' + subdomainOffset(imageId)
"https://${getDomain("${subDomain}a")}/webp/$commonId$imageId/$hash.webp"
},
author =
json.optJSONArray("artists")
?.mapJSON { it.getString("artist").toCamelCase() }
?.joinToString(),
publicUrl = json.getString("galleryurl").toAbsoluteUrl(domain),
tags =
buildSet {
json.optJSONArray("characters")
?.mapToTags("character")
?.let(::addAll)
json.optJSONArray("tags")
?.mapToTags("tag")
?.let(::addAll)
json.optJSONArray("artists")
?.mapToTags("artist")
?.let(::addAll)
json.optJSONArray("parodys")
?.mapToTags("parody")
?.let(::addAll)
json.optJSONArray("groups")
?.mapToTags("group")
?.let(::addAll)
},
chapters = listOf(
MangaChapter(
id = generateUid(manga.url),
url = manga.url,
name = json.getString("title"),
scanlator = json.getString("type").toTitleCase(),
number = 1,
branch = json.getString("language_localname"),
source = source,
uploadDate = dateFormat.tryParse(json.getString("date").substringBeforeLast("-")),
),
),
)
}
private val dateFormat = SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.ENGLISH)
private fun JSONArray.mapToTags(key: String): Set<MangaTag> {
val tags = ArraySet<MangaTag>(length())
mapJSON {
MangaTag(
title =
it.getString(key).toCamelCase().let { title ->
if (it.getStringOrNull("female")?.toIntOrNull() == 1) {
"$title ♀"
} else if (it.getStringOrNull("male")?.toIntOrNull() == 1) {
"$title ♂"
} else {
title
}
},
key = it.getString("url").tagUrlToTag(),
source = source,
).let(tags::add)
}
return tags
}
private fun String.tagUrlToTag(): String {
val urlContent = this.split("/")
val ns = urlContent[1]
val tag =
urlContent[2]
.substringBeforeLast("-")
.urlDecode()
.replace(" ", "_")
return if (tag.split(":")[0] in listOf("female", "male")) {
tag
} else {
"$ns:$tag"
}
}
override suspend fun getRelatedManga(seed: Manga): List<Manga> {
val json = webClient.httpGet("$ltnBaseUrl/galleries/${seed.url}.js")
.parseRaw()
.substringAfter("var galleryinfo = ")
.let(::JSONObject)
// any better way to get List<Int> from this json?
return json.getJSONArray("related").let {
0.until(it.length()).map { i -> it.getInt(i) }
}.toMangaList()
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val json = webClient.httpGet("$ltnBaseUrl/galleries/${chapter.url}.js")
.parseRaw()
.substringAfter("var galleryinfo = ")
.let(::JSONObject)
return json.getJSONArray("files").mapJSON { image ->
val hash = image.getString("hash")
val commonId = commonImageId()
val imageId = imageIdFromHash(hash)
val subDomain = 'a' + subdomainOffset(imageId)
MangaPage(
id = generateUid(hash),
url = "https://${getDomain("${subDomain}a")}/webp/$commonId$imageId/$hash.webp",
preview = "https://${getDomain("${subDomain}tn")}/webpsmalltn/${thumbPathFromHash(hash)}/$hash.webp",
source = source,
)
}
}
// / --->
private var scriptLastRetrieval: Long? = null
private val mutex = Mutex()
private var subdomainOffsetDefault = 0
private val subdomainOffsetMap = mutableMapOf<Int, Int>()
private var commonImageId = ""
private suspend fun refreshScript() = mutex.withLock {
if (scriptLastRetrieval == null || (scriptLastRetrieval!! + 60000) < System.currentTimeMillis()) {
val ggScript = webClient.httpGet("$ltnBaseUrl/gg.js?_=${System.currentTimeMillis()}").parseRaw()
subdomainOffsetDefault = Regex("var o = (\\d)").find(ggScript)!!.groupValues[1].toInt()
val o = Regex("o = (\\d); break;").find(ggScript)!!.groupValues[1].toInt()
subdomainOffsetMap.clear()
Regex("case (\\d+):").findAll(ggScript).forEach {
val case = it.groupValues[1].toInt()
subdomainOffsetMap[case] = o
}
commonImageId = Regex("b: '(.+)'").find(ggScript)!!.groupValues[1]
scriptLastRetrieval = System.currentTimeMillis()
}
}
// m <-- gg.js
private suspend fun subdomainOffset(imageId: Int): Int {
refreshScript()
return subdomainOffsetMap[imageId] ?: subdomainOffsetDefault
}
// b <-- gg.js
private suspend fun commonImageId(): String {
refreshScript()
return commonImageId
}
// s <-- gg.js
private fun imageIdFromHash(hash: String): Int {
val match = Regex("(..)(.)$").find(hash)
return match!!.groupValues.let { it[2] + it[1] }.toInt(16)
}
// real_full_path_from_hash <-- common.js
private fun thumbPathFromHash(hash: String): String {
return hash.replace(Regex("""^.*(..)(.)$"""), "$2/$1")
}
private suspend fun subdomainFromURL(url: String, base: String? = null): String {
var retval = "b"
if (!base.isNullOrBlank())
retval = base
val regex = Regex("""/[0-9a-f]{61}([0-9a-f]{2})([0-9a-f])""")
val hashMatch = regex.find(url) ?: return "a"
val imageId = hashMatch.groupValues.let { it[2] + it[1] }.toIntOrNull(16)
if (imageId != null) {
retval = ('a' + subdomainOffset(imageId)).toString() + retval
}
return retval
}
// rewrite_tn_paths <-- common.js
private suspend fun rewriteTnPaths(html: String): String {
val tnRegex = Regex("""//tn\.hitomi\.la/[^/]+/[0-9a-f]/[0-9a-f]{2}/[0-9a-f]{64}""")
val url = tnRegex.find(html)?.value ?: return html
val newSubdomain = subdomainFromURL(url, "tn")
val newUrl = url.replace(Regex("""//..?\.hitomi\.la/"""), "//${getDomain(newSubdomain)}/")
return html.replace(tnRegex, newUrl)
}
private fun String.toTagTitle(): String {
return toCamelCase()
.replace("♂", "(male)")
.replace("♀", "(female)")
}
}

186
src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/ImHentai.kt
View File

@@ -1,186 +0,0 @@
package org.koitharu.kotatsu.parsers.site.all
import androidx.collection.ArraySet
import kotlinx.coroutines.async
import kotlinx.coroutines.awaitAll
import kotlinx.coroutines.coroutineScope
import org.jsoup.nodes.Element
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import java.util.*
@MangaSourceParser("IMHENTAI", "ImHentai", type = ContentType.HENTAI)
internal class ImHentai(context: MangaLoaderContext) :
PagedMangaParser(context, MangaSource.IMHENTAI, pageSize = 20) {
override val availableSortOrders: Set<SortOrder> =
EnumSet.of(SortOrder.UPDATED, SortOrder.POPULARITY, SortOrder.RATING)
override val configKeyDomain = ConfigKey.Domain("imhentai.xxx")
override suspend fun getListPage(page: Int, filter: MangaListFilter?): List<Manga> {
val url = buildString {
append("https://")
append(domain)
append("/search/?page=")
append(page.toString())
when (filter) {
is MangaListFilter.Search -> {
append("&key=")
append(filter.query.urlEncoded())
}
is MangaListFilter.Advanced -> {
if (filter.tags.isNotEmpty()) {
append("&key=")
append(filter.tags.joinToString(separator = ",") { it.key })
}
var lang = "&en=1&jp=1&es=1&fr=1&kr=1&de=1&ru=1"
filter.locale?.let {
lang = "&en=0&jp=0&es=0&fr=0&kr=0&de=0&ru=0"
lang = lang.replace("${it.language}=0", "${it.language}=1")
}
append(lang)
when (filter.sortOrder) {
SortOrder.UPDATED -> append("&lt=1&pp=0")
SortOrder.POPULARITY -> append("&lt=0&pp=1")
SortOrder.RATING -> append("&lt=0&pp=0")
else -> append("&lt=1&pp=0")
}
}
null -> {
append("&lt=1&pp=0")
}
}
}
val doc = webClient.httpGet(url).parseHtml()
return doc.select("div.galleries div.thumb").map { div ->
val a = div.selectFirstOrThrow(".inner_thumb a")
val href = a.attrAsRelativeUrl("href")
Manga(
id = generateUid(href),
url = href,
publicUrl = href.toAbsoluteUrl(domain),
coverUrl = a.selectFirst("img")?.src().orEmpty(),
title = div.selectFirst(".caption")?.text().orEmpty(),
altTitle = null,
rating = RATING_UNKNOWN,
tags = emptySet(),
author = null,
state = null,
source = source,
isNsfw = isNsfwSource,
)
}
}
//Tags are deliberately reduced because there are too many and this slows down the application.
//only the most popular ones are taken.
override suspend fun getAvailableTags(): Set<MangaTag> {
return coroutineScope {
(1..3).map { page ->
async { getTags(page) }
}
}.awaitAll().flattenTo(ArraySet(360))
}
private suspend fun getTags(page: Int): Set<MangaTag> {
val url = "https://$domain/tags/popular/?page=$page"
val root = webClient.httpGet(url).parseHtml()
return root.parseTags()
}
private fun Element.parseTags() = select("div.stags a.tag_btn").mapToSet {
val href = it.attr("href").substringAfterLast("tag/").substringBeforeLast('/')
MangaTag(
key = href,
title = it.selectFirstOrThrow("h3.list_tag").text().toTitleCase(sourceLocale),
source = source,
)
}
override suspend fun getAvailableLocales(): Set<Locale> = setOf(
Locale.ENGLISH, Locale.JAPANESE, Locale("es"), Locale.FRENCH, Locale("kr"), Locale.GERMAN, Locale("ru"),
)
override suspend fun getDetails(manga: Manga): Manga = coroutineScope {
val fullUrl = manga.url.toAbsoluteUrl(domain)
val doc = webClient.httpGet(fullUrl).parseHtml()
manga.copy(
tags = doc.body().select("li:contains(Tags) a.tag").mapNotNullToSet {
val href = it.attr("href").substringAfterLast("tag/").substringBeforeLast('/')
val name = it.html().substringBeforeLast("<span")
MangaTag(
key = href,
title = name,
source = source,
)
},
author = doc.selectFirst("li:contains(Artists) a.tag")?.html()?.substringBefore("<span"),
chapters = listOf(
MangaChapter(
id = manga.id,
name = manga.title,
number = 1,
url = manga.url,
scanlator = null,
uploadDate = 0,
branch = doc.selectFirst("li:contains(Language) a.tag")?.html()?.substringBeforeLast("<span"),
source = source,
),
),
)
}
override suspend fun getRelatedManga(seed: Manga): List<Manga> {
val doc = webClient.httpGet(seed.url.toAbsoluteUrl(domain)).parseHtml()
val root = doc.body().selectFirstOrThrow("div.related")
return root.select("div.thumb").mapNotNull { div ->
val a = div.selectFirstOrThrow(".inner_thumb a")
val href = a.attrAsRelativeUrl("href")
Manga(
id = generateUid(href),
url = href,
publicUrl = href.toAbsoluteUrl(domain),
coverUrl = a.selectFirst("img")?.src().orEmpty(),
title = div.selectFirst(".caption")?.text().orEmpty(),
altTitle = null,
rating = RATING_UNKNOWN,
tags = emptySet(),
author = null,
state = null,
source = source,
isNsfw = false,
)
}
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(domain)
val doc = webClient.httpGet(fullUrl).parseHtml()
val totalPages = doc.selectFirstOrThrow(".pages").text().replace("Pages: ", "").toInt() + 1
val domainImg = doc.requireElementById("append_thumbs").selectFirstOrThrow("img").src()?.replace("1t.jpg", "")
val pages = ArrayList<MangaPage>(totalPages)
for (i in 1 until totalPages) {
val url = "$domainImg$i.jpg"
pages.add(
MangaPage(
id = generateUid(url),
url = url,
preview = null,
source = source,
),
)
}
return pages
}
}

354
src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/LineWebtoonsParser.kt
View File

@@ -1,354 +0,0 @@
package org.koitharu.kotatsu.parsers.site.all
import kotlinx.coroutines.async
import kotlinx.coroutines.coroutineScope
import okhttp3.Headers
import okhttp3.HttpUrl
import okhttp3.HttpUrl.Companion.toHttpUrl
import org.json.JSONObject
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.exception.NotFoundException
import org.koitharu.kotatsu.parsers.exception.ParseException
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import org.koitharu.kotatsu.parsers.util.json.*
import java.util.*
import javax.crypto.Mac
import javax.crypto.spec.SecretKeySpec
internal abstract class LineWebtoonsParser(
context: MangaLoaderContext,
source: MangaSource,
) : MangaParser(context, source) {
override val isMultipleTagsSupported = false
private val signer by lazy {
WebtoonsUrlSigner("gUtPzJFZch4ZyAGviiyH94P99lQ3pFdRTwpJWDlSGFfwgpr6ses5ALOxWHOIT7R1")
}
// we don't __really__ support changing this domain because:
// 1. I don't think other websites have this exact API
// 2. most communication is done with other domains (hosting API and static content), which are not configurable
// 3. we rely on the HTTP client setting the referer header to webtoons.com
//
// This effectively means that changing the domain will break the source. Yikes
override val configKeyDomain = ConfigKey.Domain("webtoons.com")
private val apiDomain = "global.apis.naver.com"
private val staticDomain = "webtoon-phinf.pstatic.net"
override val availableSortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.POPULARITY,
// doesn't actually sort by rating, but by likes
// this should be fine though
SortOrder.RATING,
SortOrder.UPDATED,
)
override val headers: Headers
get() = Headers.Builder()
.add("User-Agent", "nApps (Android 12;; linewebtoon; 3.1.0)")
.build()
override suspend fun getPageUrl(page: MangaPage): String {
return page.url.toAbsoluteUrl(staticDomain)
}
// some language tags do not map perfectly to the ones used by the API
private val languageCode: String
get() = when (val tag = sourceLocale.toLanguageTag()) {
"in" -> "id"
"zh" -> "zh-hant"
else -> tag
}
private suspend fun getChapters(titleNo: Long): List<MangaChapter> {
val firstResult = makeRequest(
url = "/lineWebtoon/webtoon/challengeEpisodeList.json?v=2&titleNo=$titleNo&startIndex=0&pageSize=30",
)
val totalEpisodeCount = firstResult
.getJSONObject("episodeList")
.getInt("totalServiceEpisodeCount")
val episodes = firstResult
.getJSONObject("episodeList")
.getJSONArray("episode")
.toJSONList()
.toMutableList()
while (episodes.count() < totalEpisodeCount) {
val page = makeRequest(
url = "/lineWebtoon/webtoon/challengeEpisodeList.json?v=2&titleNo=$titleNo&startIndex=${episodes.count()}&pageSize=30",
).getJSONObject("episodeList")
.getJSONArray("episode")
.toJSONList()
episodes.addAll(page)
}
return episodes.mapChapters { i, jo ->
MangaChapter(
id = generateUid("$titleNo-$i"),
name = jo.getString("episodeTitle"),
number = jo.getInt("episodeSeq"),
url = "$titleNo-${jo.get("episodeNo")}",
uploadDate = jo.getLong("modifyYmdt"),
branch = null,
scanlator = null,
source = source,
)
}.sortedBy { it.number }
}
override suspend fun getDetails(manga: Manga): Manga = coroutineScope {
val titleNo = manga.url.toLong()
val chaptersDeferred = async { getChapters(titleNo) }
makeRequest("/lineWebtoon/webtoon/challengeTitleInfo.json?v=2&titleNo=${titleNo}")
.getJSONObject("titleInfo")
.let { jo ->
Manga(
id = generateUid(titleNo),
title = jo.getString("title"),
altTitle = null,
url = "$titleNo",
publicUrl = "https://$domain/$languageCode/canvas/a/list?title_no=${titleNo}",
rating = jo.getFloatOrDefault("starScoreAverage", -10f) / 10f,
isNsfw = jo.getBooleanOrDefault("ageGradeNotice", isNsfwSource),
coverUrl = jo.getString("thumbnail").toAbsoluteUrl(staticDomain),
largeCoverUrl = jo.getStringOrNull("thumbnailVertical")?.toAbsoluteUrl(staticDomain),
tags = setOf(parseTag(jo.getJSONObject("genreInfo"))),
author = jo.getStringOrNull("writingAuthorName"),
description = jo.getString("synopsis"),
// I don't think the API provides this info
state = null,
chapters = chaptersDeferred.await(),
source = source,
)
}
}
override suspend fun getList(offset: Int, filter: MangaListFilter?): List<Manga> {
val manga =
when (filter) {
is MangaListFilter.Search -> {
makeRequest("/lineWebtoon/webtoon/searchChallenge?query=${filter.query.urlEncoded()}&startIndex=${offset + 1}&pageSize=20")
.getJSONObject("challengeSearch")
.getJSONArray("titleList")
.mapJSON { jo ->
val titleNo = jo.getLong("titleNo")
Manga(
id = generateUid(titleNo),
title = jo.getString("title"),
altTitle = null,
url = titleNo.toString(),
publicUrl = "https://$domain/$languageCode/canvas/a/list?title_no=$titleNo",
rating = RATING_UNKNOWN,
isNsfw = isNsfwSource,
coverUrl = jo.getString("thumbnail").toAbsoluteUrl(staticDomain),
largeCoverUrl = null,
tags = emptySet(),
author = jo.getStringOrNull("writingAuthorName"),
description = null,
state = null,
source = source,
)
}
}
is MangaListFilter.Advanced -> {
val genre = filter.tags.oneOrThrowIfMany()?.key ?: "ALL"
val sortOrderStr = when (filter.sortOrder) {
SortOrder.UPDATED -> "UPDATE"
SortOrder.POPULARITY -> "READ_COUNT"
SortOrder.RATING -> "LIKEIT"
else -> throw IllegalArgumentException("Unsupported sort order: ${filter.sortOrder}")
}
val result =
makeRequest("/lineWebtoon/webtoon/challengeGenreTitleList.json?genre=$genre&sortOrder=$sortOrderStr&startIndex=${offset + 1}&pageSize=20")
val genres = result.getJSONObject("genreList")
.getJSONArray("challengeGenres")
.mapJSON { jo -> parseTag(jo) }
.associateBy { tag -> tag.key }
result
.getJSONObject("titleList")
.getJSONArray("titles")
.mapJSON { jo ->
val titleNo = jo.getLong("titleNo")
Manga(
id = generateUid(titleNo),
title = jo.getString("title"),
altTitle = null,
url = titleNo.toString(),
publicUrl = "https://$domain/$languageCode/canvas/a/list?title_no=$titleNo",
rating = jo.getFloatOrDefault("starScoreAverage", -10f) / 10f,
isNsfw = jo.getBooleanOrDefault("ageGradeNotice", isNsfwSource),
coverUrl = jo.getString("thumbnail").toAbsoluteUrl(staticDomain),
largeCoverUrl = jo.getStringOrNull("thumbnailVertical")?.toAbsoluteUrl(staticDomain),
tags = setOfNotNull(genres[jo.getString("representGenre")]),
author = jo.getStringOrNull("writingAuthorName"),
description = jo.getString("synopsis"),
// I don't think the API provides this info
state = null,
source = source,
)
}
}
null -> {
val result =
makeRequest("/lineWebtoon/webtoon/challengeGenreTitleList.json?genre=ALL&sortOrder=UPDATE&startIndex=${offset + 1}&pageSize=20")
val genres = result.getJSONObject("genreList")
.getJSONArray("challengeGenres")
.mapJSON { jo -> parseTag(jo) }
.associateBy { tag -> tag.key }
result
.getJSONObject("titleList")
.getJSONArray("titles")
.mapJSON { jo ->
val titleNo = jo.getLong("titleNo")
Manga(
id = generateUid(titleNo),
title = jo.getString("title"),
altTitle = null,
url = titleNo.toString(),
publicUrl = "https://$domain/$languageCode/canvas/a/list?title_no=$titleNo",
rating = jo.getFloatOrDefault("starScoreAverage", -10f) / 10f,
isNsfw = jo.getBooleanOrDefault("ageGradeNotice", isNsfwSource),
coverUrl = jo.getString("thumbnail").toAbsoluteUrl(staticDomain),
largeCoverUrl = jo.getStringOrNull("thumbnailVertical")?.toAbsoluteUrl(staticDomain),
tags = setOfNotNull(genres[jo.getString("representGenre")]),
author = jo.getStringOrNull("writingAuthorName"),
description = jo.getString("synopsis"),
// I don't think the API provides this info
state = null,
source = source,
)
}
}
}
return manga
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val (titleNo, episodeNo) = requireNotNull(chapter.url.splitTwoParts('-'))
return makeRequest("/lineWebtoon/webtoon/challengeEpisodeInfo.json?v=2&titleNo=$titleNo&episodeNo=$episodeNo")
.getJSONObject("episodeInfo")
.getJSONArray("imageInfo")
.mapJSONIndexed { i, jo ->
MangaPage(
id = generateUid("$titleNo-$episodeNo-$i"),
url = jo.getString("url"),
preview = null,
source = source,
)
}
}
private fun parseTag(jo: JSONObject): MangaTag {
return MangaTag(
title = jo.getString("name"),
key = jo.getString("code"),
source = source,
)
}
override suspend fun getAvailableTags(): Set<MangaTag> {
return makeRequest("/lineWebtoon/webtoon/challengeGenreList.json")
.getJSONObject("genreList")
.getJSONArray("challengeGenres")
.mapJSONToSet { jo -> parseTag(jo) }
}
private suspend fun makeRequest(url: String): JSONObject {
val resp = webClient.httpGet(finalizeUrl(url))
val message: JSONObject? = resp.parseJson().optJSONObject("message")
return when (resp.code) {
in 200..299 -> checkNotNull(message).getJSONObject("result")
404 -> throw NotFoundException(message?.getStringOrNull("message").orEmpty(), url)
else -> {
val code = message?.getIntOrDefault("code", 0)
val errorMessage = message?.getStringOrNull("message")
throw ParseException("Api error (code=$code): $errorMessage", url)
}
}
}
private fun finalizeUrl(url: String): HttpUrl {
val httpUrl = url.toAbsoluteUrl(apiDomain).toHttpUrl()
val builder = httpUrl.newBuilder()
.addQueryParameter("serviceZone", "GLOBAL")
if (httpUrl.queryParameter("v") == null) {
builder.addQueryParameter("v", "1")
}
builder.addQueryParameter("language", languageCode)
.addQueryParameter("locale", "languageCode")
.addQueryParameter("platform", "APP_ANDROID")
signer.makeEncryptUrl(builder)
return builder.build()
}
@MangaSourceParser("LINEWEBTOONS_EN", "LineWebtoons English", "en", type = ContentType.MANGA)
class English(context: MangaLoaderContext) : LineWebtoonsParser(context, MangaSource.LINEWEBTOONS_EN)
@MangaSourceParser("LINEWEBTOONS_ZH", "LineWebtoons Chinese", "zh", type = ContentType.MANGA)
class Chinese(context: MangaLoaderContext) : LineWebtoonsParser(context, MangaSource.LINEWEBTOONS_ZH)
@MangaSourceParser("LINEWEBTOONS_TH", "LineWebtoons Thai", "th", type = ContentType.MANGA)
class Thai(context: MangaLoaderContext) : LineWebtoonsParser(context, MangaSource.LINEWEBTOONS_TH)
@MangaSourceParser("LINEWEBTOONS_ID", "LineWebtoons Indonesian", "id", type = ContentType.MANGA)
class Indonesian(context: MangaLoaderContext) : LineWebtoonsParser(context, MangaSource.LINEWEBTOONS_ID)
@MangaSourceParser("LINEWEBTOONS_ES", "LineWebtoons Spanish", "es", type = ContentType.MANGA)
class Spanish(context: MangaLoaderContext) : LineWebtoonsParser(context, MangaSource.LINEWEBTOONS_ES)
@MangaSourceParser("LINEWEBTOONS_FR", "LineWebtoons French", "fr", type = ContentType.MANGA)
class French(context: MangaLoaderContext) : LineWebtoonsParser(context, MangaSource.LINEWEBTOONS_FR)
@MangaSourceParser("LINEWEBTOONS_DE", "LineWebtoons German", "de", type = ContentType.MANGA)
class German(context: MangaLoaderContext) : LineWebtoonsParser(context, MangaSource.LINEWEBTOONS_DE)
private inner class WebtoonsUrlSigner(private val secret: String) {
private val mac = Mac.getInstance("HmacSHA1").apply {
this.init(SecretKeySpec(secret.encodeToByteArray(), "HmacSHA1"))
}
private fun getMessage(url: String, msgpad: String): String {
return url.substring(0, 0xFF.coerceAtMost(url.length)) + msgpad
}
private fun getMessageDigest(s: String): String {
val signedMessage = synchronized(mac) { mac.doFinal(s.toByteArray()) }
return context.encodeBase64(signedMessage)
}
fun makeEncryptUrl(urlBuilder: HttpUrl.Builder) {
val msgPad = Calendar.getInstance().timeInMillis.toString()
val digest = getMessageDigest(getMessage(urlBuilder.build().toString(), msgPad))
urlBuilder
.addQueryParameter("msgpad", msgPad)
.addQueryParameter("md", digest)
// .addEncodedQueryParameter("md", digest.urlEncoded())
}
}
}

268
src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/MangaPark.kt
View File

@@ -1,268 +0,0 @@
package org.koitharu.kotatsu.parsers.site.all
import androidx.collection.ArrayMap
import kotlinx.coroutines.coroutineScope
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import java.text.DateFormat
import java.text.SimpleDateFormat
import java.util.*
@MangaSourceParser("MANGAPARK", "MangaPark")
internal class MangaPark(context: MangaLoaderContext) :
PagedMangaParser(context, MangaSource.MANGAPARK, pageSize = 36) {
override val availableSortOrders: Set<SortOrder> = EnumSet.allOf(SortOrder::class.java)
override val availableStates: Set<MangaState> = EnumSet.allOf(MangaState::class.java)
override val availableContentRating: Set<ContentRating> = EnumSet.of(ContentRating.SAFE)
override val isTagsExclusionSupported: Boolean = true
override val configKeyDomain = ConfigKey.Domain("mangapark.net")
private val tagsMap = SuspendLazy(::parseTags)
init {
context.cookieJar.insertCookies(domain, "nsfw", "2")
}
override suspend fun getListPage(page: Int, filter: MangaListFilter?): List<Manga> {
val url = buildString {
append("https://")
append(domain)
append("/search?page=")
append(page.toString())
when (filter) {
is MangaListFilter.Search -> {
append("&word=")
append(filter.query.urlEncoded())
}
is MangaListFilter.Advanced -> {
append("&genres=")
if (filter.tags.isNotEmpty()) {
appendAll(filter.tags, ",") { it.key }
}
append("|")
if (filter.tagsExclude.isNotEmpty()) {
appendAll(filter.tagsExclude, ",") { it.key }
}
if (filter.contentRating.isNotEmpty()) {
filter.contentRating.oneOrThrowIfMany()?.let {
append(
when (it) {
ContentRating.SAFE -> append(",gore,bloody,violence,ecchi,adult,mature,smut,hentai")
else -> append("")
},
)
}
}
filter.states.oneOrThrowIfMany()?.let {
append("&status=")
append(
when (it) {
MangaState.ONGOING -> "ongoing"
MangaState.FINISHED -> "completed"
MangaState.PAUSED -> "hiatus"
MangaState.ABANDONED -> "cancelled"
MangaState.UPCOMING -> "pending"
},
)
}
append("&sortby=")
append(
when (filter.sortOrder) {
SortOrder.POPULARITY -> "views_d000"
SortOrder.UPDATED -> "field_update"
SortOrder.NEWEST -> "field_create"
SortOrder.ALPHABETICAL -> "field_name"
SortOrder.RATING -> "field_score"
else -> ""
},
)
filter.locale?.let {
append("&lang=")
append(it.language)
}
}
null -> append("&sortby=field_update")
}
}
val doc = webClient.httpGet(url).parseHtml()
return doc.select("div.grid.gap-5 div.flex.border-b").map { div ->
val href = div.selectFirstOrThrow("a").attrAsRelativeUrl("href")
Manga(
id = generateUid(href),
url = href,
publicUrl = href.toAbsoluteUrl(div.host ?: domain),
coverUrl = div.selectFirst("img")?.src().orEmpty(),
title = div.selectFirst("h3")?.text().orEmpty(),
altTitle = null,
rating = div.selectFirst("span.text-yellow-500")?.text()?.toFloatOrNull()?.div(10F) ?: RATING_UNKNOWN,
tags = emptySet(),
author = null,
state = null,
source = source,
isNsfw = isNsfwSource,
)
}
}
override suspend fun getAvailableTags(): Set<MangaTag> {
return tagsMap.get().values.toSet()
}
private suspend fun parseTags(): Map<String, MangaTag> {
val tagElements = webClient.httpGet("https://$domain/search").parseHtml()
.select("div.flex-col:contains(Genres) div.whitespace-nowrap")
val tagMap = ArrayMap<String, MangaTag>(tagElements.size)
for (el in tagElements) {
val name = el.selectFirstOrThrow("span.whitespace-nowrap").text().toTitleCase(sourceLocale)
if (name.isEmpty()) continue
tagMap[name] = MangaTag(
title = name,
key = el.attr("q:key") ?: continue,
source = source,
)
}
return tagMap
}
override suspend fun getAvailableLocales(): Set<Locale> = setOf(
Locale("af"), Locale("sq"), Locale("am"), Locale("ar"), Locale("hy"),
Locale("az"), Locale("be"), Locale("bn"), Locale("zh_hk"), Locale("zh_tw"),
Locale.CHINESE, Locale("ceb"), Locale("ca"), Locale("km"), Locale("my"),
Locale("bg"), Locale("bs"), Locale("hr"), Locale("cs"), Locale("da"),
Locale("nl"), Locale.ENGLISH, Locale("et"), Locale("fo"), Locale("fil"),
Locale("fi"), Locale("he"), Locale("ha"), Locale("jv"), Locale("lb"),
Locale("mn"), Locale("ro"), Locale("si"), Locale("ta"), Locale("uz"),
Locale("ur"), Locale("tg"), Locale("sd"), Locale("pt_br"), Locale("mo"),
Locale("lt"), Locale.JAPANESE, Locale.ITALIAN, Locale("ht"), Locale("lv"),
Locale("mr"), Locale("pt"), Locale("sn"), Locale("sv"), Locale("uk"),
Locale("tk"), Locale("sw"), Locale("st"), Locale("pl"), Locale("mi"),
Locale("lo"), Locale("ga"), Locale("gu"), Locale("gn"), Locale("id"),
Locale("ky"), Locale("mt"), Locale("fa"), Locale("sh"), Locale("es_419"),
Locale("tr"), Locale("to"), Locale("vi"), Locale("es"), Locale("sr"),
Locale("ps"), Locale("ml"), Locale("ku"), Locale("ig"), Locale("el"),
Locale.GERMAN, Locale("is"), Locale.KOREAN, Locale("ms"), Locale("ny"), Locale("sm"),
Locale("so"), Locale("ti"), Locale("zu"), Locale("yo"), Locale("th"),
Locale("sl"), Locale("ru"), Locale("no"), Locale("mg"), Locale("kk"),
Locale("hu"), Locale("ka"), Locale.FRENCH, Locale("hi"), Locale("kn"),
Locale("mk"), Locale("ne"), Locale("rm"), Locale("sk"), Locale("te"),
)
override suspend fun getDetails(manga: Manga): Manga = coroutineScope {
val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml()
val tagMap = tagsMap.get()
val selectTag = doc.select("div[q:key=30_2] span.whitespace-nowrap")
val tags = selectTag.mapNotNullToSet { tagMap[it.text()] }
val nsfw = tags.any { t -> t.key == "hentai" || t.key == "adult" }
val dateFormat = SimpleDateFormat("dd/MM/yyyy", sourceLocale)
manga.copy(
altTitle = doc.selectFirst("div[q:key=tz_2]")?.text().orEmpty(),
author = doc.selectFirst("div[q:key=tz_4]")?.text().orEmpty(),
description = doc.selectFirst("react-island[q:key=0a_9]")?.html().orEmpty(),
state = when (doc.selectFirst("span[q:key=Yn_5]")?.text()?.lowercase()) {
"ongoing" -> MangaState.ONGOING
"completed" -> MangaState.FINISHED
"hiatus" -> MangaState.PAUSED
"cancelled" -> MangaState.ABANDONED
else -> null
},
tags = tags,
isNsfw = nsfw,
chapters = doc.body().select("div.group.flex div.px-2").mapChapters { i, div ->
val a = div.selectFirstOrThrow("a")
val href = a.attrAsRelativeUrl("href")
val dateText = div.selectFirst("span[q:key=Ee_0]")?.text()
MangaChapter(
id = generateUid(href),
name = a.text(),
number = i + 1,
url = href,
uploadDate = parseChapterDate(
dateFormat,
dateText,
),
source = source,
scanlator = null,
branch = null,
)
},
)
}
private fun parseChapterDate(dateFormat: DateFormat, date: String?): Long {
val d = date?.lowercase() ?: return 0
return when {
d.endsWith(" ago") -> parseRelativeDate(date)
d.startsWith("just now") -> Calendar.getInstance().apply {
set(Calendar.HOUR_OF_DAY, 0)
set(Calendar.MINUTE, 0)
set(Calendar.SECOND, 0)
set(Calendar.MILLISECOND, 0)
}.timeInMillis
else -> dateFormat.tryParse(date)
}
}
private fun parseRelativeDate(date: String): Long {
val number = Regex("""(\d+)""").find(date)?.value?.toIntOrNull() ?: return 0
val cal = Calendar.getInstance()
return when {
WordSet("second").anyWordIn(date) -> cal.apply { add(Calendar.SECOND, -number) }.timeInMillis
WordSet("minute", "minutes", "mins", "min").anyWordIn(date) -> cal.apply {
add(
Calendar.MINUTE,
-number,
)
}.timeInMillis
WordSet("hour", "hours").anyWordIn(date) -> cal.apply { add(Calendar.HOUR, -number) }.timeInMillis
WordSet("day", "days").anyWordIn(date) -> cal.apply { add(Calendar.DAY_OF_MONTH, -number) }.timeInMillis
WordSet("month", "months").anyWordIn(date) -> cal.apply { add(Calendar.MONTH, -number) }.timeInMillis
WordSet("year").anyWordIn(date) -> cal.apply { add(Calendar.YEAR, -number) }.timeInMillis
else -> 0
}
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml()
val script = if (doc.selectFirst("script:containsData(comic-)") != null) {
doc.selectFirstOrThrow("script:containsData(comic-)").data()
.substringAfterLast("\"comic-").split("\",\"")
} else {
doc.selectFirstOrThrow("script:containsData(manga-)").data()
.substringAfterLast("\"manga-").split("\",\"")
}
return script.mapNotNull { url ->
if (!url.startsWith("https://")) {
return@mapNotNull null
} else {
MangaPage(
id = generateUid(url),
url = url,
preview = null,
source = source,
)
}
}
}
}

321
src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/MangaPlusParser.kt
View File

@@ -1,321 +0,0 @@
package org.koitharu.kotatsu.parsers.site.all
import okhttp3.Headers
import okhttp3.HttpUrl.Companion.toHttpUrl
import okhttp3.Interceptor
import okhttp3.MediaType.Companion.toMediaTypeOrNull
import okhttp3.Response
import okhttp3.ResponseBody.Companion.toResponseBody
import org.json.JSONArray
import org.json.JSONObject
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import org.koitharu.kotatsu.parsers.util.json.getStringOrNull
import org.koitharu.kotatsu.parsers.util.json.mapJSON
import org.koitharu.kotatsu.parsers.util.json.mapJSONNotNull
import org.koitharu.kotatsu.parsers.util.json.toJSONList
import java.util.*
internal abstract class MangaPlusParser(
context: MangaLoaderContext,
source: MangaSource,
private val sourceLang: String,
) : MangaParser(context, source), Interceptor {
private val apiUrl = "https://jumpg-webapi.tokyo-cdn.com/api"
override val configKeyDomain = ConfigKey.Domain("mangaplus.shueisha.co.jp")
override val availableSortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.POPULARITY,
SortOrder.UPDATED,
SortOrder.ALPHABETICAL,
)
private val extraHeaders = Headers.headersOf("Session-Token", UUID.randomUUID().toString())
// no tags or tag search available
override suspend fun getAvailableTags(): Set<MangaTag> = emptySet()
override suspend fun getList(offset: Int, filter: MangaListFilter?): List<Manga> {
if (offset > 0) {
return emptyList()
}
return when (filter) {
is MangaListFilter.Advanced -> {
when (filter.sortOrder) {
SortOrder.POPULARITY -> getPopularList()
SortOrder.UPDATED -> getLatestList()
else -> getAllTitleList()
}
}
is MangaListFilter.Search -> getAllTitleList(filter.query)
else -> getAllTitleList()
}
}
private suspend fun getPopularList(): List<Manga> {
val json = apiCall("/title_list/ranking")
return json.getJSONObject("titleRankingView")
.getJSONArray("titles")
.toJSONList()
.toMangaList()
}
private suspend fun getLatestList(): List<Manga> {
val json = apiCall("/title_list/updated")
return json.getJSONObject("titleUpdatedView")
.getJSONArray("latestTitle")
.mapJSON { it.getJSONObject("title") }
.toMangaList()
}
// since search is local, save network calls on related manga call
private val allTitleCache = SuspendLazy {
apiCall("/title_list/allV2")
.getJSONObject("allTitlesViewV2")
.getJSONArray("AllTitlesGroup")
.mapJSON { it.getJSONArray("titles").toJSONList() }
.flatten()
}
private suspend fun getAllTitleList(query: String? = null): List<Manga> {
return allTitleCache.get().toMangaList(query)
}
private fun List<JSONObject>.toMangaList(query: String? = null): List<Manga> {
return mapNotNull {
val language = it.getStringOrNull("language") ?: "ENGLISH"
if (language != sourceLang) {
return@mapNotNull null
}
val name = it.getString("name")
val author = it.getString("author")
.split('/')
.joinToString(transform = String::trim)
// filter out any other title or author which doesn't match search input
if (query != null && !(name.contains(query, true) || author.contains(query, true))) {
return@mapNotNull null
}
val titleId = it.getInt("titleId").toString()
Manga(
id = generateUid(titleId),
url = titleId,
publicUrl = "/titles/$titleId".toAbsoluteUrl(domain),
title = name,
coverUrl = it.getString("portraitImageUrl"),
altTitle = null,
author = author,
isNsfw = false,
rating = RATING_UNKNOWN,
state = null,
source = source,
tags = emptySet(),
)
}
}
override suspend fun getDetails(manga: Manga): Manga {
val json = apiCall("/title_detailV3?title_id=${manga.url}")
.getJSONObject("titleDetailView")
val title = json.getJSONObject("title")
val completed = json.getJSONObject("titleLabels")
.getString("releaseSchedule").let {
it == "DISABLED" || it == "COMPLETED"
}
val hiatus = json.getStringOrNull("nonAppearanceInfo")?.contains("on a hiatus") == true
return manga.copy(
title = title.getString("name"),
publicUrl = "/titles/${title.getInt("titleId")}".toAbsoluteUrl(domain),
coverUrl = title.getString("portraitImageUrl"),
author = title.getString("author")
.split("/").joinToString(transform = String::trim),
description = buildString {
json.getString("overview").let(::append)
json.getStringOrNull("viewingPeriodDescription")
?.takeIf { !completed }
?.let { append("<br><br>", it) }
},
chapters = parseChapters(
json.getJSONArray("chapterListGroup"),
title.getStringOrNull("language") ?: "ENGLISH",
),
state = when {
completed -> MangaState.FINISHED
hiatus -> MangaState.PAUSED
else -> MangaState.ONGOING
},
)
}
private fun parseChapters(chapterListGroup: JSONArray, language: String): List<MangaChapter> {
val chapterList = chapterListGroup
.toJSONList()
.flatMap {
it.optJSONArray("firstChapterList")?.toJSONList().orEmpty() +
it.optJSONArray("lastChapterList")?.toJSONList().orEmpty()
}
return chapterList.mapChapters { _, chapter ->
val chapterId = chapter.getInt("chapterId").toString()
val subtitle = chapter.getStringOrNull("subTitle") ?: return@mapChapters null
MangaChapter(
id = generateUid(chapterId),
url = chapterId,
name = subtitle,
number = chapter.getString("name")
.substringAfter("#")
.toIntOrNull() ?: -1,
uploadDate = chapter.getInt("startTimeStamp") * 1000L,
branch = when (language) {
"PORTUGUESE_BR" -> "Portuguese (Brazil)"
else -> language.lowercase().toTitleCase()
},
scanlator = null,
source = source,
)
}
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val pages = apiCall("/manga_viewer?chapter_id=${chapter.url}&split=yes&img_quality=super_high")
.getJSONObject("mangaViewer")
.getJSONArray("pages")
return pages.mapJSONNotNull {
val mangaPage = it.optJSONObject("mangaPage")
?: return@mapJSONNotNull null
val url = mangaPage.getString("imageUrl")
val encryptionKey = mangaPage.getStringOrNull("encryptionKey")
MangaPage(
id = generateUid(url),
url = url + if (encryptionKey == null) "" else "#$encryptionKey",
preview = null,
source = source,
)
}
}
// image descrambling
override fun intercept(chain: Interceptor.Chain): Response {
val request = chain.request()
val response = chain.proceed(request)
val encryptionKey = request.url.fragment
if (encryptionKey.isNullOrEmpty()) {
return response
}
val contentType = response.headers["Content-Type"] ?: "image/jpeg"
val image = requireNotNull(response.body).bytes().decodeXorCipher(encryptionKey)
val body = image.toResponseBody(contentType.toMediaTypeOrNull())
return response.newBuilder()
.body(body)
.build()
}
private fun ByteArray.decodeXorCipher(key: String): ByteArray {
val keyStream = key.chunked(2)
.map { it.toInt(16) }
return mapIndexed { i, byte -> byte.toInt() xor keyStream[i % keyStream.size] }
.map(Int::toByte)
.toByteArray()
}
private suspend fun apiCall(url: String): JSONObject {
val newUrl = "$apiUrl$url".toHttpUrl().newBuilder()
.addQueryParameter("format", "json")
.build()
val response = webClient.httpGet(newUrl, extraHeaders).parseJson()
val success = response.optJSONObject("success")
return checkNotNull(success) {
val error = response.getJSONObject("error")
val reason = error.getJSONArray("popups")
.toJSONList()
.firstOrNull { it.getStringOrNull("language") == null }
if (reason?.getStringOrNull("subject") == "Not Found" && url.contains("manga_viewer")) {
"This chapter has expired"
} else {
reason?.getStringOrNull("body") ?: "Unknown Error"
}
}
}
@MangaSourceParser("MANGAPLUSPARSER_EN", "MANGA Plus English", "en")
class English(context: MangaLoaderContext) : MangaPlusParser(
context,
MangaSource.MANGAPLUSPARSER_EN,
"ENGLISH",
)
@MangaSourceParser("MANGAPLUSPARSER_ES", "MANGA Plus Spanish", "es")
class Spanish(context: MangaLoaderContext) : MangaPlusParser(
context,
MangaSource.MANGAPLUSPARSER_ES,
"SPANISH",
)
@MangaSourceParser("MANGAPLUSPARSER_FR", "MANGA Plus French", "fr")
class French(context: MangaLoaderContext) : MangaPlusParser(
context,
MangaSource.MANGAPLUSPARSER_FR,
"FRENCH",
)
@MangaSourceParser("MANGAPLUSPARSER_ID", "MANGA Plus Indonesian", "id")
class Indonesian(context: MangaLoaderContext) : MangaPlusParser(
context,
MangaSource.MANGAPLUSPARSER_ID,
"INDONESIAN",
)
@MangaSourceParser("MANGAPLUSPARSER_PTBR", "MANGA Plus Portuguese (Brazil)", "pt")
class Portuguese(context: MangaLoaderContext) : MangaPlusParser(
context,
MangaSource.MANGAPLUSPARSER_PTBR,
"PORTUGUESE_BR",
)
@MangaSourceParser("MANGAPLUSPARSER_RU", "MANGA Plus Russian", "ru")
class Russian(context: MangaLoaderContext) : MangaPlusParser(
context,
MangaSource.MANGAPLUSPARSER_RU,
"RUSSIAN",
)
@MangaSourceParser("MANGAPLUSPARSER_TH", "MANGA Plus Thai", "th")
class Thai(context: MangaLoaderContext) : MangaPlusParser(
context,
MangaSource.MANGAPLUSPARSER_TH,
"THAI",
)
@MangaSourceParser("MANGAPLUSPARSER_VI", "MANGA Plus Vietnamese", "vi")
class Vietnamese(context: MangaLoaderContext) : MangaPlusParser(
context,
MangaSource.MANGAPLUSPARSER_VI,
"VIETNAMESE",
)
}

387
src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/NineNineNineHentaiParser.kt
View File

@@ -1,387 +0,0 @@
package org.koitharu.kotatsu.parsers.site.all
import okhttp3.HttpUrl.Companion.toHttpUrlOrNull
import okhttp3.Interceptor
import okhttp3.Response
import org.json.JSONArray
import org.json.JSONObject
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import org.koitharu.kotatsu.parsers.util.json.getStringOrNull
import org.koitharu.kotatsu.parsers.util.json.mapJSON
import org.koitharu.kotatsu.parsers.util.json.mapJSONToSet
import org.koitharu.kotatsu.parsers.util.json.toJSONList
import java.text.SimpleDateFormat
import java.util.*
@MangaSourceParser("NINENINENINEHENTAI", "999Hentai", type = ContentType.HENTAI)
internal class NineNineNineHentaiParser(context: MangaLoaderContext) :
PagedMangaParser(context, MangaSource.NINENINENINEHENTAI, size), Interceptor {
override val configKeyDomain = ConfigKey.Domain("999hentai.net")
override val availableSortOrders: EnumSet<SortOrder> = EnumSet.of(
SortOrder.POPULARITY,
SortOrder.NEWEST,
)
override val isMultipleTagsSupported = false
override suspend fun getAvailableLocales() = setOf(
Locale.ENGLISH,
Locale.CHINESE,
Locale.JAPANESE,
Locale("es"),
)
private fun Locale?.getSiteLang(): String {
if (this == null) return "all"
return when {
equals(Locale.ENGLISH) -> "en"
equals(Locale.CHINESE) -> "cn"
equals(Locale.JAPANESE) -> "jp"
equals(Locale("es")) -> "es"
else -> "all"
}
}
// Need for disable encoding (with encoding not working)
override fun intercept(chain: Interceptor.Chain): Response {
val request = chain.request()
val newRequest = if (request.header("Content-Encoding") != null) {
request.newBuilder().removeHeader("Content-Encoding").build()
} else {
request
}
return chain.proceed(newRequest)
}
private val cdnHost = SuspendLazy(::getUpdatedCdnHost)
private suspend fun getUpdatedCdnHost(): String {
val url = "https://$domain/manga-home"
val response = webClient.httpGet(url).parseHtml()
val cdn = response.selectFirst("img.v-thumbnail")?.attr("data-src")
return cdn?.toHttpUrlOrNull()?.host ?: "edge.fast4speed.rsvp"
}
override suspend fun getAvailableTags(): Set<MangaTag> {
val query = """
queryTags(
search: {format:"tagchapter",sortBy:Popular}
page: 1
limit: 100
) {
edges {
name
}
}
""".trimIndent()
val tags = apiCall(query)
.getJSONObject("queryTags")
.getJSONArray("edges")
return tags.mapJSONToSet {
val name = it.getString("name")
MangaTag(
title = name.toCamelCase(),
key = name,
source = source,
)
}
}
override suspend fun getListPage(page: Int, filter: MangaListFilter?): List<Manga> {
return when (filter) {
is MangaListFilter.Advanced -> {
if (filter.tags.isEmpty() && filter.sortOrder == SortOrder.POPULARITY) {
getPopularList(page, filter.locale)
} else {
getSearchList(page, null, filter.locale, filter.tags, filter.sortOrder)
}
}
is MangaListFilter.Search -> {
getSearchList(page, filter.query, null, null, filter.sortOrder)
}
else -> {
getPopularList(page, null)
}
}
}
private suspend fun getPopularList(
page: Int,
locale: Locale?,
): List<Manga> {
val query = """
queryPopularChapters(
size: $size
language: "${locale.getSiteLang()}"
dateRange: 1
page: $page
) {
edges {
_id
name
firstPics
}
}
""".trimIndent()
return apiCall(query)
.getJSONObject("queryPopularChapters")
.getJSONArray("edges")
.toMangaList()
}
private suspend fun getSearchList(
page: Int,
search: String?,
locale: Locale?,
tags: Set<MangaTag>?,
sort: SortOrder?,
): List<Manga> {
val searchPayload = buildString {
if (!search.isNullOrEmpty()) {
append("query:\"$search\",")
}
append("language:\"${locale.getSiteLang()}\"")
if (sort == SortOrder.POPULARITY) {
append(",sortBy:Popular")
}
if (!tags.isNullOrEmpty()) {
val tag = tags.oneOrThrowIfMany()!!.key
append(",tags:[\"$tag\"]")
}
}
val query = """
queryChapters(
limit: $size
search: {$searchPayload}
page: $page
) {
edges {
_id
name
firstPics
}
}
""".trimIndent()
return apiCall(query)
.getJSONObject("queryChapters")
.getJSONArray("edges")
.toMangaList()
}
private suspend fun JSONArray.toMangaList(): List<Manga> = mapJSON { entry ->
val id = entry.getString("_id")
val name = entry.getString("name")
val cover = runCatching {
entry.getJSONArray("firstPics")
.getJSONObject(0)
.getString("url")
}.getOrNull()
Manga(
id = generateUid(id),
title = name.replace(shortenTitleRegex, "").trim(),
altTitle = name,
coverUrl = when {
cover?.startsWith("http") == true -> cover
cover == null -> ""
else -> "https://${cdnHost.get()}/$cover"
},
author = null,
isNsfw = true,
url = id,
publicUrl = "/hchapter/$id".toAbsoluteUrl(domain),
tags = emptySet(),
source = source,
state = null,
rating = RATING_UNKNOWN,
)
}
override suspend fun getDetails(manga: Manga): Manga {
val query = """
queryChapter(
chapterId: "${manga.url}"
) {
_id
name
uploadDate
format
description
language
pages
tags
pictureUrls {
picCdn
pics
picsS
}
}
""".trimIndent()
val entry = apiCall(query)
.getJSONObject("queryChapter")
val id = entry.getString("_id")
val name = entry.getString("name")
val cover = entry.getJSONArray("pictureUrls")
.getJSONObject(0)
.let { pics ->
val cdn = pics.getString("picCdn").let {
if (it.startsWith("http")) {
"$it/"
} else {
"https://${cdnHost.get()}/$it/"
}
}
val img = pics.getJSONArray("pics").getJSONObject(0).getString("url")
val imgS = pics.getJSONArray("picsS").getJSONObject(0).getString("url")
Pair(cdn + imgS, cdn + img)
}
val tags = entry.optJSONArray("tags")?.mapJSON {
SiteTag(
name = it.getString("tagName"),
type = it.getStringOrNull("tagType"),
)
}
return manga.copy(
title = name.replace(shortenTitleRegex, "").trim(),
altTitle = name,
coverUrl = cover.first,
largeCoverUrl = cover.second,
author = tags?.filter { it.type == "artist" }?.joinToString { it.name.toCamelCase() },
isNsfw = true,
tags = tags?.mapToSet {
MangaTag(
title = it.name.toCamelCase(),
key = it.name,
source = source,
)
}.orEmpty(),
state = null,
description = entry.getStringOrNull("description"),
chapters = listOf(
MangaChapter(
id = generateUid(id),
name = name,
number = 1,
url = id,
uploadDate = runCatching {
dateFormat.parse(entry.getString("uploadDate"))!!.time
}.getOrDefault(0L),
branch = entry.getStringOrNull("language")?.let {
val locale = when (it) {
"en" -> Locale.ENGLISH
"jp" -> Locale.JAPANESE
"cn" -> Locale.CHINESE
"es" -> Locale("es")
else -> Locale.ROOT
}
return@let locale.getDisplayLanguage(locale)
},
scanlator = when (entry.getStringOrNull("format")) {
"artistcg" -> "ArtistCG"
"gamecg" -> "GameCG"
"imageset" -> "ImageSet"
else -> entry.getStringOrNull("format")?.toCamelCase()
},
source = source,
),
),
)
}
private data class SiteTag(
val name: String,
val type: String?,
)
override suspend fun getRelatedManga(seed: Manga): List<Manga> {
val query = """
queryRecommendations(
type: "chapter"
_id: "${seed.url}"
search: {sortBy:Popular}
page: 1
size: $size
) {
chapters {
_id
name
firstPics
}
}
""".trimIndent()
return apiCall(query)
.getJSONObject("queryRecommendations")
.getJSONArray("chapters")
.toMangaList()
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val query = """
queryChapter(
chapterId: "${chapter.url}"
) {
pictureUrls {
picCdn
pics
picsS
}
}
""".trimIndent()
val pages = apiCall(query)
.getJSONObject("queryChapter")
.getJSONArray("pictureUrls")
.getJSONObject(0)
val cdn = pages.getString("picCdn").let {
if (it.startsWith("http")) {
"$it/"
} else {
"https://${cdnHost.get()}/$it/"
}
}
val pics = pages.getJSONArray("pics").toJSONList()
val picsS = pages.getJSONArray("picsS").toJSONList()
return pics.zip(picsS).map {
val img = it.first.getString("url")
val imgS = it.second.getString("url")
MangaPage(
id = generateUid(img),
url = cdn + img,
preview = cdn + imgS,
source = source,
)
}
}
private suspend fun apiCall(query: String): JSONObject {
return webClient.graphQLQuery("https://api.$domain/api", query).getJSONObject("data")
}
companion object {
private const val size = 20
private val shortenTitleRegex = Regex("""(\[[^]]*]|[({][^)}]*[)}])""")
private val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.ENGLISH)
}
}

198
src/main/kotlin/org/koitharu/kotatsu/parsers/site/animebootstrap/AnimeBootstrapParser.kt
View File

@@ -1,198 +0,0 @@
package org.koitharu.kotatsu.parsers.site.animebootstrap
import kotlinx.coroutines.async
import kotlinx.coroutines.coroutineScope
import org.json.JSONArray
import org.jsoup.nodes.Document
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import java.util.*
internal abstract class AnimeBootstrapParser(
context: MangaLoaderContext,
source: MangaSource,
domain: String,
pageSize: Int = 24,
) : PagedMangaParser(context, source, pageSize) {
override val configKeyDomain = ConfigKey.Domain(domain)
override val isMultipleTagsSupported = false
override val availableSortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.UPDATED,
SortOrder.POPULARITY,
SortOrder.ALPHABETICAL,
SortOrder.NEWEST,
)
protected open val listUrl = "/manga"
protected open val datePattern = "dd MMM. yyyy"
init {
paginator.firstPage = 1
searchPaginator.firstPage = 1
}
override suspend fun getListPage(page: Int, filter: MangaListFilter?): List<Manga> {
val url = buildString {
append("https://")
append(domain)
append(listUrl)
append("?page=")
append(page.toString())
append("&type=all")
when (filter) {
is MangaListFilter.Search -> {
append("&search=")
append(filter.query.urlEncoded())
}
is MangaListFilter.Advanced -> {
filter.tags.oneOrThrowIfMany()?.let {
append("&categorie=")
append(it.key)
}
append("&sort=")
when (filter.sortOrder) {
SortOrder.POPULARITY -> append("view")
SortOrder.UPDATED -> append("updated")
SortOrder.ALPHABETICAL -> append("default")
SortOrder.NEWEST -> append("published")
else -> append("updated")
}
}
null -> append("&sort=updated")
}
}
val doc = webClient.httpGet(url).parseHtml()
return doc.select("div.col-6 div.product__item").map { div ->
val href = div.selectFirstOrThrow("a").attrAsRelativeUrl("href")
Manga(
id = generateUid(href),
url = href,
publicUrl = href.toAbsoluteUrl(div.host ?: domain),
coverUrl = div.selectFirstOrThrow("div.product__item__pic").attr("data-setbg").orEmpty(),
title = div.selectFirstOrThrow("div.product__item__text").text().orEmpty(),
altTitle = null,
rating = RATING_UNKNOWN,
tags = emptySet(),
author = null,
state = null,
source = source,
isNsfw = isNsfwSource,
)
}
}
override suspend fun getAvailableTags(): Set<MangaTag> {
val doc = webClient.httpGet("https://$domain$listUrl").parseHtml()
return doc.select("div.product__page__filter div:contains(Genre:) option ").mapNotNullToSet { option ->
val key = option.attr("value") ?: return@mapNotNullToSet null
val name = option.text().toTitleCase(sourceLocale)
MangaTag(
key = key,
title = name,
source = source,
)
}
}
protected open val selectDesc = "div.anime__details__text p"
protected open val selectState = "div.anime__details__widget li:contains(Ongoing)"
protected open val selectTag = "div.anime__details__widget li:contains(Categorie) a"
override suspend fun getDetails(manga: Manga): Manga = coroutineScope {
val fullUrl = manga.url.toAbsoluteUrl(domain)
val doc = webClient.httpGet(fullUrl).parseHtml()
val chaptersDeferred = async { getChapters(doc) }
val desc = doc.selectFirstOrThrow(selectDesc).html()
val state = if (doc.select(selectState).isNullOrEmpty()) {
MangaState.FINISHED
} else {
MangaState.ONGOING
}
manga.copy(
tags = doc.body().select(selectTag).mapNotNullToSet { a ->
MangaTag(
key = a.attr("href").substringAfterLast('='),
title = a.text().toTitleCase().replace(",", ""),
source = source,
)
},
description = desc,
state = state,
chapters = chaptersDeferred.await(),
)
}
protected open val selectChapter = "div.anime__details__episodes a"
protected open suspend fun getChapters(doc: Document): List<MangaChapter> {
return doc.body().select(selectChapter).mapChapters(reversed = true) { i, a ->
val href = a.attr("href")
MangaChapter(
id = generateUid(href),
name = a.text(),
number = i + 1,
url = href,
uploadDate = 0,
source = source,
scanlator = null,
branch = null,
)
}
}
protected open val selectPage = "div.read-img img"
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(domain)
val doc = webClient.httpGet(fullUrl).parseHtml()
if (doc.select("script:containsData(page_image)").isNullOrEmpty()) {
return doc.select(selectPage).map { img ->
val url = img.attr("onerror").replace("this.onerror=null;this.src=`", "").replace("`;", "")
MangaPage(
id = generateUid(url),
url = url,
preview = null,
source = source,
)
}
} else {
val script = doc.selectFirstOrThrow("script:containsData(page_image)")
val images = JSONArray(script.data().substringAfterLast("var pages = ").substringBefore(';'))
val pages = ArrayList<MangaPage>(images.length())
for (i in 0 until images.length()) {
val pageTake = images.getJSONObject(i)
pages.add(
MangaPage(
id = generateUid(pageTake.getString("page_image")),
url = pageTake.getString("page_image"),
preview = null,
source = source,
),
)
}
return pages
}
}
}

138
src/main/kotlin/org/koitharu/kotatsu/parsers/site/animebootstrap/fr/PapScan.kt
View File

@@ -1,138 +0,0 @@
package org.koitharu.kotatsu.parsers.site.animebootstrap.fr
import kotlinx.coroutines.async
import kotlinx.coroutines.coroutineScope
import org.jsoup.nodes.Document
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.site.animebootstrap.AnimeBootstrapParser
import org.koitharu.kotatsu.parsers.util.*
import java.text.SimpleDateFormat
import java.util.EnumSet
import java.util.Locale
@MangaSourceParser("PAPSCAN", "PapScan", "fr")
internal class PapScan(context: MangaLoaderContext) :
AnimeBootstrapParser(context, MangaSource.PAPSCAN, "papscan.com") {
override val sourceLocale: Locale = Locale.ENGLISH
override val isMultipleTagsSupported = false
override val listUrl = "/liste-manga"
override val selectState = "div.anime__details__widget li:contains(En cours)"
override val selectTag = "div.anime__details__widget li:contains(Genre) a"
override val selectChapter = "ul.chapters li"
override val availableSortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.POPULARITY,
SortOrder.ALPHABETICAL,
SortOrder.ALPHABETICAL_DESC,
)
override suspend fun getListPage(page: Int, filter: MangaListFilter?): List<Manga> {
val url = buildString {
append("https://")
append(domain)
append("/filterList")
append("?page=")
append(page.toString())
when (filter) {
is MangaListFilter.Search -> {
append("&alpha=")
append(filter.query.urlEncoded())
}
is MangaListFilter.Advanced -> {
filter.tags.oneOrThrowIfMany()?.let {
append("&cat=")
append(it.key)
}
append("&sortBy=")
when (filter.sortOrder) {
SortOrder.POPULARITY -> append("views")
SortOrder.ALPHABETICAL_DESC -> append("name&asc=false")
SortOrder.ALPHABETICAL -> append("name&asc=true")
else -> append("updated")
}
}
null -> append("&sortBy=updated")
}
}
val doc = webClient.httpGet(url).parseHtml()
return doc.select("div.product__item").map { div ->
val href = div.selectFirstOrThrow("h5 a").attrAsRelativeUrl("href")
Manga(
id = generateUid(href),
url = href,
publicUrl = href.toAbsoluteUrl(div.host ?: domain),
coverUrl = div.selectFirstOrThrow("div.product__item__pic").attr("data-setbg").orEmpty(),
title = div.selectFirstOrThrow("div.product__item__text h5").text().orEmpty(),
altTitle = null,
rating = RATING_UNKNOWN,
tags = emptySet(),
author = null,
state = null,
source = source,
isNsfw = isNsfwSource,
)
}
}
override suspend fun getAvailableTags(): Set<MangaTag> {
val doc = webClient.httpGet("https://$domain$listUrl").parseHtml()
return doc.select("a.category ").mapNotNullToSet { a ->
val key = a.attr("href").substringAfterLast('=')
val name = a.text()
MangaTag(
key = key,
title = name,
source = source,
)
}
}
override suspend fun getDetails(manga: Manga): Manga = coroutineScope {
val fullUrl = manga.url.toAbsoluteUrl(domain)
val doc = webClient.httpGet(fullUrl).parseHtml()
val chaptersDeferred = async { getChapters(doc) }
val desc = doc.selectFirstOrThrow(selectDesc).html()
val state = if (doc.select(selectState).isNullOrEmpty()) {
MangaState.FINISHED
} else {
MangaState.ONGOING
}
manga.copy(
tags = doc.body().select(selectTag).mapNotNullToSet { a ->
MangaTag(
key = a.attr("href").removeSuffix('/').substringAfterLast('/'),
title = a.text().toTitleCase(),
source = source,
)
},
description = desc,
state = state,
chapters = chaptersDeferred.await(),
)
}
override suspend fun getChapters(doc: Document): List<MangaChapter> {
val dateFormat = SimpleDateFormat(datePattern, sourceLocale)
return doc.body().select(selectChapter).mapChapters(reversed = true) { i, li ->
val href = li.selectFirstOrThrow("a").attr("href")
val dateText = li.selectFirst("span.date-chapter-title-rtl")?.text()
MangaChapter(
id = generateUid(href),
name = li.selectFirstOrThrow("span em").text(),
number = i + 1,
url = href,
uploadDate = dateFormat.tryParse(dateText),
source = source,
scanlator = null,
branch = null,
)
}
}
}

10
src/main/kotlin/org/koitharu/kotatsu/parsers/site/animebootstrap/id/KomikzoId.kt
View File

@@ -1,10 +0,0 @@
package org.koitharu.kotatsu.parsers.site.animebootstrap.id
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.site.animebootstrap.AnimeBootstrapParser
@MangaSourceParser("KOMIKZOID", "KomikzoId", "id")
internal class KomikzoId(context: MangaLoaderContext) :
AnimeBootstrapParser(context, MangaSource.KOMIKZOID, "komikzoid.id")

10
src/main/kotlin/org/koitharu/kotatsu/parsers/site/animebootstrap/id/NeuManga.kt
View File

@@ -1,10 +0,0 @@
package org.koitharu.kotatsu.parsers.site.animebootstrap.id
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.site.animebootstrap.AnimeBootstrapParser
@MangaSourceParser("NEUMANGA", "NeuManga.xyz", "id")
internal class NeuManga(context: MangaLoaderContext) :
AnimeBootstrapParser(context, MangaSource.NEUMANGA, "neumanga.xyz")

10
src/main/kotlin/org/koitharu/kotatsu/parsers/site/animebootstrap/id/SekteKomik.kt
View File

@@ -1,10 +0,0 @@
package org.koitharu.kotatsu.parsers.site.animebootstrap.id
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.site.animebootstrap.AnimeBootstrapParser
@MangaSourceParser("SEKTEKOMIK", "SekteKomik", "id")
internal class SekteKomik(context: MangaLoaderContext) :
AnimeBootstrapParser(context, MangaSource.SEKTEKOMIK, "sektekomik.xyz")

214
src/main/kotlin/org/koitharu/kotatsu/parsers/site/ar/FlixScans.kt
View File

@@ -1,214 +0,0 @@
package org.koitharu.kotatsu.parsers.site.ar
import kotlinx.coroutines.async
import kotlinx.coroutines.coroutineScope
import org.json.JSONArray
import org.json.JSONObject
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import org.koitharu.kotatsu.parsers.util.json.mapJSON
import org.koitharu.kotatsu.parsers.util.json.mapJSONIndexed
import java.text.SimpleDateFormat
import java.util.*
@MangaSourceParser("FLIXSCANS", "FlixScans.com", "ar")
internal class FlixScans(context: MangaLoaderContext) : PagedMangaParser(context, MangaSource.FLIXSCANS, 18) {
override val availableSortOrders: Set<SortOrder> = EnumSet.of(SortOrder.UPDATED)
override val availableStates: Set<MangaState> = EnumSet.allOf(MangaState::class.java)
override val availableContentRating: Set<ContentRating> = EnumSet.of(ContentRating.ADULT)
override val configKeyDomain = ConfigKey.Domain("flixscans.com")
override suspend fun getListPage(page: Int, filter: MangaListFilter?): List<Manga> {
val json = when (filter) {
is MangaListFilter.Search -> {
if (page > 1) {
return emptyList()
}
val url = "https://api.$domain/api/v1/search/serie"
val body = JSONObject()
body.put("title", filter.query.urlEncoded())
webClient.httpPost(url, body).parseJson().getJSONArray("data")
}
is MangaListFilter.Advanced -> {
val url = buildString {
append("https://api.")
append(domain)
append("/api/v1/")
if (filter.tags.isNotEmpty() || filter.states.isNotEmpty() || filter.contentRating.isNotEmpty()) {
if (page > 1) {
return emptyList()
}
append("search/advance?=")
if (filter.tags.isNotEmpty()) {
val tagQuery = filter.tags.joinToString(separator = ",") { it.key }
append("&genres=")
append(tagQuery)
}
if (filter.states.isNotEmpty()) {
filter.states.oneOrThrowIfMany()?.let {
append("&status=")
append(
when (it) {
MangaState.ONGOING -> "ongoing"
MangaState.FINISHED -> "completed"
MangaState.ABANDONED -> "droped"
MangaState.PAUSED -> "onhold"
MangaState.UPCOMING -> "soon"
},
)
}
}
filter.contentRating.oneOrThrowIfMany()?.let {
append("&adult=")
append(
when (it) {
ContentRating.ADULT -> "true"
else -> ""
},
)
}
append("&serie_type=webtoon")
} else {
append("webtoon/pages/latest/romance?page=")
append(page.toString())
}
}
webClient.httpGet(url).parseJson().getJSONArray("data")
}
null -> {
val url = "https://api.$domain/api/v1/webtoon/pages/latest/romance?page=$page"
webClient.httpGet(url).parseJson().getJSONArray("data")
}
}
return json.mapJSON { j ->
val href = "https://$domain/series/${j.getString("prefix")}-${j.getString("id")}-${j.getString("slug")}"
val cover = "https://api.$domain/storage/" + j.getString("thumbnail")
Manga(
id = generateUid(href),
title = j.getString("title"),
altTitle = null,
url = href,
publicUrl = href.toAbsoluteUrl(domain),
rating = RATING_UNKNOWN,
isNsfw = false,
coverUrl = cover,
tags = emptySet(),
state = when (j.getString("status")) {
"ongoing" -> MangaState.ONGOING
"completed" -> MangaState.FINISHED
"onhold" -> MangaState.PAUSED
"droped" -> MangaState.ABANDONED
else -> null
},
author = null,
source = source,
)
}
}
override suspend fun getAvailableTags(): Set<MangaTag> {
val doc = webClient.httpGet("https://$domain/search/advance").parseHtml()
val json = JSONArray(doc.requireElementById("__NUXT_DATA__").data())
val tagsList = json.getJSONArray(3).toString().replace("[", "").replace("]", "").split(",")
return tagsList.mapNotNullToSet { idTag ->
val id = idTag.toInt()
val idKey = json.getJSONObject(id).getInt("id")
val key = json.getInt(idKey).toString()
val idName = json.getJSONObject(id).getInt("name")
val name = json.getString(idName)
MangaTag(
key = key,
title = name,
source = source,
)
}
}
override suspend fun getDetails(manga: Manga): Manga = coroutineScope {
val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml()
val chaptersDeferred = async { loadChapters(manga.url) }
val json = JSONArray(doc.requireElementById("__NUXT_DATA__").data())
val descId = json.getJSONObject(6).getInt("story")
val desc = json.getString(descId)
val tagsId = json.getJSONObject(6).getInt("genres")
val tagsList = json.getJSONArray(tagsId).toString().replace("[", "").replace("]", "").split(",")
val ratingId = json.getJSONObject(6).getInt("rating")
val rating = json.getString(ratingId)
val nsfwId = json.getJSONObject(6).getInt("nsfw")
val nsfw = json.getBoolean(nsfwId)
manga.copy(
description = desc,
tags = tagsList.mapToSet { idTag ->
val id = idTag.toInt()
val idKey = json.getJSONObject(id).getInt("id")
val key = json.get(idKey).toString()
val idName = json.getJSONObject(id).getInt("name")
val name = json.get(idName).toString()
MangaTag(
key = key,
title = name,
source = source,
)
},
rating = rating?.toFloatOrNull()?.div(5f) ?: RATING_UNKNOWN,
isNsfw = nsfw,
chapters = chaptersDeferred.await(),
)
}
private val dateFormat = SimpleDateFormat("yyyy-MM-dd", sourceLocale)
private suspend fun loadChapters(baseUrl: String): List<MangaChapter> {
val key = baseUrl.substringAfter("-").substringBefore("-")
val seriesKey = baseUrl.substringAfterLast("/").substringBefore("-")
val json = JSONArray(webClient.httpGet("https://api.$domain/api/v1/webtoon/chapters/$key-desc").parseRaw())
return json.mapJSONIndexed { i, j ->
val url = "https://$domain/read/webtoon/$seriesKey-${j.getString("id")}-${j.getString("slug")}"
val date = j.getString("createdAt").substringBeforeLast("T")
MangaChapter(
id = generateUid(url),
url = url,
name = j.getString("slug").replace('-', ' '),
number = i + 1,
branch = null,
uploadDate = dateFormat.tryParse(date),
scanlator = null,
source = source,
)
}
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(domain)
val doc = webClient.httpGet(fullUrl).parseHtml()
val json = JSONArray(doc.requireElementById("__NUXT_DATA__").data())
val chapterData = json.getJSONObject(6).getInt("chapterData")
val pageLocate = json.getJSONObject(chapterData).getInt("webtoon")
val jsonPages = json.getJSONArray(pageLocate)
val pages = ArrayList<MangaPage>(jsonPages.length())
for (i in 0 until jsonPages.length()) {
val id = jsonPages.getInt(i)
val url = "https://api.$domain/storage/" + json.getString(id)
pages.add(
MangaPage(
id = generateUid(url),
url = url,
preview = null,
source = source,
),
)
}
return pages
}
}

128
src/main/kotlin/org/koitharu/kotatsu/parsers/site/ar/MangaStorm.kt
View File

@@ -1,128 +0,0 @@
package org.koitharu.kotatsu.parsers.site.ar
import okhttp3.Headers
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.network.UserAgents
import org.koitharu.kotatsu.parsers.util.*
import java.util.*
@MangaSourceParser("MANGASTORM", "MangaStorm", "ar")
internal class MangaStorm(context: MangaLoaderContext) : PagedMangaParser(context, MangaSource.MANGASTORM, 30) {
override val availableSortOrders: Set<SortOrder> = EnumSet.of(SortOrder.POPULARITY, SortOrder.UPDATED)
override val configKeyDomain = ConfigKey.Domain("mangastorm.org")
override val isMultipleTagsSupported = false
override val headers: Headers = Headers.Builder()
.add("User-Agent", UserAgents.CHROME_DESKTOP)
.build()
override suspend fun getListPage(page: Int, filter: MangaListFilter?): List<Manga> {
val url = buildString {
append("https://")
append(domain)
when (filter) {
is MangaListFilter.Search -> {
append("/mangas?page=")
append(page)
append("&query=")
append(filter.query.urlEncoded())
}
is MangaListFilter.Advanced -> {
if (filter.tags.isNotEmpty()) {
val tag = filter.tags.oneOrThrowIfMany()
append("/categories/")
append(tag?.key.orEmpty())
append("?page=")
append(page)
} else {
if (filter.sortOrder == SortOrder.POPULARITY) {
append("/mangas?page=")
append(page)
} else {
if (page > 1) {
return emptyList()
}
}
}
}
null -> {
append("/mangas?page=")
append(page)
}
}
}
val doc = webClient.httpGet(url).parseHtml()
return doc.select("div.row div.col").map { div ->
val href = div.selectFirstOrThrow("a").attrAsRelativeUrl("href")
Manga(
id = generateUid(href),
title = div.select(".manga-ct-title").text(),
altTitle = null,
url = href,
publicUrl = href.toAbsoluteUrl(domain),
rating = RATING_UNKNOWN,
isNsfw = false,
coverUrl = div.selectFirstOrThrow("img").src().orEmpty(),
tags = emptySet(),
state = null,
author = null,
source = source,
)
}
}
override suspend fun getAvailableTags(): Set<MangaTag> = emptySet()
override suspend fun getDetails(manga: Manga): Manga {
val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml()
val root = doc.selectFirstOrThrow(".card-body .col-lg-9")
return manga.copy(
altTitle = null,
state = null,
tags = root.select(".flex-wrap a").mapNotNullToSet { a ->
MangaTag(
key = a.attr("href").substringAfterLast('/'),
title = a.text(),
source = source,
)
},
author = null,
description = root.selectFirstOrThrow(".card-text").text(),
chapters = doc.select(".card-body a.btn-fixed-width").mapChapters(reversed = true) { i, a ->
val url = a.attrAsRelativeUrl("href")
MangaChapter(
id = generateUid(url),
name = a.text(),
number = i + 1,
url = url,
scanlator = null,
uploadDate = 0,
branch = null,
source = source,
)
},
)
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(domain)
val doc = webClient.httpGet(fullUrl).parseHtml().requireElementById("content")
return doc.select("div.text-center .img-fluid").map { img ->
val url = img.src()?.toRelativeUrl(domain) ?: img.parseFailed("Image src not found")
MangaPage(
id = generateUid(url),
url = url,
preview = null,
source = source,
)
}
}
}

209
src/main/kotlin/org/koitharu/kotatsu/parsers/site/ar/TeamXNovel.kt
View File

@@ -1,209 +0,0 @@
package org.koitharu.kotatsu.parsers.site.ar
import kotlinx.coroutines.async
import kotlinx.coroutines.awaitAll
import kotlinx.coroutines.coroutineScope
import org.jsoup.nodes.Element
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import java.text.SimpleDateFormat
import java.util.*
@MangaSourceParser("TEAMXNOVEL", "TeamXNovel", "ar")
internal class TeamXNovel(context: MangaLoaderContext) : PagedMangaParser(context, MangaSource.TEAMXNOVEL, 10) {
override val availableSortOrders: Set<SortOrder> = EnumSet.of(SortOrder.UPDATED, SortOrder.POPULARITY)
override val availableStates: Set<MangaState> =
EnumSet.of(MangaState.ONGOING, MangaState.FINISHED, MangaState.ABANDONED)
override val configKeyDomain = ConfigKey.Domain("teamxnovel.com")
override val isMultipleTagsSupported = false
override suspend fun getListPage(page: Int, filter: MangaListFilter?): List<Manga> {
val url = buildString {
append("https://")
append(domain)
when (filter) {
is MangaListFilter.Search -> {
append("/series?search=")
append(filter.query.urlEncoded())
if (page > 1) {
append("&page=")
append(page.toString())
}
}
is MangaListFilter.Advanced -> {
if (filter.tags.isNotEmpty()) {
val tag = filter.tags.oneOrThrowIfMany()
append("/series?genre=")
append(tag?.key.orEmpty())
if (page > 1) {
append("&page=")
append(page.toString())
}
append("&")
} else {
when (filter.sortOrder) {
SortOrder.POPULARITY -> append("/series")
SortOrder.UPDATED -> append("/")
else -> append("/")
}
if (page > 1) {
append("?page=")
append(page.toString())
append("&")
} else {
append("?")
}
}
if (filter.sortOrder == SortOrder.POPULARITY || filter.tags.isNotEmpty()) {
filter.states.oneOrThrowIfMany()?.let {
append("status=")
append(
when (it) {
MangaState.ONGOING -> "مستمرة"
MangaState.FINISHED -> "مكتمل"
MangaState.ABANDONED -> "متوقف"
else -> "مستمرة"
},
)
}
}
}
null -> append("/?page=$page")
}
}
val doc = webClient.httpGet(url).parseHtml()
return doc.select("div.listupd .bs .bsx").ifEmpty {
doc.select("div.post-body .box")
}.map { div ->
val href = div.selectFirstOrThrow("a").attrAsRelativeUrl("href")
Manga(
id = generateUid(href),
title = div.select(".tt, h3").text(),
altTitle = null,
url = href,
publicUrl = href.toAbsoluteUrl(domain),
rating = RATING_UNKNOWN,
isNsfw = false,
coverUrl = div.selectFirstOrThrow("img").src().orEmpty(),
tags = emptySet(),
state = when (div.selectFirst(".status")?.text()) {
"مستمرة" -> MangaState.ONGOING
"مكتمل" -> MangaState.FINISHED
"متوقف" -> MangaState.ABANDONED
else -> null
},
author = null,
source = source,
)
}
}
override suspend fun getAvailableTags(): Set<MangaTag> {
val doc = webClient.httpGet("https://$domain/series").parseHtml()
return doc.requireElementById("select_genre").select("option").mapNotNullToSet {
MangaTag(
key = it.attr("value"),
title = it.text().toTitleCase(sourceLocale),
source = source,
)
}
}
override suspend fun getDetails(manga: Manga): Manga {
val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml()
val mangaUrl = manga.url.toAbsoluteUrl(domain)
val maxPageChapterSelect = doc.select(".pagination .page-item a")
var maxPageChapter = 1
if (!maxPageChapterSelect.isNullOrEmpty()) {
maxPageChapterSelect.map {
val i = it.attr("href").substringAfterLast("=").toInt()
if (i > maxPageChapter) {
maxPageChapter = i
}
}
}
return manga.copy(
altTitle = null,
state = when (doc.selectFirstOrThrow(".full-list-info:contains(الحالة:) a").text()) {
"مستمرة" -> MangaState.ONGOING
"مكتمل" -> MangaState.FINISHED
"متوقف" -> MangaState.ABANDONED
else -> null
},
tags = doc.select(".review-author-info a").mapNotNullToSet { a ->
MangaTag(
key = a.attr("href").substringAfterLast("="),
title = a.text(),
source = source,
)
},
author = null,
description = doc.selectFirstOrThrow(".review-content").text(),
chapters = run {
if (maxPageChapter == 1) {
parseChapters(doc)
} else {
coroutineScope {
val result = ArrayList(parseChapters(doc))
result.ensureCapacity(result.size * maxPageChapter)
(2..maxPageChapter).map { i ->
async {
loadChapters(mangaUrl, i)
}
}.awaitAll()
.flattenTo(result)
result
}
}
}.reversed(),
)
}
private suspend fun loadChapters(baseUrl: String, page: Int): List<MangaChapter> {
return parseChapters(webClient.httpGet("$baseUrl?page=$page").parseHtml().body())
}
private val dateFormat = SimpleDateFormat("yyyy-MM-dd HH:mm:ss", sourceLocale)
private fun parseChapters(root: Element): List<MangaChapter> {
return root.requireElementById("chapter-contact").select(".eplister ul li")
.map { li ->
val url = li.selectFirstOrThrow("a").attrAsRelativeUrl("href")
MangaChapter(
id = generateUid(url),
name = li.selectFirstOrThrow(".epl-title").text(),
number = url.substringAfterLast('/').toIntOrNull() ?: 0,
url = url,
scanlator = null,
uploadDate = dateFormat.tryParse(li.selectFirstOrThrow(".epl-date").text()),
branch = null,
source = source,
)
}
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(domain)
val doc = webClient.httpGet(fullUrl).parseHtml()
return doc.select(".image_list img").map { img ->
val url = img.src()?.toRelativeUrl(domain) ?: img.parseFailed("Image src not found")
MangaPage(
id = generateUid(url),
url = url,
preview = null,
source = source,
)
}
}
}

136
src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/BeeToon.kt
View File

@@ -1,136 +0,0 @@
package org.koitharu.kotatsu.parsers.site.en
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import java.text.SimpleDateFormat
import java.util.*
@MangaSourceParser("BEETOON", "BeeToon.net", "en")
internal class BeeToon(context: MangaLoaderContext) :
PagedMangaParser(context, MangaSource.BEETOON, pageSize = 30) {
override val availableSortOrders: Set<SortOrder> = EnumSet.of(SortOrder.UPDATED, SortOrder.POPULARITY)
override val configKeyDomain = ConfigKey.Domain("manhwafull.net")
override val isMultipleTagsSupported = false
override suspend fun getListPage(page: Int, filter: MangaListFilter?): List<Manga> {
val url = buildString {
append("https://")
append(domain)
when (filter) {
is MangaListFilter.Search -> {
if (page > 1) {
return emptyList()
}
append("/?s=")
append(filter.query.urlEncoded())
}
is MangaListFilter.Advanced -> {
if (filter.tags.isNotEmpty()) {
val tag = filter.tags.oneOrThrowIfMany()
append("/genre/")
append(tag?.key.orEmpty())
append("/page-")
append(page)
append("/")
} else {
when (filter.sortOrder) {
SortOrder.UPDATED -> append("/latest-update/")
SortOrder.POPULARITY -> append("/popular-manga/")
else -> append("/latest-update/")
}
append("page-")
append(page)
append("/")
}
}
null -> append("/latest-update/page-$page/")
}
}
val doc = webClient.httpGet(url).parseHtml()
return doc.select(".comics-grid .entry").map { div ->
val href = div.selectFirstOrThrow("a").attrAsRelativeUrl("href")
Manga(
id = generateUid(href),
url = href,
publicUrl = href.toAbsoluteUrl(domain),
coverUrl = div.selectFirst("img")?.src().orEmpty(),
title = div.selectFirst(".name")?.text().orEmpty(),
altTitle = null,
rating = div.selectFirst(".counter")?.text()?.toFloatOrNull()?.div(5f) ?: RATING_UNKNOWN,
tags = emptySet(),
author = null,
state = when (div.selectLastOrThrow(".status span").text()) {
"Ongoing" -> MangaState.ONGOING
"Completed" -> MangaState.FINISHED
else -> null
},
source = source,
isNsfw = isNsfwSource,
)
}
}
override suspend fun getAvailableTags(): Set<MangaTag> {
val doc = webClient.httpGet("https://$domain/").parseHtml()
return doc.requireElementById("menu-item-3").select("ul.sub-menu li a").mapNotNullToSet {
MangaTag(
key = it.attr("href").removeSuffix('/').substringAfterLast('/'),
title = it.text(),
source = source,
)
}
}
override suspend fun getDetails(manga: Manga): Manga {
val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml()
return manga.copy(
description = doc.getElementById("desc")?.text().orEmpty(),
rating = doc.selectFirst(".counter")?.text()?.toFloatOrNull()?.div(5f) ?: RATING_UNKNOWN,
tags = doc.body().select(".info .genre a").mapNotNullToSet {
MangaTag(
key = it.attr("href").removeSuffix('/').substringAfterLast('/'),
title = it.text(),
source = source,
)
},
author = doc.selectFirst(".info .author a")?.text(),
chapters = doc.select(".items-chapters a").mapChapters(reversed = true) { i, a ->
val url = a.attrAsRelativeUrl("href").toAbsoluteUrl(domain)
MangaChapter(
id = generateUid(url),
name = a.selectFirstOrThrow(".chap").text(),
number = i + 1,
url = url,
scanlator = null,
uploadDate = SimpleDateFormat("MM/dd/yyyy HH:mm:ss", Locale.ENGLISH)
.tryParse(a.selectFirst(".chapter-date")?.attr("title") ?: "0"),
branch = null,
source = source,
)
},
)
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml()
return doc.select(".chapter-content-inner center img").map { img ->
val urlPage = img.src()?.toRelativeUrl(domain) ?: img.parseFailed("Image src not found")
MangaPage(
id = generateUid(urlPage),
url = urlPage,
preview = null,
source = source,
)
}
}
}

176
src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/ComicExtra.kt
View File

@@ -1,176 +0,0 @@
package org.koitharu.kotatsu.parsers.site.en
import okhttp3.Headers
import org.koitharu.kotatsu.parsers.ErrorMessages
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.network.UserAgents
import org.koitharu.kotatsu.parsers.util.*
import java.text.SimpleDateFormat
import java.util.*
@MangaSourceParser("COMICEXTRA", "ComicExtra", "en")
internal class ComicExtra(context: MangaLoaderContext) : PagedMangaParser(context, MangaSource.COMICEXTRA, 25) {
override val availableSortOrders: Set<SortOrder> =
EnumSet.of(SortOrder.POPULARITY, SortOrder.UPDATED, SortOrder.NEWEST)
override val availableStates: Set<MangaState> = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED)
override val configKeyDomain = ConfigKey.Domain("comicextra.me")
override val isMultipleTagsSupported = false
override val headers: Headers = Headers.Builder()
.add("User-Agent", UserAgents.CHROME_DESKTOP)
.build()
override suspend fun getListPage(page: Int, filter: MangaListFilter?): List<Manga> {
val url = buildString {
append("https://")
append(domain)
append("/")
when (filter) {
is MangaListFilter.Search -> {
append("comic-search?key=")
append(filter.query.urlEncoded())
if (page > 1) {
append("&page=")
append(page.toString())
}
}
is MangaListFilter.Advanced -> {
if (filter.tags.isNotEmpty() && filter.states.isEmpty()) {
filter.tags.oneOrThrowIfMany()?.let {
append(it.key)
}
} else if (filter.tags.isEmpty() && filter.states.isNotEmpty()) {
filter.states.oneOrThrowIfMany()?.let {
append(
when (it) {
MangaState.ONGOING -> "/ongoing-comic"
MangaState.FINISHED -> "/completed-comic"
else -> "/ongoing-comic"
},
)
}
} else if (filter.tags.isNotEmpty() && filter.states.isNotEmpty()) {
throw IllegalArgumentException(ErrorMessages.FILTER_BOTH_STATES_GENRES_NOT_SUPPORTED)
} else {
when (filter.sortOrder) {
SortOrder.POPULARITY -> append("popular-comic")
SortOrder.UPDATED -> append("new-comic")
SortOrder.NEWEST -> append("recent-comic")
else -> append("new-comic")
}
}
if (page > 1) {
append("/")
append(page.toString())
}
}
null -> {
append("popular-comic")
if (page > 1) {
append("/")
append(page.toString())
}
}
}
}
val doc = webClient.httpGet(url).parseHtml()
return doc.select("div.movie-list-index div.cartoon-box").map { div ->
val href = div.selectFirstOrThrow("a").attrAsRelativeUrl("href")
Manga(
id = generateUid(href),
title = div.selectFirstOrThrow("h3").text(),
altTitle = null,
url = href,
publicUrl = href.toAbsoluteUrl(domain),
rating = RATING_UNKNOWN,
isNsfw = false,
coverUrl = div.selectFirstOrThrow("img").attrAsAbsoluteUrl("src"),
tags = emptySet(),
state = when (div.selectFirstOrThrow(".detail:contains(Stasus: )").text()) {
"Stasus: Ongoing" -> MangaState.ONGOING
"Stasus: Completed" -> MangaState.FINISHED
else -> null
},
author = null,
source = source,
)
}
}
override suspend fun getAvailableTags(): Set<MangaTag> {
val doc = webClient.httpGet("https://$domain/popular-comic").parseHtml()
return doc.select("li.tag-item a").mapNotNullToSet { a ->
MangaTag(
key = a.attr("href").substringAfterLast('/'),
title = a.text(),
source = source,
)
}
}
override suspend fun getDetails(manga: Manga): Manga {
val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml()
val dateFormat = SimpleDateFormat("MM/dd/yy", sourceLocale)
return manga.copy(
altTitle = doc.selectFirstOrThrow("dt.movie-dt:contains(Alternate name:) + dd").text(),
state = when (doc.selectFirstOrThrow("dt.movie-dt:contains(Status:) + dd a").text()) {
"Ongoing" -> MangaState.ONGOING
"Completed" -> MangaState.FINISHED
else -> null
},
tags = doc.select("dt.movie-dt:contains(Genres:) + dd a").mapNotNullToSet { a ->
MangaTag(
key = a.attr("href").substringAfterLast("/"),
title = a.text(),
source = source,
)
},
author = doc.select("dt.movie-dt:contains(Author:) + dd").text(),
description = doc.getElementById("film-content")?.text(),
chapters = doc.requireElementById("list").select("tr")
.mapChapters(reversed = true) { i, tr ->
val a = tr.selectFirstOrThrow("a")
val url = a.attrAsRelativeUrl("href") + "/full"
val name = a.text()
val dateText = tr.select("td").last()?.text()
MangaChapter(
id = generateUid(url),
name = name,
number = i + 1,
url = url,
scanlator = null,
uploadDate = dateFormat.tryParse(dateText),
branch = null,
source = source,
)
},
)
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(domain)
val doc = webClient.httpGet(fullUrl).parseHtml()
return doc.select(".chapter-container img").map { img ->
val url = img.src()?.toRelativeUrl(domain) ?: img.parseFailed("Image src not found")
MangaPage(
id = generateUid(url),
url = url,
preview = null,
source = source,
)
}
}
}

213
src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/DynastyScans.kt
View File

@@ -1,213 +0,0 @@
package org.koitharu.kotatsu.parsers.site.en
import androidx.collection.ArraySet
import kotlinx.coroutines.async
import kotlinx.coroutines.awaitAll
import kotlinx.coroutines.coroutineScope
import okhttp3.Headers
import org.json.JSONArray
import org.jsoup.nodes.Document
import org.jsoup.nodes.Element
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.network.UserAgents
import org.koitharu.kotatsu.parsers.util.*
import java.text.SimpleDateFormat
import java.util.*
@MangaSourceParser("DYNASTYSCANS", "DynastyScans", "en")
internal class DynastyScans(context: MangaLoaderContext) : PagedMangaParser(context, MangaSource.DYNASTYSCANS, 117) {
override val availableSortOrders: Set<SortOrder> = EnumSet.of(SortOrder.ALPHABETICAL)
override val configKeyDomain = ConfigKey.Domain("dynasty-scans.com")
override val headers: Headers = Headers.Builder()
.add("User-Agent", UserAgents.CHROME_DESKTOP)
.build()
override val isMultipleTagsSupported = false
override suspend fun getListPage(page: Int, filter: MangaListFilter?): List<Manga> {
when (filter) {
is MangaListFilter.Search -> {
val url = buildString {
append("https://")
append(domain)
append("/search?q=")
append(filter.query.urlEncoded())
append("&")
append("classes[]".urlEncoded())
append("=Serie&page=")
append(page.toString())
}
return parseMangaListQuery(webClient.httpGet(url).parseHtml())
}
is MangaListFilter.Advanced -> {
val url = buildString {
append("https://")
append(domain)
if (filter.tags.isNotEmpty()) {
append("/tags/")
filter.tags.oneOrThrowIfMany()?.let {
append(it.key)
}
append("?view=groupings")
} else {
append("/series?view=cover")
}
append("&page=")
append(page.toString())
}
return parseMangaList(webClient.httpGet(url).parseHtml())
}
null -> {
val url = buildString {
append("https://")
append(domain)
append("/series?view=cover&page=")
append(page.toString())
}
return parseMangaList(webClient.httpGet(url).parseHtml())
}
}
}
private fun parseMangaList(doc: Document): List<Manga> {
return doc.select("li.span2")
.map { div ->
val href = div.selectFirstOrThrow("a").attrAsRelativeUrl("href")
Manga(
id = generateUid(href),
title = div.selectFirstOrThrow("div.caption").text(),
altTitle = null,
url = href,
publicUrl = href.toAbsoluteUrl(domain),
rating = RATING_UNKNOWN,
isNsfw = false,
coverUrl = div.selectFirstOrThrow("img").attrAsAbsoluteUrl("src"),
tags = setOf(),
state = null,
author = null,
source = source,
)
}
}
private fun parseMangaListQuery(doc: Document): List<Manga> {
return doc.select("dl.chapter-list dd")
.map { div ->
val href = div.selectFirstOrThrow("a").attrAsRelativeUrl("href")
Manga(
id = generateUid(href),
title = div.selectFirstOrThrow("a").text(),
altTitle = null,
url = href,
publicUrl = href.toAbsoluteUrl(domain),
rating = RATING_UNKNOWN,
isNsfw = false,
coverUrl = "",
tags = div.select("span.tags a").mapNotNullToSet { a ->
MangaTag(
key = a.attr("href").removeSuffix('/').substringAfterLast('/'),
title = a.text(),
source = source,
)
},
state = null,
author = null,
source = source,
)
}
}
override suspend fun getAvailableTags(): Set<MangaTag> {
return coroutineScope {
(1..3).map { page ->
async { getTags(page) }
}
}.awaitAll().flattenTo(ArraySet(360))
}
private suspend fun getTags(page: Int): Set<MangaTag> {
val url = "https://$domain/tags?page=$page"
val root = webClient.httpGet(url).parseHtml()
return root.selectFirstOrThrow(".tag-list ").parseTags()
}
private fun Element.parseTags() = select("a").mapToSet {
MangaTag(
key = it.attr("href").removeSuffix('/').substringAfterLast('/'),
title = it.text(),
source = source,
)
}
override suspend fun getDetails(manga: Manga): Manga {
val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml()
val chapters = getChapters(doc)
val root = doc.requireElementById("main")
return manga.copy(
altTitle = null,
state = when (root.select("h2.tag-title small").last()?.text()) {
"— Ongoing" -> MangaState.ONGOING
"— Completed", "— Completed and Licensed" -> MangaState.FINISHED
"— Dropped", "— Licensed and Removed", "— Abandoned" -> MangaState.ABANDONED
"— On Hiatus" -> MangaState.PAUSED
else -> null
},
coverUrl = root.selectFirst("img.thumbnail")?.src()
.orEmpty(), // It is needed if the manga was found via the search.
tags = root.selectFirstOrThrow("div.tag-tags").parseTags(),
author = null,
description = null,
chapters = chapters,
)
}
private fun getChapters(doc: Document): List<MangaChapter> {
val dateFormat = SimpleDateFormat("MMM dd yy", sourceLocale)
return doc.body().select("dl.chapter-list dd").mapChapters { i, li ->
val a = li.selectFirstOrThrow("a")
val href = a.attrAsRelativeUrl("href")
val dateText = li.select("small").last()?.text()?.replace("released ", "")?.replace("'", "")
MangaChapter(
id = generateUid(href),
name = a.text(),
number = i + 1,
url = href,
uploadDate = dateFormat.tryParse(dateText),
source = source,
scanlator = null,
branch = null,
)
}
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val chapterUrl = chapter.url.toAbsoluteUrl(domain)
val docs = webClient.httpGet(chapterUrl).parseHtml()
val script = docs.selectFirstOrThrow("script:containsData(var pages =)")
val json = JSONArray(script.data().substringAfter('=').substringBeforeLast(';'))
val pages = ArrayList<MangaPage>(json.length())
for (i in 0 until json.length()) {
val url = "https://" + domain + json.getString(i).substringAfter(":\"").substringBefore("\",")
pages.add(
MangaPage(
id = generateUid(url),
url = url,
preview = null,
source = source,
),
)
}
return pages
}
}

199
src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/FlixScansOrg.kt
View File

@@ -1,199 +0,0 @@
package org.koitharu.kotatsu.parsers.site.en
import kotlinx.coroutines.async
import kotlinx.coroutines.coroutineScope
import org.json.JSONArray
import org.koitharu.kotatsu.parsers.ErrorMessages
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import org.koitharu.kotatsu.parsers.util.json.mapJSON
import org.koitharu.kotatsu.parsers.util.json.toJSONList
import java.text.SimpleDateFormat
import java.util.*
@MangaSourceParser("FLIXSCANSORG", "FlixScans.org", "en")
internal class FlixScansOrg(context: MangaLoaderContext) : PagedMangaParser(context, MangaSource.FLIXSCANSORG, 18) {
override val availableSortOrders: Set<SortOrder> = EnumSet.of(SortOrder.UPDATED)
override val availableStates: Set<MangaState> = EnumSet.allOf(MangaState::class.java)
override val availableContentRating: Set<ContentRating> = EnumSet.of(ContentRating.ADULT)
override val configKeyDomain = ConfigKey.Domain("flixscans.org")
override val isSearchSupported = false
override suspend fun getListPage(page: Int, filter: MangaListFilter?): List<Manga> {
val json = when (filter) {
is MangaListFilter.Search -> {
throw IllegalArgumentException(ErrorMessages.SEARCH_NOT_SUPPORTED)
}
is MangaListFilter.Advanced -> {
val url = buildString {
append("https://api.")
append(domain)
append("/api/v1/search/advance?=&serie_type=webtoon&page=")
append(page.toString())
append("&genres=")
append(filter.tags.joinToString(separator = ",") { it.key })
filter.states.oneOrThrowIfMany()?.let {
append("&status=")
append(
when (it) {
MangaState.ONGOING -> "ongoing"
MangaState.FINISHED -> "completed"
MangaState.ABANDONED -> "droped"
MangaState.PAUSED -> "onhold"
MangaState.UPCOMING -> "soon"
},
)
}
filter.contentRating.oneOrThrowIfMany()?.let {
append("&adult=")
append(
when (it) {
ContentRating.ADULT -> "true"
else -> ""
},
)
}
append("&serie_type=webtoon")
}
webClient.httpGet(url).parseJson().getJSONArray("data")
}
null -> {
val url = "https://api.$domain/api/v1/search/advance?=&serie_type=webtoon&page=$page"
webClient.httpGet(url).parseJson().getJSONArray("data")
}
}
return json.mapJSON { j ->
val href = "https://$domain/series/${j.getString("prefix")}-${j.getString("id")}-${j.getString("slug")}"
val cover = "https://media.$domain/" + j.getString("thumbnail")
Manga(
id = generateUid(href),
title = j.getString("title"),
altTitle = null,
url = href,
publicUrl = href.toAbsoluteUrl(domain),
rating = RATING_UNKNOWN,
isNsfw = isNsfwSource,
coverUrl = cover,
tags = emptySet(),
state = when (j.getString("status")) {
"ongoing" -> MangaState.ONGOING
"completed" -> MangaState.FINISHED
"onhold" -> MangaState.PAUSED
"droped" -> MangaState.ABANDONED
else -> null
},
author = null,
source = source,
)
}
}
override suspend fun getAvailableTags(): Set<MangaTag> {
val doc = webClient.httpGet("https://$domain/search/advance").parseHtml()
val json = JSONArray(doc.requireElementById("__NUXT_DATA__").data())
val tagsList = json.getJSONArray(3).toString().replace("[", "").replace("]", "").split(",")
return tagsList.mapNotNullToSet { idTag ->
val id = idTag.toInt()
val idKey = json.getJSONObject(id).getInt("id")
val key = json.getInt(idKey).toString()
val idName = json.getJSONObject(id).getInt("name")
val name = json.getString(idName)
MangaTag(
key = key,
title = name,
source = source,
)
}
}
override suspend fun getDetails(manga: Manga): Manga = coroutineScope {
val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml()
val chaptersDeferred = async { loadChapters(manga.url) }
val json = JSONArray(doc.requireElementById("__NUXT_DATA__").data())
val descId = json.getJSONObject(6).getInt("story")
val desc = json.getString(descId)
val tagsId = json.getJSONObject(6).getInt("genres")
val tagsList = json.getJSONArray(tagsId).toString().replace("[", "").replace("]", "").split(",")
val ratingId = json.getJSONObject(6).getInt("rating")
val rating = json.getString(ratingId)
val nsfwId = json.getJSONObject(6).getInt("nsfw")
val nsfw = json.getBoolean(nsfwId)
manga.copy(
description = desc,
tags = tagsList.mapToSet { idTag ->
val id = idTag.toInt()
val idKey = json.getJSONObject(id).getInt("id")
val key = json.get(idKey).toString()
val idName = json.getJSONObject(id).getInt("name")
val name = json.get(idName).toString()
MangaTag(
key = key,
title = name,
source = source,
)
},
rating = rating?.toFloatOrNull()?.div(5f) ?: RATING_UNKNOWN,
isNsfw = nsfw,
chapters = chaptersDeferred.await(),
)
}
private val dateFormat = SimpleDateFormat("yyyy-MM-dd", sourceLocale)
private suspend fun loadChapters(baseUrl: String): List<MangaChapter> {
val key = baseUrl.substringAfter("-").substringBefore("-")
val seriesKey = baseUrl.substringAfterLast("/").substringBefore("-")
val json = JSONArray(
webClient.httpGet("https://api.$domain/api/v1/webtoon/chapters/$key-desc").parseRaw(),
).toJSONList().reversed()
return json.mapChapters { i, j ->
val url = "https://$domain/read/webtoon/$seriesKey-${j.getString("id")}-${j.getString("slug")}"
val date = j.getString("createdAt").substringBeforeLast("T")
MangaChapter(
id = generateUid(url),
url = url,
name = j.getString("slug").replace('-', ' '),
number = i + 1,
branch = null,
uploadDate = dateFormat.tryParse(date),
scanlator = null,
source = source,
)
}
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(domain)
val doc = webClient.httpGet(fullUrl).parseHtml()
val json = JSONArray(doc.requireElementById("__NUXT_DATA__").data())
val chapterData = json.getJSONObject(6).getInt("chapterData")
val pageLocate = json.getJSONObject(chapterData).getInt("webtoon")
val jsonPages = json.getJSONArray(pageLocate)
val pages = ArrayList<MangaPage>(jsonPages.length())
for (i in 0 until jsonPages.length()) {
val id = jsonPages.getInt(i)
val url = "https://media.$domain/" + json.getString(id)
pages.add(
MangaPage(
id = generateUid(url),
url = url,
preview = null,
source = source,
),
)
}
return pages
}
}

164
src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/MangaGeko.kt
View File

@@ -1,164 +0,0 @@
package org.koitharu.kotatsu.parsers.site.en
import kotlinx.coroutines.async
import kotlinx.coroutines.coroutineScope
import okhttp3.Headers
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.network.UserAgents
import org.koitharu.kotatsu.parsers.util.*
import java.text.SimpleDateFormat
import java.util.*
@MangaSourceParser("MANGAGEKO", "MangaGeko", "en")
internal class MangaGeko(context: MangaLoaderContext) : PagedMangaParser(context, MangaSource.MANGAGEKO, 30) {
override val availableSortOrders: Set<SortOrder> =
EnumSet.of(SortOrder.POPULARITY, SortOrder.UPDATED, SortOrder.NEWEST)
override val configKeyDomain = ConfigKey.Domain("www.mgeko.com", "www.mangageko.com")
override val isMultipleTagsSupported = false
override val headers: Headers = Headers.Builder()
.add("User-Agent", UserAgents.CHROME_DESKTOP)
.build()
override suspend fun getListPage(page: Int, filter: MangaListFilter?): List<Manga> {
val url = buildString {
append("https://")
append(domain)
when (filter) {
is MangaListFilter.Search -> {
if (page > 1) {
return emptyList()
}
append("/search/?search=")
append(filter.query.urlEncoded())
}
is MangaListFilter.Advanced -> {
append("/browse-comics/?results=")
append(page)
append("&filter=")
when (filter.sortOrder) {
SortOrder.POPULARITY -> append("views")
SortOrder.UPDATED -> append("Updated")
SortOrder.NEWEST -> append("New")
else -> append("Updated")
}
if (filter.tags.isNotEmpty()) {
filter.tags.oneOrThrowIfMany()?.let {
append("&genre=")
append(it.key)
}
}
}
null -> {
append("/browse-comics/?results=")
append(page)
append("&filter=Updated")
}
}
}
val doc = webClient.httpGet(url).parseHtml()
return doc.select("li.novel-item").map { div ->
val href = div.selectFirstOrThrow("a").attrAsRelativeUrl("href")
Manga(
id = generateUid(href),
title = div.selectFirstOrThrow("h4").text(),
altTitle = null,
url = href,
publicUrl = href.toAbsoluteUrl(domain),
rating = RATING_UNKNOWN,
isNsfw = false,
coverUrl = div.selectFirstOrThrow("img").src().orEmpty(),
tags = emptySet(),
state = null,
author = div.selectFirstOrThrow("h6").text().removePrefix("Author(S): "),
source = source,
)
}
}
override suspend fun getAvailableTags(): Set<MangaTag> {
val doc = webClient.httpGet("https://$domain/browse-comics/").parseHtml()
return doc.select("label.checkbox-inline").mapNotNullToSet { label ->
MangaTag(
key = label.selectFirstOrThrow("input").attr("value"),
title = label.text(),
source = source,
)
}
}
override suspend fun getDetails(manga: Manga): Manga = coroutineScope {
val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml()
val chaptersDeferred = async { loadChapters(manga.url) }
manga.copy(
altTitle = doc.selectFirstOrThrow(".alternative-title").text(),
state = when (doc.selectFirstOrThrow(".header-stats span:contains(Status) strong").text()) {
"Ongoing" -> MangaState.ONGOING
"Completed" -> MangaState.FINISHED
else -> null
},
tags = doc.select(".categories ul li a").mapNotNullToSet { a ->
MangaTag(
key = a.attr("href").substringAfterLast("="),
title = a.text(),
source = source,
)
},
author = doc.selectFirstOrThrow(".author").text(),
description = doc.selectFirstOrThrow(".description").html(),
chapters = chaptersDeferred.await(),
)
}
private suspend fun loadChapters(mangaUrl: String): List<MangaChapter> {
val urlChapter = mangaUrl + "all-chapters/"
val doc = webClient.httpGet(urlChapter.toAbsoluteUrl(domain)).parseHtml()
val dateFormat = SimpleDateFormat("MMM dd, yyyy", sourceLocale)
return doc.requireElementById("chapters").select("ul.chapter-list li")
.mapChapters(reversed = true) { i, li ->
val a = li.selectFirstOrThrow("a")
val url = a.attrAsRelativeUrl("href")
val name = li.selectFirstOrThrow(".chapter-title").text()
val dateText = li.select(".chapter-update").attr("datetime").substringBeforeLast(',')
.replace(".", "").replace("Sept", "Sep")
MangaChapter(
id = generateUid(url),
name = name,
number = i + 1,
url = url,
scanlator = null,
uploadDate = dateFormat.tryParse(dateText),
branch = null,
source = source,
)
}
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(domain)
val doc = webClient.httpGet(fullUrl).parseHtml()
return doc.requireElementById("chapter-reader").select("img").map { img ->
val url = img.src()?.toRelativeUrl(domain) ?: img.parseFailed("Image src not found")
MangaPage(
id = generateUid(url),
url = url,
preview = null,
source = source,
)
}
}
}

183
src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/Mangaowl.kt
View File

@@ -1,183 +0,0 @@
package org.koitharu.kotatsu.parsers.site.en
import kotlinx.coroutines.coroutineScope
import okhttp3.Headers
import org.jsoup.nodes.Document
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.network.UserAgents
import org.koitharu.kotatsu.parsers.util.*
import org.koitharu.kotatsu.parsers.util.json.mapJSON
import java.text.SimpleDateFormat
import java.util.*
@MangaSourceParser("MANGAOWL", "MangaOwl.to", "en")
internal class Mangaowl(context: MangaLoaderContext) :
PagedMangaParser(context, MangaSource.MANGAOWL, pageSize = 24) {
override val availableSortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.POPULARITY,
SortOrder.NEWEST,
SortOrder.UPDATED,
SortOrder.RATING,
)
override val availableStates: Set<MangaState> = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED)
override val configKeyDomain = ConfigKey.Domain("mangaowl.to")
override val headers: Headers = Headers.Builder()
.add("User-Agent", UserAgents.CHROME_DESKTOP)
.build()
override suspend fun getListPage(page: Int, filter: MangaListFilter?): List<Manga> {
val url = buildString {
append("https://")
append(domain)
when (filter) {
is MangaListFilter.Search -> {
append("/10-search?q=")
append(filter.query.urlEncoded())
append("&page=")
append(page.toString())
}
is MangaListFilter.Advanced -> {
append("/10-comics")
append("?page=")
append(page.toString())
filter.tags.forEach { tag ->
append("&genres=")
append(tag.key)
}
filter.states.oneOrThrowIfMany()?.let {
append("&status=")
append(
when (it) {
MangaState.ONGOING -> "ongoing"
MangaState.FINISHED -> "completed"
else -> ""
},
)
}
append("&ordering=")
append(
when (filter.sortOrder) {
SortOrder.POPULARITY -> "view_count"
SortOrder.UPDATED -> "-modified_at"
SortOrder.NEWEST -> "created_at"
SortOrder.RATING -> "rating"
else -> "modified_at"
},
)
}
null -> {
append("/10-comics?ordering=-modified_at&page=")
append(page.toString())
}
}
}
val doc = webClient.httpGet(url).parseHtml()
return doc.select("div.manga-item.column").map { div ->
val href = div.selectFirstOrThrow("a").attrAsRelativeUrl("href")
Manga(
id = generateUid(href),
url = href,
publicUrl = href.toAbsoluteUrl(div.host ?: domain),
coverUrl = div.selectFirst("img")?.src().orEmpty(),
title = div.selectFirst("a.one-line")?.text().orEmpty(),
altTitle = null,
rating = div.select("span").last()?.text()?.toFloatOrNull()?.div(5f) ?: RATING_UNKNOWN,
tags = emptySet(),
author = null,
state = null,
source = source,
isNsfw = false,
)
}
}
override suspend fun getAvailableTags(): Set<MangaTag> {
val doc = webClient.httpGet("https://$domain/10-genres").parseHtml()
return doc.select("div.genres-container span.genre-item a").mapNotNullToSet { a ->
val key = a.attr("href").removeSuffix('/').substringAfterLast('/').substringBefore("-")
MangaTag(
key = key,
title = a.text(),
source = source,
)
}
}
override suspend fun getDetails(manga: Manga): Manga = coroutineScope {
val fullUrl = manga.url.toAbsoluteUrl(domain)
val doc = webClient.httpGet(fullUrl).parseHtml()
manga.copy(
tags = doc.body().select("div.comic-attrs div.column.my-2:contains(Genres) a").mapNotNullToSet { a ->
MangaTag(
key = a.attr("href").removeSuffix("/").substringAfterLast('/').substringBefore("-"),
title = a.text().toTitleCase().replace(",", ""),
source = source,
)
},
description = doc.select("span.story-desc").html(),
state = when (doc.select("div.section-status:contains(Status) span").last()?.text()) {
"Ongoing" -> MangaState.ONGOING
"Completed" -> MangaState.FINISHED
else -> null
},
chapters = getChapters(manga.url, doc),
)
}
private fun getChapters(mangaUrl: String, doc: Document): List<MangaChapter> {
val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'", sourceLocale)
val script = doc.selectFirstOrThrow("script:containsData(chapters:)")
val json = script.data().substringAfter("chapters:[")
.substringBeforeLast(')')
.substringBefore("],latest_chapter:")
.split("},")
val slug = mangaUrl.substringAfterLast("/")
return json.mapChapters { i, t ->
if (t.contains("Chapter")) {
val id = t.substringAfter("id:").substringBefore(",created_at")
val url = "/reading/$slug/$id"
val date = t.substringAfter("created_at:\"").substringBefore('"')
val name = t.substringAfter("name:\"").substringBefore('"')
MangaChapter(
id = generateUid(url),
name = name,
number = i + 1,
url = url,
uploadDate = dateFormat.tryParse(date),
source = source,
scanlator = null,
branch = null,
)
} else {
null
}
}
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val id = chapter.url.substringAfterLast("/")
val json = webClient.httpGet("https://api.mangaowl.to/v1/chapters/$id/images?page_size=100").parseJson()
return json.getJSONArray("results").mapJSON { jo ->
MangaPage(
id = generateUid(jo.getString("image")),
preview = null,
source = chapter.source,
url = jo.getString("image"),
)
}
}
}

239
src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/Manhwa18Parser.kt
View File

@@ -1,239 +0,0 @@
package org.koitharu.kotatsu.parsers.site.en
import androidx.collection.ArrayMap
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import java.util.*
@MangaSourceParser("MANHWA18", "Manhwa18.net", "en", type = ContentType.HENTAI)
class Manhwa18Parser(context: MangaLoaderContext) :
PagedMangaParser(context, MangaSource.MANHWA18, pageSize = 18, searchPageSize = 18) {
override val configKeyDomain: ConfigKey.Domain = ConfigKey.Domain("manhwa18.net")
override val availableSortOrders: Set<SortOrder>
get() = EnumSet.of(
SortOrder.UPDATED,
SortOrder.POPULARITY,
SortOrder.ALPHABETICAL,
SortOrder.NEWEST,
SortOrder.RATING,
)
override val availableStates: Set<MangaState> = EnumSet.of(
MangaState.ONGOING,
MangaState.FINISHED,
MangaState.PAUSED,
)
override val isTagsExclusionSupported = true
private val tagsMap = SuspendLazy(::parseTags)
override suspend fun getFavicons(): Favicons {
return Favicons(
listOf(
Favicon("https://$domain/uploads/logos/logo-mini.png", 92, null),
),
domain,
)
}
override suspend fun getListPage(page: Int, filter: MangaListFilter?): List<Manga> {
val url = buildString {
append("https://")
append(domain)
append("/tim-kiem?page=")
append(page.toString())
when (filter) {
is MangaListFilter.Search -> {
append("&q=")
append(filter.query.urlEncoded())
}
is MangaListFilter.Advanced -> {
append("&accept_genres=")
if (filter.tags.isNotEmpty()) {
append(
filter.tags.joinToString(",") { it.key },
)
}
append("&reject_genres=")
if (filter.tagsExclude.isNotEmpty()) {
append(
filter.tagsExclude.joinToString(",") { it.key },
)
}
append("&sort=")
append(
when (filter.sortOrder) {
SortOrder.ALPHABETICAL -> "az"
SortOrder.ALPHABETICAL_DESC -> "za"
SortOrder.POPULARITY -> "top"
SortOrder.UPDATED -> "update"
SortOrder.NEWEST -> "new"
SortOrder.RATING -> "like"
},
)
filter.states.oneOrThrowIfMany()?.let {
append("&status=")
append(
when (it) {
MangaState.ONGOING -> "1"
MangaState.FINISHED -> "3"
MangaState.PAUSED -> "2"
else -> ""
},
)
}
}
null -> append("&sort=update")
}
}
val docs = webClient.httpGet(url).parseHtml()
return docs.select(".card-body .thumb-item-flow")
.map {
val titleElement = it.selectFirstOrThrow(".thumb_attr.series-title > a")
val absUrl = titleElement.attrAsAbsoluteUrl("href")
Manga(
id = generateUid(absUrl.toRelativeUrl(domain)),
title = titleElement.text(),
altTitle = null,
url = absUrl.toRelativeUrl(domain),
publicUrl = absUrl,
rating = RATING_UNKNOWN,
isNsfw = true,
coverUrl = it.selectFirst("div.img-in-ratio")?.attrAsAbsoluteUrl("data-bg").orEmpty(),
tags = emptySet(),
state = null,
author = null,
largeCoverUrl = null,
description = null,
source = MangaSource.MANHWA18,
)
}
}
override suspend fun getDetails(manga: Manga): Manga {
val docs = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml()
val cardInfoElement = docs.selectFirst("div.series-information")
val author = cardInfoElement?.selectFirst(".info-name:contains(Author)")?.parent()
?.select("a")
?.joinToString(", ") { it.text() }
val availableTags = tagsMap.get()
val tags = cardInfoElement?.selectFirst(".info-name:contains(Genre)")?.parent()
?.select("a")
?.mapNotNullToSet { availableTags[it.text().lowercase(Locale.ENGLISH)] }
val state = cardInfoElement?.selectFirst(".info-name:contains(Status)")?.parent()
?.selectFirst("a")
?.let {
when (it.text().lowercase()) {
"on going" -> MangaState.ONGOING
"completed" -> MangaState.FINISHED
"on hold" -> MangaState.PAUSED
else -> null
}
}
return manga.copy(
altTitle = cardInfoElement?.selectFirst("b:contains(Other names)")?.parent()?.ownText()?.removePrefix(": "),
author = author,
description = docs.selectFirst(".series-summary .summary-content")?.html(),
tags = tags.orEmpty(),
state = state,
chapters = docs.select(".card-body > .list-chapters > a").mapChapters(reversed = true) { index, element ->
val chapterUrl = element.attrAsAbsoluteUrlOrNull("href")?.toRelativeUrl(domain)
?: return@mapChapters null
val uploadDate = parseUploadDate(element.selectFirst(".chapter-time")?.text())
MangaChapter(
id = generateUid(chapterUrl),
name = element.selectFirst(".chapter-name")?.text().orEmpty(),
number = index + 1,
url = chapterUrl,
scanlator = null,
uploadDate = uploadDate,
branch = null,
source = MangaSource.MANHWA18,
)
},
)
}
// 7 minutes ago
// 5 hours ago
// 2 days ago
// 2 weeks ago
// 4 years ago
private fun parseUploadDate(timeStr: String?): Long {
timeStr ?: return 0
val timeWords = timeStr.split(' ')
if (timeWords.size != 3) return 0
val timeWord = timeWords[1]
val timeAmount = timeWords[0].toIntOrNull() ?: return 0
val timeUnit = when (timeWord) {
"minute", "minutes" -> Calendar.MINUTE
"hour", "hours" -> Calendar.HOUR
"day", "days" -> Calendar.DAY_OF_YEAR
"week", "weeks" -> Calendar.WEEK_OF_YEAR
"month", "months" -> Calendar.MONTH
"year", "years" -> Calendar.YEAR
else -> return 0
}
val cal = Calendar.getInstance()
cal.add(timeUnit, -timeAmount)
return cal.time.time
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val chapterUrl = chapter.url.toAbsoluteUrl(domain)
val doc = webClient.httpGet(chapterUrl).parseHtml()
return doc.requireElementById("chapter-content").select("img").mapNotNull {
val url = it.attrAsRelativeUrlOrNull("data-src")
?: it.attrAsRelativeUrlOrNull("src")
?: return@mapNotNull null
MangaPage(
id = generateUid(url),
url = url,
preview = null,
source = MangaSource.MANHWA18,
)
}
}
override suspend fun getAvailableTags(): Set<MangaTag> {
return tagsMap.get().values.toSet()
}
private suspend fun parseTags(): Map<String, MangaTag> {
val doc = webClient.httpGet("https://$domain/tim-kiem?q=").parseHtml()
val list = doc.getElementsByAttribute("data-genre-id")
if (list.isEmpty()) {
return emptyMap()
}
val result = ArrayMap<String, MangaTag>(list.size)
for (item in list) {
val id = item.attr("data-genre-id")
val name = item.text()
result[name.lowercase(Locale.ENGLISH)] = MangaTag(
title = name.toTitleCase(Locale.ENGLISH),
key = id,
source = source,
)
}
return result
}
}

165
src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/ManhwasMen.kt
View File

@@ -1,165 +0,0 @@
package org.koitharu.kotatsu.parsers.site.en
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import java.text.DateFormat
import java.text.SimpleDateFormat
import java.util.*
@MangaSourceParser("MANHWASMEN", "ManhwasMen", "en", type = ContentType.HENTAI)
class ManhwasMen(context: MangaLoaderContext) :
PagedMangaParser(context, MangaSource.MANHWASMEN, pageSize = 30, searchPageSize = 30) {
override val configKeyDomain: ConfigKey.Domain = ConfigKey.Domain("manhwas.men")
override val isMultipleTagsSupported = false
override val availableSortOrders: Set<SortOrder>
get() = EnumSet.of(SortOrder.POPULARITY)
override val availableStates: Set<MangaState> = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED)
override suspend fun getListPage(page: Int, filter: MangaListFilter?): List<Manga> {
val url = buildString {
append("https://")
append(domain)
append("/manga-list")
append("?page=")
append(page.toString())
when (filter) {
is MangaListFilter.Search -> {
append("&search=")
append(filter.query.urlEncoded())
}
is MangaListFilter.Advanced -> {
filter.tags.oneOrThrowIfMany()?.let {
append("&genero=")
append(it.key)
}
filter.states.oneOrThrowIfMany()?.let {
append("&estado=")
append(
when (it) {
MangaState.ONGOING -> "ongoing"
MangaState.FINISHED -> "complete"
else -> ""
},
)
}
}
null -> {}
}
}
val doc = webClient.httpGet(url).parseHtml()
return doc.select("ul.animes li").map { li ->
val href = li.selectFirstOrThrow("a").attrAsRelativeUrl("href")
Manga(
id = generateUid(href),
url = href,
publicUrl = href.toAbsoluteUrl(domain),
coverUrl = li.selectFirst("img")?.src().orEmpty(),
title = li.selectFirst(".title")?.text().orEmpty(),
altTitle = null,
rating = RATING_UNKNOWN,
tags = emptySet(),
author = null,
state = null,
source = source,
isNsfw = isNsfwSource,
)
}
}
override suspend fun getAvailableTags(): Set<MangaTag> {
val tags = webClient.httpGet("https://$domain/manga-list").parseHtml()
.selectLastOrThrow(".filter-bx .form-group select.custom-select").select("option").drop(1)
return tags.mapNotNullToSet { option ->
MangaTag(
key = option.attr("value").substringAfterLast("="),
title = option.text(),
source = source,
)
}
}
override suspend fun getDetails(manga: Manga): Manga {
val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml()
val simpleDateFormat = SimpleDateFormat("dd/MM/yyyy", sourceLocale)
return manga.copy(
tags = doc.body().select(".genres a").mapNotNullToSet { a ->
MangaTag(
key = a.attr("href").substringAfterLast('='),
title = a.text(),
source = source,
)
},
description = doc.select(".sinopsis").html(),
state = when (doc.selectLast("span.anime-type-peli")?.text()?.lowercase()) {
"ongoing" -> MangaState.ONGOING
"complete" -> MangaState.FINISHED
else -> null
},
chapters = doc.select(".episodes-list li").mapChapters(reversed = true) { i, li ->
val url = li.selectFirstOrThrow("a").attrAsRelativeUrl("href")
MangaChapter(
id = generateUid(url),
name = li.selectFirstOrThrow(".flex-grow-1 span").text(),
number = i + 1,
url = url,
scanlator = null,
uploadDate = parseChapterDate(
simpleDateFormat,
li.selectLastOrThrow(".flex-grow-1 span").text(),
),
branch = null,
source = source,
)
},
)
}
private fun parseChapterDate(dateFormat: DateFormat, date: String?): Long {
val d = date?.lowercase() ?: return 0
return when {
d.endsWith(" ago") -> parseRelativeDate(date)
else -> dateFormat.tryParse(date)
}
}
private fun parseRelativeDate(date: String): Long {
val number = Regex("""(\d+)""").find(date)?.value?.toIntOrNull() ?: return 0
val cal = Calendar.getInstance()
return when {
WordSet("second").anyWordIn(date) -> cal.apply { add(Calendar.SECOND, -number) }.timeInMillis
WordSet("minute", "minutes").anyWordIn(date) -> cal.apply { add(Calendar.MINUTE, -number) }.timeInMillis
WordSet("hour", "hours").anyWordIn(date) -> cal.apply { add(Calendar.HOUR, -number) }.timeInMillis
WordSet("day", "days").anyWordIn(date) -> cal.apply { add(Calendar.DAY_OF_MONTH, -number) }.timeInMillis
WordSet("week", "weeks").anyWordIn(date) -> cal.apply { add(Calendar.WEEK_OF_YEAR, -number) }.timeInMillis
WordSet("month", "months").anyWordIn(date) -> cal.apply { add(Calendar.MONTH, -number) }.timeInMillis
WordSet("year").anyWordIn(date) -> cal.apply { add(Calendar.YEAR, -number) }.timeInMillis
else -> 0
}
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml().requireElementById("chapter_imgs")
return doc.select("img").map { img ->
val url = img.src()?.toRelativeUrl(domain) ?: img.parseFailed("Image src not found")
MangaPage(
id = generateUid(url),
url = url,
preview = null,
source = source,
)
}
}
}

104
src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/Po2Scans.kt
View File

@@ -1,104 +0,0 @@
package org.koitharu.kotatsu.parsers.site.en
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import java.text.SimpleDateFormat
import java.util.*
@MangaSourceParser("PO2SCANS", "Po2Scans", "en")
internal class Po2Scans(context: MangaLoaderContext) : MangaParser(context, MangaSource.PO2SCANS) {
override val availableSortOrders: Set<SortOrder> = EnumSet.of(SortOrder.ALPHABETICAL)
override val configKeyDomain = ConfigKey.Domain("po2scans.com")
override suspend fun getList(offset: Int, filter: MangaListFilter?): List<Manga> {
if (offset > 0) {
return emptyList()
}
val url = buildString {
append("https://")
append(domain)
append("/series")
when (filter) {
is MangaListFilter.Search -> {
append("?search=")
append(filter.query.urlEncoded())
}
is MangaListFilter.Advanced -> {}
null -> {}
}
}
val doc = webClient.httpGet(url).parseHtml()
return doc.select(".series-list").map { div ->
val href = "/" + div.selectFirstOrThrow("a").attrAsRelativeUrl("href")
Manga(
id = generateUid(href),
title = div.selectFirstOrThrow("h2").text(),
altTitle = null,
url = href,
publicUrl = href.toAbsoluteUrl(domain),
rating = RATING_UNKNOWN,
isNsfw = false,
coverUrl = div.selectFirstOrThrow("img").src()?.toAbsoluteUrl(domain).orEmpty(),
tags = emptySet(),
state = null,
author = null,
source = source,
)
}
}
override suspend fun getAvailableTags(): Set<MangaTag> = emptySet()
override suspend fun getDetails(manga: Manga): Manga {
val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml()
val dateFormat = SimpleDateFormat("dd MMM, yy", Locale.ENGLISH)
return manga.copy(
altTitle = null,
state = when (doc.select(".status span").last()?.text()) {
"Ongoing" -> MangaState.ONGOING
"Done" -> MangaState.FINISHED
else -> null
},
tags = emptySet(),
author = doc.select(".author span").last()?.text(),
description = doc.selectFirstOrThrow(".summary").text(),
chapters = doc.select(".chap-section .chap")
.mapChapters(reversed = true) { i, div ->
val a = div.selectFirstOrThrow("a")
val url = "/" + a.attrAsRelativeUrl("href").toAbsoluteUrl(domain)
MangaChapter(
id = generateUid(url),
name = a.text(),
number = i + 1,
url = url,
scanlator = null,
uploadDate = dateFormat.tryParse(div.select(".detail span").last()?.text()),
branch = null,
source = source,
)
},
)
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(domain)
val doc = webClient.httpGet(fullUrl).parseHtml()
return doc.select(".swiper-slide img").map { img ->
val url = img.src()?.replace("./assets", "/assets")?.toRelativeUrl(domain)
?: img.parseFailed("Image src not found")
MangaPage(
id = generateUid(url),
url = url,
preview = null,
source = source,
)
}
}
}

176
src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/Pururin.kt
View File

@@ -1,176 +0,0 @@
package org.koitharu.kotatsu.parsers.site.en
import androidx.collection.ArraySet
import kotlinx.coroutines.async
import kotlinx.coroutines.awaitAll
import kotlinx.coroutines.coroutineScope
import org.jsoup.nodes.Element
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import java.util.*
@MangaSourceParser("PURURIN", "Pururin", "en", ContentType.HENTAI)
internal class Pururin(context: MangaLoaderContext) :
PagedMangaParser(context, MangaSource.PURURIN, pageSize = 20) {
override val availableSortOrders: Set<SortOrder> =
EnumSet.of(SortOrder.UPDATED, SortOrder.POPULARITY, SortOrder.RATING, SortOrder.ALPHABETICAL)
override val configKeyDomain = ConfigKey.Domain("pururin.to")
override val isMultipleTagsSupported = false
override suspend fun getListPage(page: Int, filter: MangaListFilter?): List<Manga> {
val url = buildString {
append("https://")
append(domain)
when (filter) {
is MangaListFilter.Search -> {
append("/search?q=")
append(filter.query.urlEncoded())
append("&page=")
append(page.toString())
}
is MangaListFilter.Advanced -> {
append("/browse")
filter.tags.oneOrThrowIfMany()?.let {
append("/tags/content/")
append(it.key)
append("/")
}
append("?page=")
append(page)
append("&sort=")
when (filter.sortOrder) {
SortOrder.UPDATED -> append("")
SortOrder.POPULARITY -> append("most-viewed")
SortOrder.RATING -> append("highest-rated")
SortOrder.ALPHABETICAL -> append("title")
else -> append("")
}
}
null -> {
append("/browse?page=")
append(page)
}
}
}
val doc = webClient.httpGet(url).parseHtml()
return doc.select(".row-gallery a.card-gallery").map { a ->
val href = a.attrAsRelativeUrl("href")
Manga(
id = generateUid(href),
url = href,
publicUrl = href.toAbsoluteUrl(domain),
coverUrl = a.selectFirst("img.card-img-top")?.src().orEmpty(),
title = a.selectFirst(".title")?.text().orEmpty(),
altTitle = null,
rating = RATING_UNKNOWN,
tags = emptySet(),
author = null,
state = null,
source = source,
isNsfw = isNsfwSource,
)
}
}
override suspend fun getAvailableTags(): Set<MangaTag> {
return coroutineScope {
(1..4).map { page ->
async { getTags(page) }
}
}.awaitAll().flattenTo(ArraySet(360))
}
private suspend fun getTags(page: Int): Set<MangaTag> {
val url = "https://$domain/tags/content?order=uses&page=$page"
val root = webClient.httpGet(url).parseHtml()
return root.parseTags()
}
private fun Element.parseTags() = select("table tr td a").mapToSet {
val href = it.attr("href").substringAfterLast("content/").substringBeforeLast('/')
MangaTag(
key = href,
title = it.text(),
source = source,
)
}
override suspend fun getDetails(manga: Manga): Manga = coroutineScope {
val fullUrl = manga.url.toAbsoluteUrl(domain)
val doc = webClient.httpGet(fullUrl).parseHtml()
manga.copy(
description = doc.selectFirst("p.mb-2")?.text().orEmpty(),
rating = doc.selectFirst("td span.rating")?.attr("content")?.toFloatOrNull()?.div(5f) ?: RATING_UNKNOWN,
tags = doc.body().select("tr:contains(Contents) ul.list-inline a").mapNotNullToSet {
val href = it.attr("href").substringAfterLast("content/").substringBeforeLast('/')
MangaTag(
key = href,
title = it.text(),
source = source,
)
},
author = doc.selectFirst("a[itemprop=author]")?.text(),
chapters = listOf(
MangaChapter(
id = manga.id,
name = manga.title,
number = 1,
url = manga.url,
scanlator = null,
uploadDate = 0,
branch = null,
source = source,
),
),
)
}
override suspend fun getRelatedManga(seed: Manga): List<Manga> {
val doc = webClient.httpGet(seed.url.toAbsoluteUrl(domain)).parseHtml()
val root = doc.body().selectFirstOrThrow(".row-gallery-small")
return root.select("a.card-gallery").mapNotNull { a ->
val href = a.attrAsRelativeUrl("href")
Manga(
id = generateUid(href),
url = href,
publicUrl = href.toAbsoluteUrl(domain),
coverUrl = a.selectFirst("img.card-img-top")?.src().orEmpty(),
title = a.selectFirst(".title")?.text().orEmpty(),
altTitle = null,
rating = RATING_UNKNOWN,
tags = emptySet(),
author = null,
state = null,
source = source,
isNsfw = false,
)
}
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(domain)
val doc = webClient.httpGet(fullUrl).parseHtml()
return doc.select(".gallery-preview img").map { url ->
val img = url.src()?.toRelativeUrl(domain) ?: url.parseFailed("Image src not found")
val urlImage = img.replace("t.", ".")
MangaPage(
id = generateUid(urlImage),
url = urlImage,
preview = null,
source = source,
)
}
}
}

150
src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/ReaperComics.kt
View File

@@ -1,150 +0,0 @@
package org.koitharu.kotatsu.parsers.site.en
import okhttp3.Headers
import org.jsoup.nodes.Document
import org.koitharu.kotatsu.parsers.ErrorMessages
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.network.UserAgents
import org.koitharu.kotatsu.parsers.util.*
import java.text.DateFormat
import java.text.SimpleDateFormat
import java.util.*
@MangaSourceParser("REAPERCOMICS", "ReaperComics", "en")
internal class ReaperComics(context: MangaLoaderContext) :
PagedMangaParser(context, MangaSource.REAPERCOMICS, pageSize = 30) {
override val availableSortOrders: Set<SortOrder> = EnumSet.of(SortOrder.UPDATED, SortOrder.ALPHABETICAL)
override val configKeyDomain = ConfigKey.Domain("reaperscans.com")
override val isSearchSupported = false
override val headers: Headers = Headers.Builder()
.add("User-Agent", UserAgents.CHROME_DESKTOP)
.build()
override suspend fun getListPage(page: Int, filter: MangaListFilter?): List<Manga> {
val url = buildString {
append("https://")
append(domain)
when (filter) {
is MangaListFilter.Search -> {
throw IllegalArgumentException(ErrorMessages.SEARCH_NOT_SUPPORTED) // TODO
}
is MangaListFilter.Advanced -> {
append("/")
if (filter.sortOrder == SortOrder.UPDATED) {
append("latest/")
}
append("comics?page=")
append(page.toString())
}
null -> {
append("/latest/comics?page=")
append(page.toString())
}
}
}
return parseMangaList(webClient.httpGet(url).parseHtml())
}
private fun parseMangaList(docs: Document): List<Manga> {
return docs.select("main div.relative, main li.col-span-1").map {
val a = it.selectFirstOrThrow("a")
val url = a.attrAsAbsoluteUrl("href")
Manga(
id = generateUid(url),
url = url,
title = (it.selectFirst("p a") ?: it.selectLast("a"))?.text().orEmpty(),
altTitle = null,
publicUrl = url,
rating = RATING_UNKNOWN,
isNsfw = isNsfwSource,
coverUrl = it.selectFirstOrThrow("img").src().orEmpty(),
tags = emptySet(),
state = null,
author = null,
source = source,
)
}
}
override suspend fun getAvailableTags(): Set<MangaTag> = emptySet()
override suspend fun getDetails(manga: Manga): Manga {
val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml()
val simpleDateFormat = SimpleDateFormat("dd/MM/yyyy", sourceLocale)
return manga.copy(
description = doc.selectFirst("div.p-4 p.prose")?.html(),
state = when (doc.selectFirst("dl.mt-2 div:contains(Status) dd")?.text()?.lowercase()) {
"ongoing" -> MangaState.ONGOING
"complete" -> MangaState.FINISHED
else -> null
},
chapters = doc.select("div.p-2 div.pb-4 ul li").mapChapters(reversed = true) { i, li ->
val a = li.selectFirstOrThrow("a")
val chapterUrl = a.attrAsAbsoluteUrl("href").toRelativeUrl(domain)
MangaChapter(
id = generateUid(chapterUrl),
name = li.selectFirst("div.truncate p.truncate")?.text().orEmpty(),
number = i + 1,
url = chapterUrl,
scanlator = null,
uploadDate = parseChapterDate(
simpleDateFormat,
li.selectFirst("div.truncate div.items-center")?.text(),
),
branch = null,
source = source,
)
},
)
}
private fun parseChapterDate(dateFormat: DateFormat, date: String?): Long {
val d = date?.lowercase() ?: return 0
return when {
d.endsWith(" ago") -> parseRelativeDate(date)
else -> dateFormat.tryParse(date)
}
}
private fun parseRelativeDate(date: String): Long {
val number = Regex("""(\d+)""").find(date)?.value?.toIntOrNull() ?: return 0
val cal = Calendar.getInstance()
return when {
WordSet("second").anyWordIn(date) -> cal.apply { add(Calendar.SECOND, -number) }.timeInMillis
WordSet("minute", "minutes").anyWordIn(date) -> cal.apply { add(Calendar.MINUTE, -number) }.timeInMillis
WordSet("hour", "hours").anyWordIn(date) -> cal.apply { add(Calendar.HOUR, -number) }.timeInMillis
WordSet("day", "days").anyWordIn(date) -> cal.apply { add(Calendar.DAY_OF_MONTH, -number) }.timeInMillis
WordSet("week", "weeks").anyWordIn(date) -> cal.apply { add(Calendar.WEEK_OF_YEAR, -number) }.timeInMillis
WordSet("month", "months").anyWordIn(date) -> cal.apply { add(Calendar.MONTH, -number) }.timeInMillis
WordSet("year").anyWordIn(date) -> cal.apply { add(Calendar.YEAR, -number) }.timeInMillis
else -> 0
}
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml()
return doc.select("img.max-w-full").map { img ->
val url = img.src()?.toRelativeUrl(domain) ?: img.parseFailed("Image src not found")
MangaPage(
id = generateUid(url),
url = url,
preview = null,
source = source,
)
}
}
}

142
src/main/kotlin/org/koitharu/kotatsu/parsers/site/es/TempleScanEsp.kt
View File

@@ -1,142 +0,0 @@
package org.koitharu.kotatsu.parsers.site.es
import kotlinx.coroutines.coroutineScope
import okhttp3.Headers
import org.jsoup.nodes.Document
import org.koitharu.kotatsu.parsers.ErrorMessages
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.network.UserAgents
import org.koitharu.kotatsu.parsers.util.*
import java.util.*
@MangaSourceParser("TEMPLESCANESP", "TempleScanEsp", "es", ContentType.HENTAI)
internal class TempleScanEsp(context: MangaLoaderContext) :
PagedMangaParser(context, MangaSource.TEMPLESCANESP, pageSize = 15) {
override val availableSortOrders: Set<SortOrder> = EnumSet.of(SortOrder.NEWEST, SortOrder.UPDATED)
override val configKeyDomain = ConfigKey.Domain("templescanesp.net")
override val isSearchSupported = false
override val headers: Headers = Headers.Builder()
.add("User-Agent", UserAgents.CHROME_DESKTOP)
.build()
override suspend fun getListPage(page: Int, filter: MangaListFilter?): List<Manga> {
val url = buildString {
append("https://")
append(domain)
when (filter) {
is MangaListFilter.Search -> {
throw IllegalArgumentException(ErrorMessages.SEARCH_NOT_SUPPORTED)
}
is MangaListFilter.Advanced -> {
if (filter.sortOrder == SortOrder.NEWEST) {
append("/comics?page=")
append(page.toString())
} else {
if (page > 1) {
return emptyList()
}
}
}
null -> {
append("/comics?page=")
append(page.toString())
}
}
}
val doc = webClient.httpGet(url).parseHtml()
return doc.select("div.grid figure").ifEmpty {
doc.requireElementById("projectsDiv").select("figure")
}.map { div ->
val href = div.selectFirstOrThrow("a").attrAsRelativeUrl("href")
Manga(
id = generateUid(href),
url = href,
publicUrl = href.toAbsoluteUrl(div.host ?: domain),
coverUrl = div.selectFirst("img")?.src().orEmpty(),
title = div.selectFirst("figcaption")?.text().orEmpty(),
altTitle = null,
rating = RATING_UNKNOWN,
tags = emptySet(),
author = null,
state = null,
source = source,
isNsfw = isNsfwSource,
)
}
}
override suspend fun getAvailableTags(): Set<MangaTag> = emptySet()
override suspend fun getDetails(manga: Manga): Manga = coroutineScope {
val fullUrl = manga.url.toAbsoluteUrl(domain)
val doc = webClient.httpGet(fullUrl).parseHtml()
val chaptersDeferred = getChapters(doc)
manga.copy(
description = doc.requireElementById("section-sinopsis").html(),
chapters = chaptersDeferred,
)
}
private fun getChapters(doc: Document): List<MangaChapter> {
return doc.body().select("div.grid-capitulos div.contenedor-capitulo-miniatura")
.mapChapters(reversed = true) { i, div ->
val href = div.selectFirstOrThrow("a").attrAsRelativeUrl("href")
val date = parseUploadDate(div.selectFirstOrThrow("time").text())
MangaChapter(
id = generateUid(href),
name = div.requireElementById("name").text(),
number = i + 1,
url = href,
uploadDate = date,
source = source,
scanlator = null,
branch = null,
)
}
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(domain)
val doc = webClient.httpGet(fullUrl).parseHtml()
return doc.select("main.contenedor-imagen img").map { url ->
val img = url.src()?.toRelativeUrl(domain) ?: url.parseFailed("Image src not found")
MangaPage(
id = generateUid(img),
url = img,
preview = null,
source = source,
)
}
}
private fun parseUploadDate(timeStr: String?): Long {
timeStr ?: return 0
val timeWords = timeStr.split(' ')
if (timeWords.size != 3) return 0
val timeWord = timeWords[1]
val timeAmount = timeWords[0].toIntOrNull() ?: return 0
val timeUnit = when (timeWord) {
"minute", "minutes" -> Calendar.MINUTE
"hour", "hours" -> Calendar.HOUR
"day", "days" -> Calendar.DAY_OF_YEAR
"week", "weeks" -> Calendar.WEEK_OF_YEAR
"month", "months" -> Calendar.MONTH
"year", "years" -> Calendar.YEAR
else -> return 0
}
val cal = Calendar.getInstance()
cal.add(timeUnit, -timeAmount)
return cal.time.time
}
}

253
src/main/kotlin/org/koitharu/kotatsu/parsers/site/es/TuMangaOnlineParser.kt
View File

@@ -1,253 +0,0 @@
package org.koitharu.kotatsu.parsers.site.es
import okhttp3.Headers
import okhttp3.HttpUrl.Companion.toHttpUrl
import org.jsoup.nodes.Document
import org.jsoup.nodes.Element
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import java.text.SimpleDateFormat
import java.util.*
@MangaSourceParser("TUMANGAONLINE", "TuMangaOnline", "es")
class TuMangaOnlineParser(context: MangaLoaderContext) : PagedMangaParser(
context,
source = MangaSource.TUMANGAONLINE,
pageSize = 24,
) {
override val configKeyDomain = ConfigKey.Domain("visortmo.com")
private val chapterDateFormat = SimpleDateFormat("yyyy-MM-dd", sourceLocale)
override val availableContentRating: Set<ContentRating> = EnumSet.of(ContentRating.SAFE, ContentRating.ADULT)
override val availableSortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.ALPHABETICAL,
SortOrder.ALPHABETICAL_DESC,
SortOrder.UPDATED,
SortOrder.NEWEST,
SortOrder.POPULARITY,
SortOrder.RATING,
)
override suspend fun getListPage(page: Int, filter: MangaListFilter?): List<Manga> {
val url = buildString {
append("https://")
append(domain)
append("/library")
when (filter) {
is MangaListFilter.Search -> {
append("?title=")
append(filter.query.urlEncoded())
}
is MangaListFilter.Advanced -> {
append("?order_item=")
append(
when (filter.sortOrder) {
SortOrder.POPULARITY -> "likes_count&order_dir=desc"
SortOrder.UPDATED -> "release_date&order_dir=desc"
SortOrder.NEWEST -> "creation&order_dir=desc"
SortOrder.ALPHABETICAL -> "alphabetically&order_dir=asc"
SortOrder.ALPHABETICAL_DESC -> "alphabetically&order_dir=desc"
SortOrder.RATING -> "score&order_dir=desc"
},
)
append("&filter_by=title")
if (filter.tags.isNotEmpty()) {
for (tag in filter.tags) {
append("&genders[]=")
append(tag.key)
}
}
filter.contentRating.oneOrThrowIfMany()?.let {
append("&erotic=")
append(
when (it) {
ContentRating.SAFE -> "false"
ContentRating.ADULT -> "true"
else -> ""
},
)
}
}
null -> {
append("?order_item=release_date&order_dir=desc&filter_by=title")
}
}
append("&_pg=1&page=")
append(page.toString())
}
val doc = webClient.httpGet(url, headers).parseHtml()
val items = doc.body().select("div.element")
return items.mapNotNull { item ->
val href =
item.selectFirst("a")?.attrAsRelativeUrlOrNull("href")?.substringAfter(' ') ?: return@mapNotNull null
Manga(
id = generateUid(href),
title = item.selectFirst("h4.text-truncate")?.text() ?: return@mapNotNull null,
coverUrl = item.select("style").toString().substringAfter("('").substringBeforeLast("')"),
altTitle = null,
author = null,
rating = item.selectFirst("span.score")?.text()?.toFloatOrNull()?.div(10F) ?: RATING_UNKNOWN,
url = href,
isNsfw = item.select("i").hasClass("fas fa-heartbeat fa-2x"),
tags = emptySet(),
state = null,
publicUrl = href.toAbsoluteUrl(doc.host ?: domain),
source = source,
)
}
}
override suspend fun getDetails(manga: Manga): Manga {
val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml()
val contents = doc.body().selectFirstOrThrow("section.element-header-content")
return manga.copy(
description = contents.selectFirst("p.element-description")?.html(),
tags = contents.select("h6 a").mapNotNullToSet { a ->
MangaTag(
key = a.attr("href").substringBefore("&").substringAfterLast("="),
title = a.text(),
source = source,
)
},
largeCoverUrl = contents.selectFirst(".book-thumbnail")?.attrAsAbsoluteUrlOrNull("src"),
state = parseStatus(contents.select("span.book-status").text().orEmpty()),
author = contents.selectFirst("h5.card-title")?.attr("title")?.substringAfter(", "),
chapters = if (doc.select("div.chapters").isEmpty()) {
doc.select(oneShotChapterListSelector).mapChapters(reversed = true) { _, item ->
oneShotChapterFromElement(item)
}
} else {
val chapters = ChaptersListBuilder(10)
doc.select(regularChapterListSelector).reversed().forEachIndexed { i, item ->
val chapterName = item.select("div.col-10.text-truncate").text().replace("&nbsp;", " ").trim()
val scanElement = item.select("ul.chapter-list > li")
scanElement.forEach { chapters.add(regularChapterFromElement(it, chapterName, i)) }
}
chapters.toList()
},
)
}
private val oneShotChapterListSelector = "div.chapter-list-element > ul.list-group li.list-group-item"
private fun oneShotChapterFromElement(element: Element): MangaChapter {
val href = element.selectFirstOrThrow("div.row > .text-right > a").attrAsRelativeUrl("href")
return MangaChapter(
id = generateUid(href),
name = "One Shot",
number = 1,
url = href,
scanlator = element.select("div.col-md-6.text-truncate").text(),
branch = null,
uploadDate = chapterDateFormat.tryParse(element.select("span.badge.badge-primary.p-2").first()?.text()),
source = source,
)
}
private val regularChapterListSelector = "div.chapters > ul.list-group li.p-0.list-group-item"
private fun regularChapterFromElement(element: Element, chName: String, number: Int): MangaChapter {
val href = element.selectFirstOrThrow("div.row > .text-right > a").attrAsRelativeUrl("href")
return MangaChapter(
id = generateUid(href),
name = chName,
number = number + 1,
url = href,
scanlator = element.select("div.col-md-6.text-truncate").text(),
branch = null,
uploadDate = chapterDateFormat.tryParse(element.select("span.badge.badge-primary.p-2").first()?.text()),
source = source,
)
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val redirectDoc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain), headers).parseHtml()
var doc = redirectToReadingPage(redirectDoc)
val currentUrl = doc.location()
val newUrl = if (!currentUrl.contains("cascade")) {
currentUrl.substringBefore("paginated") + "cascade"
} else {
currentUrl
}
if (currentUrl != newUrl) {
doc = webClient.httpGet(newUrl, headers).parseHtml()
}
return doc.select("div.viewer-container img:not(noscript img)").map {
val href = if (it.hasAttr("data-src")) {
it.attr("abs:data-src")
} else {
it.attr("abs:src")
}
MangaPage(
id = generateUid(href),
url = href,
preview = null,
source = source,
)
}
}
private suspend fun redirectToReadingPage(document: Document): Document {
val script1 = document.selectFirst("script:containsData(uniqid)")
val script2 = document.selectFirst("script:containsData(window.location.replace)")
val redirectHeaders = Headers.Builder().set("Referer", document.baseUri()).build()
if (script1 != null) {
val data = script1.data()
val regexParams = """\{uniqid:'(.+)',cascade:(.+)\}""".toRegex()
val regexAction = """form\.action\s?=\s?'(.+)'""".toRegex()
val params = regexParams.find(data)!!
val action = regexAction.find(data)!!.groupValues[1].toHttpUrl()
val formBody = mapOf(
"uniqid" to params.groupValues[1],
"cascade" to params.groupValues[2],
)
return redirectToReadingPage(webClient.httpPost(action, formBody, redirectHeaders).parseHtml())
}
if (script2 != null) {
val data = script2.data()
val regexRedirect = """window\.location\.replace\('(.+)'\)""".toRegex()
val url = regexRedirect.find(data)!!.groupValues[1]
return redirectToReadingPage(webClient.httpGet(url, redirectHeaders).parseHtml())
}
return document
}
override suspend fun getAvailableTags(): Set<MangaTag> {
val doc = webClient.httpGet("https://$domain/library", headers).parseHtml()
val elements = doc.body().select("div#books-genders > div > div")
return elements.mapNotNullToSet { element ->
MangaTag(
title = element.select("label").text(),
key = element.select("input").attr("value"),
source = source,
)
}
}
private fun parseStatus(status: String) = when {
status.contains("Publicándose") -> MangaState.ONGOING
status.contains("Finalizado") -> MangaState.FINISHED
else -> null
}
}

320
src/main/kotlin/org/koitharu/kotatsu/parsers/site/fmreader/FmreaderParser.kt
View File

@@ -1,320 +0,0 @@
package org.koitharu.kotatsu.parsers.site.fmreader
import kotlinx.coroutines.async
import kotlinx.coroutines.coroutineScope
import org.jsoup.nodes.Document
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import java.text.DateFormat
import java.text.SimpleDateFormat
import java.util.*
internal abstract class FmreaderParser(
context: MangaLoaderContext,
source: MangaSource,
domain: String,
pageSize: Int = 20,
) : PagedMangaParser(context, source, pageSize) {
override val configKeyDomain = ConfigKey.Domain(domain)
override val availableSortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.UPDATED,
SortOrder.POPULARITY,
SortOrder.ALPHABETICAL,
SortOrder.ALPHABETICAL_DESC,
)
override val availableStates: Set<MangaState> = EnumSet.of(
MangaState.ONGOING,
MangaState.FINISHED,
MangaState.ABANDONED,
)
override val isTagsExclusionSupported = true
protected open val listUrl = "/manga-list.html"
protected open val datePattern = "MMMM d, yyyy"
protected open val tagPrefix = "manga-list-genre-"
init {
paginator.firstPage = 1
searchPaginator.firstPage = 1
}
@JvmField
protected val ongoing: Set<String> = setOf(
"on going",
"incomplete",
"en curso",
)
@JvmField
protected val finished: Set<String> = setOf(
"completed",
"completado",
)
@JvmField
protected val abandoned: Set<String> = hashSetOf(
"canceled",
"cancelled",
"drop",
)
override suspend fun getListPage(page: Int, filter: MangaListFilter?): List<Manga> {
val url = buildString {
append("https://")
append(domain)
append(listUrl)
append("?page=")
append(page.toString())
when (filter) {
is MangaListFilter.Search -> {
append("&name=")
append(filter.query.urlEncoded())
}
is MangaListFilter.Advanced -> {
append("&genre=")
append(filter.tags.joinToString(",") { it.key })
append("&ungenre=")
append(filter.tagsExclude.joinToString(",") { it.key })
append("&sort=")
when (filter.sortOrder) {
SortOrder.POPULARITY -> append("views")
SortOrder.UPDATED -> append("last_update")
SortOrder.ALPHABETICAL -> append("name&sort_type=ASC")
SortOrder.ALPHABETICAL_DESC -> append("name&sort_type=DESC")
else -> append("last_update")
}
append("&m_status=")
filter.states.oneOrThrowIfMany()?.let {
append(
when (it) {
MangaState.ONGOING -> "2"
MangaState.FINISHED -> "1"
MangaState.ABANDONED -> "3"
else -> ""
},
)
}
}
null -> append("&sort=last_update")
}
}
return parseMangaList(webClient.httpGet(url).parseHtml())
}
protected open fun parseMangaList(doc: Document): List<Manga> {
return doc.select("div.thumb-item-flow").map { div ->
val href = div.selectFirstOrThrow("div.series-title a").attrAsRelativeUrl("href")
Manga(
id = generateUid(href),
url = href,
publicUrl = href.toAbsoluteUrl(div.host ?: domain),
coverUrl = div.selectFirstOrThrow("div.img-in-ratio").attr("data-bg")
?: div.selectFirstOrThrow("div.img-in-ratio").attr("style").substringAfter("(")
.substringBefore(")"),
title = div.selectFirstOrThrow("div.series-title").text().orEmpty(),
altTitle = null,
rating = RATING_UNKNOWN,
tags = emptySet(),
author = null,
state = null,
source = source,
isNsfw = isNsfwSource,
)
}
}
protected open val selectBodyTag = "ul.filter-type li a"
override suspend fun getAvailableTags(): Set<MangaTag> {
val doc = webClient.httpGet("https://$domain/$listUrl").parseHtml()
return doc.select(selectBodyTag).mapNotNullToSet { a ->
val href = a.attr("href").substringAfter(tagPrefix).substringBeforeLast(".html")
MangaTag(
key = href,
title = a.text().toTitleCase(sourceLocale),
source = source,
)
}
}
protected open val selectDesc = "div.summary-content"
protected open val selectState = "ul.manga-info li:contains(Status) a"
protected open val selectAlt = "ul.manga-info li:contains(Other names)"
protected open val selectAut = "ul.manga-info li:contains(Author(s)) a"
protected open val selectTag = "ul.manga-info li:contains(Genre(s)) a"
override suspend fun getDetails(manga: Manga): Manga = coroutineScope {
val fullUrl = manga.url.toAbsoluteUrl(domain)
val doc = webClient.httpGet(fullUrl).parseHtml()
val chaptersDeferred = async { getChapters(doc) }
val desc = doc.selectFirst(selectDesc)?.html()
val stateDiv = doc.selectFirst(selectState)
val state = stateDiv?.let {
when (it.text().lowercase()) {
in ongoing -> MangaState.ONGOING
in finished -> MangaState.FINISHED
in abandoned -> MangaState.ABANDONED
else -> null
}
}
val alt = doc.body().selectFirst(selectAlt)?.text()?.replace("Other names", "")
val auth = doc.body().selectFirst(selectAut)?.text()
manga.copy(
tags = doc.body().select(selectTag).mapNotNullToSet { a ->
MangaTag(
key = a.attr("href").substringAfter(tagPrefix).substringBeforeLast(".html"),
title = a.text().toTitleCase(),
source = source,
)
},
description = desc,
altTitle = alt,
author = auth,
state = state,
chapters = chaptersDeferred.await(),
)
}
protected open val selectDate = "div.chapter-time"
protected open val selectChapter = "ul.list-chapters a"
protected open suspend fun getChapters(doc: Document): List<MangaChapter> {
val dateFormat = SimpleDateFormat(datePattern, sourceLocale)
return doc.body().select(selectChapter).mapChapters(reversed = true) { i, a ->
val href = a.attrAsRelativeUrl("href")
val dateText = a.selectFirst(selectDate)?.text()
MangaChapter(
id = generateUid(href),
name = a.selectFirstOrThrow("div.chapter-name").text(),
number = i + 1,
url = href,
uploadDate = parseChapterDate(
dateFormat,
dateText,
),
source = source,
scanlator = null,
branch = null,
)
}
}
protected open val selectPage = "div.chapter-content img"
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(domain)
val doc = webClient.httpGet(fullUrl).parseHtml()
return doc.select(selectPage).map { img ->
val url = img.src()?.toRelativeUrl(domain) ?: img.parseFailed("Image src not found")
MangaPage(
id = generateUid(url),
url = url,
preview = null,
source = source,
)
}
}
protected fun parseChapterDate(dateFormat: DateFormat, date: String?): Long {
// Clean date (e.g. 5th December 2019 to 5 December 2019) before parsing it
val d = date?.lowercase() ?: return 0
return when {
d.endsWith(" ago") ||
d.endsWith(" atrás") ||
// short Hours
d.endsWith(" h") ||
// short Day
d.endsWith(" d") -> parseRelativeDate(date)
// Handle 'yesterday' and 'today', using midnight
d.startsWith("year") -> Calendar.getInstance().apply {
add(Calendar.DAY_OF_MONTH, -1) // yesterday
set(Calendar.HOUR_OF_DAY, 0)
set(Calendar.MINUTE, 0)
set(Calendar.SECOND, 0)
set(Calendar.MILLISECOND, 0)
}.timeInMillis
d.startsWith("today") -> Calendar.getInstance().apply {
set(Calendar.HOUR_OF_DAY, 0)
set(Calendar.MINUTE, 0)
set(Calendar.SECOND, 0)
set(Calendar.MILLISECOND, 0)
}.timeInMillis
date.contains(Regex("""\d(st|nd|rd|th)""")) -> date.split(" ").map {
if (it.contains(Regex("""\d\D\D"""))) {
it.replace(Regex("""\D"""), "")
} else {
it
}
}.let { dateFormat.tryParse(it.joinToString(" ")) }
else -> dateFormat.tryParse(date)
}
}
// Parses dates in this form:
// 21 hours ago
private fun parseRelativeDate(date: String): Long {
val number = Regex("""(\d+)""").find(date)?.value?.toIntOrNull() ?: return 0
val cal = Calendar.getInstance()
return when {
WordSet("second").anyWordIn(date) -> cal.apply { add(Calendar.SECOND, -number) }.timeInMillis
WordSet("min", "minute", "minutes", "minuto", "minutos").anyWordIn(date) -> cal.apply {
add(
Calendar.MINUTE,
-number,
)
}.timeInMillis
WordSet("hour", "hours", "hora", "horas", "h").anyWordIn(date) -> cal.apply {
add(
Calendar.HOUR,
-number,
)
}.timeInMillis
WordSet("day", "days", "día", "dia").anyWordIn(date) -> cal.apply {
add(
Calendar.DAY_OF_MONTH,
-number,
)
}.timeInMillis
WordSet("week", "weeks", "semana", "semanas").anyWordIn(date) -> cal.apply {
add(
Calendar.WEEK_OF_YEAR,
-number,
)
}.timeInMillis
WordSet("month", "months", "mes", "meses").anyWordIn(date) -> cal.apply {
add(
Calendar.MONTH,
-number,
)
}.timeInMillis
WordSet("year", "año", "años").anyWordIn(date) -> cal.apply { add(Calendar.YEAR, -number) }.timeInMillis
else -> 0
}
}
}

140
src/main/kotlin/org/koitharu/kotatsu/parsers/site/fmreader/en/Manhwa18Com.kt
View File

@@ -1,140 +0,0 @@
package org.koitharu.kotatsu.parsers.site.fmreader.en
import kotlinx.coroutines.async
import kotlinx.coroutines.coroutineScope
import org.jsoup.nodes.Document
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.site.fmreader.FmreaderParser
import org.koitharu.kotatsu.parsers.util.*
import java.text.SimpleDateFormat
@MangaSourceParser("MANHWA18COM", "Manhwa18.com", "en", ContentType.HENTAI)
internal class Manhwa18Com(context: MangaLoaderContext) :
FmreaderParser(context, MangaSource.MANHWA18COM, "manhwa18.com") {
override val listUrl = "/tim-kiem"
override val selectState = "div.info-item:contains(Status) span.info-value "
override val selectAlt = "div.info-item:contains(Other name) span.info-value "
override val selectTag = "div.info-item:contains(Genre) span.info-value a"
override val datePattern = "dd/MM/yyyy"
override val selectPage = "div#chapter-content img"
override val selectBodyTag = "div.advanced-wrapper .genre_label"
override suspend fun getListPage(page: Int, filter: MangaListFilter?): List<Manga> {
val url = buildString {
append("https://")
append(domain)
append("/tim-kiem?page=")
append(page.toString())
when (filter) {
is MangaListFilter.Search -> {
append("&q=")
append(filter.query.urlEncoded())
}
is MangaListFilter.Advanced -> {
append("&accept_genres=")
append(filter.tags.joinToString(",") { it.key })
append("&reject_genres=")
append(filter.tagsExclude.joinToString(",") { it.key })
append("&sort=")
append(
when (filter.sortOrder) {
SortOrder.ALPHABETICAL -> "az"
SortOrder.ALPHABETICAL_DESC -> "za"
SortOrder.POPULARITY -> "top"
SortOrder.UPDATED -> "update"
SortOrder.NEWEST -> "new"
SortOrder.RATING -> "like"
},
)
filter.states.oneOrThrowIfMany()?.let {
append("&status=")
append(
when (it) {
MangaState.ONGOING -> "1"
MangaState.FINISHED -> "3"
MangaState.PAUSED -> "2"
else -> ""
},
)
}
}
null -> append("&sort=update")
}
}
return parseMangaList(webClient.httpGet(url).parseHtml())
}
override suspend fun getAvailableTags(): Set<MangaTag> {
val doc = webClient.httpGet("https://$domain/$listUrl").parseHtml()
return doc.select(selectBodyTag).mapNotNullToSet { label ->
val key = label.attr("data-genre-id")
MangaTag(
key = key,
title = label.selectFirstOrThrow(".gerne-name").text(),
source = source,
)
}
}
override suspend fun getDetails(manga: Manga): Manga = coroutineScope {
val fullUrl = manga.url.toAbsoluteUrl(domain)
val doc = webClient.httpGet(fullUrl).parseHtml()
val chaptersDeferred = async { getChapters(doc) }
val desc = doc.selectFirstOrThrow(selectDesc).html()
val stateDiv = doc.selectFirst(selectState)
val state = stateDiv?.let {
when (it.text().lowercase()) {
in ongoing -> MangaState.ONGOING
in finished -> MangaState.FINISHED
else -> null
}
}
val alt = doc.body().selectFirst(selectAlt)?.text()?.replace("Other name", "")
val auth = doc.body().selectFirst(selectAut)?.text()
manga.copy(
tags = doc.body().select(selectTag).mapNotNullToSet { a ->
MangaTag(
key = a.attr("href").substringAfter("manga-list-genre-").substringBeforeLast(".html"),
title = a.text().toTitleCase(),
source = source,
)
},
description = desc,
altTitle = alt,
author = auth,
state = state,
chapters = chaptersDeferred.await(),
)
}
override suspend fun getChapters(doc: Document): List<MangaChapter> {
val dateFormat = SimpleDateFormat(datePattern, sourceLocale)
return doc.body().select(selectChapter).mapChapters(reversed = true) { i, a ->
val href = a.attrAsRelativeUrl("href")
val dateText = a.selectFirst(selectDate)?.text()?.substringAfter("- ")
MangaChapter(
id = generateUid(href),
name = a.selectFirstOrThrow("div.chapter-name").text(),
number = i + 1,
url = href,
uploadDate = parseChapterDate(
dateFormat,
dateText,
),
source = source,
scanlator = null,
branch = null,
)
}
}
}

114
src/main/kotlin/org/koitharu/kotatsu/parsers/site/fmreader/es/OlimpoScans.kt
View File

@@ -1,114 +0,0 @@
package org.koitharu.kotatsu.parsers.site.fmreader.es
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.site.fmreader.FmreaderParser
import org.koitharu.kotatsu.parsers.util.*
@MangaSourceParser("OLIMPOSCANS", "OlimpoScans", "es")
internal class OlimpoScans(context: MangaLoaderContext) :
FmreaderParser(context, MangaSource.OLIMPOSCANS, "olimposcans.com") {
override val selectState = "ul.manga-info li:contains(Estado) a"
override val selectAlt = "ul.manga-info li:contains(Otros nombres)"
override val selectTag = "ul.manga-info li:contains(Género) a"
override val tagPrefix = "lista-de-comics-genero-"
override val isMultipleTagsSupported = false
override val isTagsExclusionSupported = false
override suspend fun getListPage(page: Int, filter: MangaListFilter?): List<Manga> {
val url = buildString {
append("https://")
append(domain)
when (filter) {
is MangaListFilter.Search -> {
append(listUrl)
append("?page=")
append(page.toString())
append("&name=")
append(filter.query.urlEncoded())
}
is MangaListFilter.Advanced -> {
if (filter.tags.isNotEmpty()) {
filter.tags.oneOrThrowIfMany()?.let {
append("/lista-de-comics-genero-")
append(it.key)
append(".html")
}
} else {
append(listUrl)
append("?page=")
append(page.toString())
append("&sort=")
when (filter.sortOrder) {
SortOrder.POPULARITY -> append("views")
SortOrder.UPDATED -> append("last_update")
SortOrder.ALPHABETICAL -> append("name&sort_type=ASC")
SortOrder.ALPHABETICAL_DESC -> append("name&sort_type=DESC")
else -> append("last_update")
}
}
append("&m_status=")
filter.states.oneOrThrowIfMany()?.let {
append(
when (it) {
MangaState.ONGOING -> "2"
MangaState.FINISHED -> "1"
MangaState.ABANDONED -> "3"
else -> ""
},
)
}
}
null -> {
append(listUrl)
append("?page=")
append(page.toString())
append("&sort=last_update")
}
}
}
val doc = webClient.httpGet(url).parseHtml()
val lastPage =
doc.selectLast(".pagination a")?.attr("href")?.substringAfterLast("page=")?.substringBeforeLast("&artist")
?.toInt() ?: 1
if (lastPage < page) {
return emptyList()
}
return doc.select("div.thumb-item-flow").map { div ->
val href = "/" + div.selectFirstOrThrow("div.series-title a").attrAsRelativeUrl("href")
Manga(
id = generateUid(href),
url = href,
publicUrl = href.toAbsoluteUrl(div.host ?: domain),
coverUrl = div.selectFirstOrThrow("div.img-in-ratio").attr("data-bg").toAbsoluteUrl(domain),
title = div.selectFirstOrThrow("div.series-title").text().orEmpty(),
altTitle = null,
rating = RATING_UNKNOWN,
tags = emptySet(),
author = null,
state = null,
source = source,
isNsfw = isNsfwSource,
)
}
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = ("/" + chapter.url).toAbsoluteUrl(domain)
val doc = webClient.httpGet(fullUrl).parseHtml()
return doc.select(selectPage).map { img ->
val url = ("/proxy.php?link=" + img.src()).toRelativeUrl(domain)
MangaPage(
id = generateUid(url),
url = url,
preview = null,
source = source,
)
}
}
}

85
src/main/kotlin/org/koitharu/kotatsu/parsers/site/fmreader/ja/Klz9.kt
View File

@@ -1,85 +0,0 @@
package org.koitharu.kotatsu.parsers.site.fmreader.ja
import org.jsoup.nodes.Document
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.site.fmreader.FmreaderParser
import org.koitharu.kotatsu.parsers.util.*
import java.text.SimpleDateFormat
@MangaSourceParser("KLZ9", "Klz9", "ja")
internal class Klz9(context: MangaLoaderContext) :
FmreaderParser(context, MangaSource.KLZ9, "klz9.com") {
override val selectDesc = "div.row:contains(Description)"
override val selectState = "ul.manga-info li:contains(Status) a"
override val selectAlt = "ul.manga-info li:contains(Other name (s))"
override val selectTag = "ul.manga-info li:contains(Genre(s)) a"
override val selectChapter = "tr"
override val selectDate = "td i"
override val selectPage = "img"
override val selectBodyTag = "div.panel-body a"
override fun parseMangaList(doc: Document): List<Manga> {
return doc.select("div.thumb-item-flow").map { div ->
val href = "/" + div.selectFirstOrThrow("a").attrAsRelativeUrl("href")
Manga(
id = generateUid(href),
url = href,
publicUrl = href.toAbsoluteUrl(div.host ?: domain),
coverUrl = div.selectFirstOrThrow("div.img-in-ratio").attr("style").substringAfter("('")
.substringBeforeLast("')"),
title = div.selectFirstOrThrow("div.series-title").text().orEmpty(),
altTitle = null,
rating = RATING_UNKNOWN,
tags = emptySet(),
author = null,
state = null,
source = source,
isNsfw = isNsfwSource,
)
}
}
override suspend fun getChapters(doc: Document): List<MangaChapter> {
val slug = doc.selectFirstOrThrow("div.h0rating").attr("slug")
val docLoad =
webClient.httpGet("https://$domain/app/manga/controllers/cont.listChapter.php?slug=$slug").parseHtml()
val dateFormat = SimpleDateFormat(datePattern, sourceLocale)
return docLoad.body().select(selectChapter).mapChapters(reversed = true) { i, a ->
val href = "/" + a.selectFirstOrThrow("a.chapter").attrAsRelativeUrl("href")
val dateText = a.selectFirst(selectDate)?.text()
MangaChapter(
id = generateUid(href),
name = a.selectFirstOrThrow("a").text(),
number = i + 1,
url = href,
uploadDate = parseChapterDate(
dateFormat,
dateText,
),
source = source,
scanlator = null,
branch = null,
)
}
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(domain)
val doc = webClient.httpGet(fullUrl).parseHtml()
val cid = doc.selectFirstOrThrow("#chapter").attr("value")
val docLoad = webClient.httpGet("https://$domain/app/manga/controllers/cont.listImg.php?cid=$cid").parseHtml()
return docLoad.select(selectPage).map { img ->
val url = img.src()?.toRelativeUrl(domain) ?: img.parseFailed("Image src not found")
MangaPage(
id = generateUid(url),
url = url,
preview = null,
source = source,
)
}
}
}

57
src/main/kotlin/org/koitharu/kotatsu/parsers/site/fmreader/ja/WeLoveManga.kt
View File

@@ -1,57 +0,0 @@
package org.koitharu.kotatsu.parsers.site.fmreader.ja
import org.jsoup.nodes.Document
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.model.MangaChapter
import org.koitharu.kotatsu.parsers.model.MangaPage
import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.site.fmreader.FmreaderParser
import org.koitharu.kotatsu.parsers.util.*
import java.text.SimpleDateFormat
@MangaSourceParser("WELOVEMANGA", "WeLoveManga", "ja")
internal class WeLoveManga(context: MangaLoaderContext) :
FmreaderParser(context, MangaSource.WELOVEMANGA, "welovemanga.one") {
override suspend fun getChapters(doc: Document): List<MangaChapter> {
val mid = doc.selectFirstOrThrow("div.cmt input").attr("value")
val docLoad =
webClient.httpGet("https://$domain/app/manga/controllers/cont.Listchapter.php?mid=$mid").parseHtml()
val dateFormat = SimpleDateFormat(datePattern, sourceLocale)
return docLoad.body().select(selectChapter).mapChapters(reversed = true) { i, a ->
val href = a.selectFirstOrThrow("a").attrAsRelativeUrl("href")
val dateText = a.selectFirst(selectDate)?.text()
MangaChapter(
id = generateUid(href),
name = a.selectFirstOrThrow("a").text(),
number = i + 1,
url = href,
uploadDate = parseChapterDate(
dateFormat,
dateText,
),
source = source,
scanlator = null,
branch = null,
)
}
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(domain)
val doc = webClient.httpGet(fullUrl).parseHtml()
val cid = doc.selectFirstOrThrow("#chapter").attr("value")
val docLoad = webClient.httpGet("https://$domain/app/manga/controllers/cont.listImg.php?cid=$cid").parseHtml()
return docLoad.select("img").map { img ->
val url = img.src()?.toRelativeUrl(domain) ?: img.parseFailed("Image src not found")
MangaPage(
id = generateUid(url),
url = url,
preview = null,
source = source,
)
}
}
}

10
src/main/kotlin/org/koitharu/kotatsu/parsers/site/fmreader/ja/Weloma.kt
View File

@@ -1,10 +0,0 @@
package org.koitharu.kotatsu.parsers.site.fmreader.ja
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.site.fmreader.FmreaderParser
@MangaSourceParser("WELOMA", "Weloma", "ja")
internal class Weloma(context: MangaLoaderContext) :
FmreaderParser(context, MangaSource.WELOMA, "weloma.art")

191
src/main/kotlin/org/koitharu/kotatsu/parsers/site/foolslide/FoolSlideParser.kt
View File

@@ -1,191 +0,0 @@
package org.koitharu.kotatsu.parsers.site.foolslide
import kotlinx.coroutines.coroutineScope
import org.json.JSONArray
import org.jsoup.nodes.Document
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import java.text.SimpleDateFormat
import java.util.*
internal abstract class FoolSlideParser(
context: MangaLoaderContext,
source: MangaSource,
domain: String,
pageSize: Int = 25,
) : PagedMangaParser(context, source, pageSize) {
override val configKeyDomain = ConfigKey.Domain(domain)
override val availableSortOrders: Set<SortOrder> = EnumSet.of(SortOrder.ALPHABETICAL)
protected open val listUrl = "directory/"
protected open val searchUrl = "search/"
protected open val pagination = true // false if the manga list has no pages
protected open val datePattern = "yyyy.MM.dd"
init {
paginator.firstPage = 1
searchPaginator.firstPage = 1
}
override suspend fun getListPage(page: Int, filter: MangaListFilter?): List<Manga> {
val doc =
when (filter) {
is MangaListFilter.Search -> {
if (page > 1) {
return emptyList()
}
val url = buildString {
append("https://")
append(domain)
append("/")
append(searchUrl)
}
webClient.httpPost(url, "search=${filter.query.urlEncoded()}").parseHtml()
}
is MangaListFilter.Advanced -> {
val url = buildString {
append("https://")
append(domain)
append("/")
append(listUrl)
// For some sites that don't have enough manga and page 2 links to page 1
if (!pagination) {
if (page > 1) {
return emptyList()
}
} else {
append(page.toString())
}
}
webClient.httpGet(url).parseHtml()
}
null -> {
val url = buildString {
append("https://")
append(domain)
append("/")
append(listUrl)
if (!pagination) {
if (page > 1) {
return emptyList()
}
} else {
append(page.toString())
}
}
webClient.httpGet(url).parseHtml()
}
}
return doc.select("div.list div.group").map { div ->
val href = div.selectFirstOrThrow("a").attrAsRelativeUrl("href")
Manga(
id = generateUid(href),
url = href,
publicUrl = href.toAbsoluteUrl(div.host ?: domain),
coverUrl = div.selectFirst("img")?.src().orEmpty(),// in search no img
title = div.selectFirstOrThrow(".title a").text().orEmpty(),
altTitle = null,
rating = RATING_UNKNOWN,
tags = emptySet(),
author = null,
state = null,
source = source,
isNsfw = isNsfwSource,
)
}
}
override suspend fun getAvailableTags(): Set<MangaTag> = emptySet()
protected open val selectInfo = "div.info"
override suspend fun getDetails(manga: Manga): Manga = coroutineScope {
val fullUrl = manga.url.toAbsoluteUrl(domain)
val testAdultPage = webClient.httpGet(fullUrl).parseHtml()
val doc = if (testAdultPage.selectFirst("div.info form") != null) {
webClient.httpPost(fullUrl, "adult=true").parseHtml()
} else {
testAdultPage
}
val chapters = getChapters(doc)
val desc = if (doc.selectFirstOrThrow(selectInfo).html().contains("</b>")) {
doc.selectFirstOrThrow(selectInfo).text().substringAfterLast(": ")
} else {
doc.selectFirstOrThrow(selectInfo).text()
}
val author = if (doc.selectFirstOrThrow(selectInfo).html().contains("</b>")) {
doc.selectFirstOrThrow(selectInfo).text().substringAfter(": ").substringBefore("Art")
} else {
null
}
manga.copy(
coverUrl = doc.selectFirst(".thumbnail img")?.src() ?: manga.coverUrl,
description = desc,
altTitle = null,
author = author,
state = null,
chapters = chapters,
)
}
protected open val selectDate = ".meta_r"
protected open val selectChapter = "div.list div.element"
protected open suspend fun getChapters(doc: Document): List<MangaChapter> {
val dateFormat = SimpleDateFormat(datePattern, sourceLocale)
return doc.body().select(selectChapter).mapChapters(reversed = true) { i, div ->
val a = div.selectFirstOrThrow(".title a")
val href = a.attrAsRelativeUrl("href")
val dateText = div.selectFirstOrThrow(selectDate).text().substringAfter(", ")
MangaChapter(
id = generateUid(href),
name = a.text(),
number = i + 1,
url = href,
uploadDate = if (div.selectFirstOrThrow(selectDate).text().contains(", ")) {
dateFormat.tryParse(dateText)
} else {
0
},
source = source,
scanlator = null,
branch = null,
)
}
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(domain)
val doc = webClient.httpGet(fullUrl).parseHtml()
val script = doc.selectFirstOrThrow("script:containsData(var pages = )")
val images = JSONArray(script.data().substringAfterLast("var pages = ").substringBefore(';'))
val pages = ArrayList<MangaPage>(images.length())
for (i in 0 until images.length()) {
val pageTake = images.getJSONObject(i)
pages.add(
MangaPage(
id = generateUid(pageTake.getString("url")),
url = pageTake.getString("url"),
preview = null,
source = source,
),
)
}
return pages
}
}

10
src/main/kotlin/org/koitharu/kotatsu/parsers/site/foolslide/en/Deathtollscans.kt
View File

@@ -1,10 +0,0 @@
package org.koitharu.kotatsu.parsers.site.foolslide.en
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.site.foolslide.FoolSlideParser
@MangaSourceParser("DEATHTOLLSCANS", "DeathTollScans", "en")
internal class Deathtollscans(context: MangaLoaderContext) :
FoolSlideParser(context, MangaSource.DEATHTOLLSCANS, "reader.deathtollscans.net", 26)

12
src/main/kotlin/org/koitharu/kotatsu/parsers/site/foolslide/en/Mangatellers.kt
View File

@@ -1,12 +0,0 @@
package org.koitharu.kotatsu.parsers.site.foolslide.en
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.site.foolslide.FoolSlideParser
@MangaSourceParser("MANGATELLERS", "Mangatellers", "en")
internal class Mangatellers(context: MangaLoaderContext) :
FoolSlideParser(context, MangaSource.MANGATELLERS, "reader.mangatellers.gr") {
override val pagination = false
}

10
src/main/kotlin/org/koitharu/kotatsu/parsers/site/foolslide/en/ReaderEvilflowers.kt
View File

@@ -1,10 +0,0 @@
package org.koitharu.kotatsu.parsers.site.foolslide.en
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.site.foolslide.FoolSlideParser
@MangaSourceParser("READER_EVILFLOWERS", "Evil Flowers", "en")
internal class ReaderEvilflowers(context: MangaLoaderContext) :
FoolSlideParser(context, MangaSource.READER_EVILFLOWERS, "reader.evilflowers.com")

44
src/main/kotlin/org/koitharu/kotatsu/parsers/site/foolslide/en/Seinagi.kt
View File

@@ -1,44 +0,0 @@
package org.koitharu.kotatsu.parsers.site.foolslide.en
import kotlinx.coroutines.coroutineScope
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.site.foolslide.FoolSlideParser
import org.koitharu.kotatsu.parsers.util.*
@MangaSourceParser("SEINAGI", "Seinagi", "en")
internal class Seinagi(context: MangaLoaderContext) :
FoolSlideParser(context, MangaSource.SEINAGI, "reader.seinagi.org.es") {
override val pagination = false
override suspend fun getDetails(manga: Manga): Manga = coroutineScope {
val fullUrl = manga.url.toAbsoluteUrl(domain)
val testAdultPage = webClient.httpGet(fullUrl).parseHtml()
val doc = if (testAdultPage.selectFirst("div.info form") != null) {
webClient.httpPost(fullUrl, "adult=true").parseHtml()
} else {
testAdultPage
}
val chapters = getChapters(doc)
val desc = if (doc.selectFirstOrThrow(selectInfo).html().contains("Description")) {
doc.selectFirstOrThrow(selectInfo).text().substringAfter("Description: ").substringBefore("Readings")
} else {
doc.selectFirstOrThrow(selectInfo).text()
}
val author = if (doc.selectFirstOrThrow(selectInfo).html().contains("Author")) {
doc.selectFirstOrThrow(selectInfo).text().substringAfter("Author: ").substringBefore("Art")
} else {
null
}
manga.copy(
coverUrl = doc.selectFirst(".thumbnail img")?.src() ?: manga.coverUrl,
description = desc,
altTitle = null,
author = author,
state = null,
chapters = chapters,
)
}
}

13
src/main/kotlin/org/koitharu/kotatsu/parsers/site/foolslide/es/MenudoFansub.kt
View File

@@ -1,13 +0,0 @@
package org.koitharu.kotatsu.parsers.site.foolslide.es
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.site.foolslide.FoolSlideParser
@MangaSourceParser("MENUDO_FANSUB", "Menudo Fansub", "es")
internal class MenudoFansub(context: MangaLoaderContext) :
FoolSlideParser(context, MangaSource.MENUDO_FANSUB, "www.menudo-fansub.com", 25) {
override val searchUrl = "slide/search/"
override val listUrl = "slide/directory/"
}

Some files were not shown because too many files have changed in this diff Show More