1 Commits

Author SHA1 Message Date
9952f8f00e [MangaTrend] New source (close #65) 2022-09-25 17:31:06 +03:00
892 changed files with 3481 additions and 30063 deletions

View File

@@ -6,8 +6,8 @@ end_of_line = lf
indent_style = tab
max_line_length = 120
tab_width = 4
insert_final_newline = true
disabled_rules = no-wildcard-imports, no-unused-imports
insert_final_newline = false
disabled_rules=no-wildcard-imports,no-unused-imports
[{*.kt,*.kts}]
ij_kotlin_allow_trailing_comma = true

View File

@@ -1,65 +1,65 @@
name: 🐞 Issue report
description: Report a source issue with a source
labels: [ bug ]
description: Report a source issue in Kotatsu
labels: [bug]
body:
- type: input
id: source
attributes:
label: Source information
description: |
You can find the source name in navigation drawer.
placeholder: |
Example: "MangaDex"
validations:
required: true
- type: input
id: source
attributes:
label: Source information
description: |
You can find the source name in navigation drawer.
placeholder: |
Example: "MangaDex"
validations:
required: true
- type: textarea
id: reproduce-steps
attributes:
label: Steps to reproduce
description: Provide an example of the issue.
placeholder: |
Example:
1. First step
2. Second step
3. Issue here
validations:
required: false
- type: textarea
id: reproduce-steps
attributes:
label: Steps to reproduce
description: Provide an example of the issue.
placeholder: |
Example:
1. First step
2. Second step
3. Issue here
validations:
required: false
- type: input
id: kotatsu-version
attributes:
label: Kotatsu version
description: |
You can find your Kotatsu version in **Settings → About**.
placeholder: |
Example: "3.3"
validations:
required: true
- type: input
id: kotatsu-version
attributes:
label: Kotatsu version
description: |
You can find your Kotatsu version in **Settings → About**.
placeholder: |
Example: "3.3"
validations:
required: true
- type: input
id: android-version
attributes:
label: Android version
description: |
You can find this somewhere in your Android settings.
placeholder: |
Example: "Android 12"
validations:
required: false
- type: input
id: android-version
attributes:
label: Android version
description: |
You can find this somewhere in your Android settings.
placeholder: |
Example: "Android 12"
validations:
required: false
- type: textarea
id: other-details
attributes:
label: Other details
placeholder: |
Additional details and attachments.
- type: textarea
id: other-details
attributes:
label: Other details
placeholder: |
Additional details and attachments.
- type: checkboxes
id: acknowledgements
attributes:
label: Acknowledgements
options:
- label: I have searched the existing issues and this is a new ticket, **NOT** a duplicate or related to another open issue.
required: true
- type: checkboxes
id: acknowledgements
attributes:
label: Acknowledgements
options:
- label: I have searched the existing issues and this is a new ticket, **NOT** a duplicate or related to another open issue.
required: true

View File

@@ -1,53 +1,50 @@
name: 🌐 Source request
description: Suggest a new source for Kotatsu
labels: [ source request ]
labels: [source request]
body:
- type: markdown
attributes:
value: Please specify source **name** and **language** in the issue title
- type: input
id: name
attributes:
label: Source name
placeholder: |
Example: "Example Scans"
validations:
required: true
- type: input
id: name
attributes:
label: Source name
placeholder: |
Example: "Example Scans"
validations:
required: true
- type: input
id: link
attributes:
label: Source link
placeholder: |
Example: "https://example.org"
validations:
required: true
- type: input
id: link
attributes:
label: Source link
placeholder: |
Example: "https://example.org"
validations:
required: true
- type: input
id: language
attributes:
label: Language
placeholder: |
Example: "English"
validations:
required: true
- type: input
id: language
attributes:
label: Language
placeholder: |
Example: "English"
validations:
required: true
- type: textarea
id: other-details
attributes:
label: Other details
placeholder: |
Additional details and attachments.
- type: textarea
id: other-details
attributes:
label: Other details
placeholder: |
Additional details and attachments.
- type: checkboxes
id: acknowledgements
attributes:
label: Acknowledgements
options:
- label: I have searched the existing issues and this is a new ticket, **NOT** a duplicate or related to another open issue.
required: true
- label: I have checked that the source does not already exist on the app.
required: true
- label: I have checked that the source does not already exist by searching the [GitHub repository](https://github.com/KotatsuApp/kotatsu-parsers) and verified it does not appear in the code base.
required: true
- type: checkboxes
id: acknowledgements
attributes:
label: Acknowledgements
options:
- label: I have searched the existing issues and this is a new ticket, **NOT** a duplicate or related to another open issue.
required: true
- label: I have checked that the source does not already exist on the app.
required: true
- label: I have checked that the source does not already exist by searching the [GitHub repository](https://github.com/KotatsuApp/kotatsu-parsers) and verified it does not appear in the code base.
required: true

View File

@@ -1,27 +1,27 @@
name: Parsers test
on:
workflow_dispatch:
pull_request:
paths:
- 'src/main/kotlin/org/koitharu/kotatsu/parsers/site/**'
workflow_dispatch:
pull_request:
paths:
- 'src/main/kotlin/org/koitharu/kotatsu/parsers/site/*'
permissions:
contents: read
contents: read
jobs:
build-and-test:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/setup-java@v3
with:
java-version: '11'
distribution: 'temurin'
cache: 'gradle'
- run: ./gradlew :test --tests "org.koitharu.kotatsu.parsers.MangaParserTest" || true
- run: ./gradlew generateTestsReport
- uses: actions/upload-artifact@v3
with:
name: Report
path: build/test-results-html/TEST-org.koitharu.kotatsu.parsers.MangaParserTest.htm
build-and-test:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/setup-java@v3
with:
java-version: '11'
distribution: 'temurin'
cache: 'gradle'
- run: ./gradlew :test --tests "org.koitharu.kotatsu.parsers.MangaParserTest" || true
- run: ./gradlew generateTestsReport
- uses: actions/upload-artifact@v3
with:
name: Report
path: build/test-results-html/TEST-org.koitharu.kotatsu.parsers.MangaParserTest.htm

2
.idea/kotlinc.xml generated
View File

@@ -1,6 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="KotlinJpsPluginSettings">
<option name="version" value="1.9.0" />
<option name="version" value="1.6.21" />
</component>
</project>

View File

@@ -35,11 +35,11 @@ If it does not contain any documentation about
API, [explore network requests](https://firefox-source-docs.mozilla.org/devtools-user/):
some websites use ajax.
- [Example](https://github.com/KotatsuApp/kotatsu-parsers/blob/master/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ru/DesuMeParser.kt)
- [Example](https://github.com/KotatsuApp/kotatsu-parsers/blob/master/src/main/kotlin/org/koitharu/kotatsu/parsers/site/DesuMeParser.kt)
of Json API usage.
- [Example](https://github.com/KotatsuApp/kotatsu-parsers/blob/master/src/main/kotlin/org/koitharu/kotatsu/parsers/site/be/AnibelParser.kt)
- [Example](https://github.com/KotatsuApp/kotatsu-parsers/blob/master/src/main/kotlin/org/koitharu/kotatsu/parsers/site/AnibelParser.kt)
of GraphQL API usage
- [Example](https://github.com/KotatsuApp/kotatsu-parsers/blob/master/src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/MangaTownParser.kt)
- [Example](https://github.com/KotatsuApp/kotatsu-parsers/blob/master/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaTownParser.kt)
of pure HTML parsing.
If website is based on some engine it is rationally to use common base class for this one (for example, Madara wordress
@@ -59,13 +59,13 @@ All functions in `MangaParser` class are documented. Pay attention to some pecul
- `sortOrders` set should not be empty. If your source is not support sorting, specify one most relevance value.
- If you cannot obtain direct links to pages images inside `getPages` method, it is ok to use an intermediate url
as `Page.url` and fetch a direct link at `getPageUrl` function.
- `getFaviconUrl` function is deprecated, but Kotatsu application before v4 still use it, so it is recommended to
override it too.
- You can use _asserts_ to check some optional fields. For example. `Manga.author` field is not required, but if your
source provide such information, add `assert(it != null)`. This will not have any effect on production but help to
find issues during unit testing.
- If your source website (or it's api) uses pages for pagination instead of offset you should extend `PagedMangaParser`
instead of `MangaParser`.
- Your parser may also implement the `Interceptor` interface for additional manipulation of all network requests and/or
responses, including image loading.
## Development process
@@ -87,5 +87,4 @@ It is recommended to run unit tests before submitting a PR.
## Help
If you need a help or have some questions, ask a community in our [Telegram chat](https://t.me/kotatsuapp)
or [Discord server](https://discord.gg/NNJ5RgVBC5).
If you need a help or have some questions, ask a community in our [Discord server](https://discord.gg/NNJ5RgVBC5).

View File

@@ -1,12 +1,12 @@
# Kotatsu parsers
This library provides manga sources.
Library that provides manga sources.
[![](https://jitpack.io/v/KotatsuApp/kotatsu-parsers.svg)](https://jitpack.io/#KotatsuApp/kotatsu-parsers) ![Kotlin](https://img.shields.io/github/languages/top/KotatsuApp/kotatsu-parsers) ![License](https://img.shields.io/github/license/KotatsuApp/Kotatsu) [![Telegram](https://img.shields.io/badge/chat-telegram-60ACFF)](https://t.me/kotatsuapp) [![Discord](https://img.shields.io/discord/898363402467045416?color=5865f2&label=discord)](https://discord.gg/NNJ5RgVBC5)
[![](https://jitpack.io/v/KotatsuApp/kotatsu-parsers.svg)](https://jitpack.io/#KotatsuApp/kotatsu-parsers) ![Kotlin](https://img.shields.io/github/languages/top/KotatsuApp/kotatsu-parsers) ![License](https://img.shields.io/github/license/KotatsuApp/Kotatsu) [![Discord](https://img.shields.io/discord/898363402467045416?color=5865f2&label=discord)](https://discord.gg/NNJ5RgVBC5)
## Usage
1. Add it to your root build.gradle at the end of repositories:
1. Add it in your root build.gradle at the end of repositories:
```groovy
allprojects {
@@ -35,19 +35,18 @@ This library provides manga sources.
}
```
Versions are available on [JitPack](https://jitpack.io/#KotatsuApp/kotatsu-parsers)
See for versions at [JitPack](https://jitpack.io/#KotatsuApp/kotatsu-parsers)
3. Usage in code
```kotlin
val parser = mangaLoaderContext.newParserInstance(MangaSource.MANGADEX)
val parser = MangaSource.MANGADEX.newParser(mangaLoaderContext)
```
`mangaLoaderContext` is an implementation of the `MangaLoaderContext` class.
See examples
of [Android](https://github.com/KotatsuApp/Kotatsu/blob/devel/app/src/main/java/org/koitharu/kotatsu/core/parser/MangaLoaderContextImpl.kt)
See [Android](https://github.com/KotatsuApp/Kotatsu/blob/devel/app/src/main/java/org/koitharu/kotatsu/core/parser/MangaLoaderContextImpl.kt)
and [Non-Android](https://github.com/KotatsuApp/kotatsu-dl/blob/master/src/main/kotlin/org/koitharu/kotatsu_dl/env/MangaLoaderContextImpl.kt)
implementation.
implementation examples.
Note that the `MangaSource.LOCAL` and `MangaSource.DUMMY` parsers cannot be instantiated.
@@ -57,5 +56,5 @@ See [CONTRIBUTING.md](./CONTRIBUTING.md) for the guidelines.
## DMCA disclaimer
The developers of this application have no affiliation with the content available in the app. It is collected from
sources freely available through any web browser.
The developers of this application does not have any affiliation with the content available in the app. It is collecting
from the sources freely available through any web browser.

View File

@@ -2,8 +2,8 @@ import tasks.ReportGenerateTask
plugins {
id 'java-library'
id 'org.jetbrains.kotlin.jvm' version '1.9.20'
id 'com.google.devtools.ksp' version '1.9.20-1.0.14'
id 'org.jetbrains.kotlin.jvm'
id 'com.google.devtools.ksp'
id 'maven-publish'
}
@@ -16,27 +16,28 @@ test {
compileKotlin {
kotlinOptions {
jvmTarget = '1.8'
freeCompilerArgs += [
'-opt-in=kotlin.RequiresOptIn',
'-opt-in=kotlin.contracts.ExperimentalContracts',
'-opt-in=kotlinx.coroutines.ExperimentalCoroutinesApi',
'-opt-in=org.koitharu.kotatsu.parsers.InternalParsersApi',
'-opt-in=kotlin.RequiresOptIn',
'-opt-in=kotlin.contracts.ExperimentalContracts',
'-opt-in=kotlinx.coroutines.ExperimentalCoroutinesApi',
'-opt-in=org.koitharu.kotatsu.parsers.InternalParsersApi',
]
}
}
compileTestKotlin {
kotlinOptions {
jvmTarget = '1.8'
freeCompilerArgs += [
'-opt-in=kotlin.RequiresOptIn',
'-opt-in=kotlinx.coroutines.ExperimentalCoroutinesApi',
'-opt-in=org.koitharu.kotatsu.parsers.InternalParsersApi',
'-opt-in=kotlin.RequiresOptIn',
'-opt-in=kotlinx.coroutines.ExperimentalCoroutinesApi',
'-opt-in=org.koitharu.kotatsu.parsers.InternalParsersApi',
]
}
}
kotlin {
jvmToolchain(11)
sourceSets {
main.kotlin.srcDirs += 'build/generated/ksp/main/kotlin'
}
@@ -53,20 +54,20 @@ afterEvaluate {
}
dependencies {
implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-core:1.7.3'
implementation 'com.squareup.okhttp3:okhttp:4.12.0'
implementation 'com.squareup.okio:okio:3.6.0'
api 'org.jsoup:jsoup:1.16.2'
implementation 'org.json:json:20231013'
implementation 'androidx.collection:collection-ktx:1.3.0'
implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-core:1.6.4'
implementation 'com.squareup.okhttp3:okhttp:4.10.0'
implementation 'com.squareup.okio:okio:3.2.0'
api 'org.jsoup:jsoup:1.15.2'
implementation 'org.json:json:20220320'
implementation 'androidx.collection:collection-ktx:1.2.0'
ksp project(':kotatsu-parsers-ksp')
testImplementation 'org.junit.jupiter:junit-jupiter-api:5.10.0'
testImplementation 'org.junit.jupiter:junit-jupiter-engine:5.10.0'
testImplementation 'org.junit.jupiter:junit-jupiter-params:5.10.0'
testImplementation 'org.jetbrains.kotlinx:kotlinx-coroutines-test:1.7.3'
testImplementation 'org.junit.jupiter:junit-jupiter-api:5.9.0'
testImplementation 'org.junit.jupiter:junit-jupiter-engine:5.9.0'
testImplementation 'org.junit.jupiter:junit-jupiter-params:5.9.0'
testImplementation 'org.jetbrains.kotlinx:kotlinx-coroutines-test:1.6.4'
testImplementation 'io.webfolder:quickjs:1.1.0'
}
tasks.register('generateTestsReport', ReportGenerateTask)
task generateTestsReport(type: ReportGenerateTask)

View File

@@ -1,5 +1,5 @@
plugins {
id 'org.jetbrains.kotlin.jvm' version '1.8.22'
id('org.jetbrains.kotlin.jvm') version '1.6.21'
}
repositories {
mavenCentral()
@@ -8,6 +8,6 @@ repositories {
dependencies {
implementation gradleApi()
implementation 'org.simpleframework:simple-xml:2.7.1'
implementation 'com.soywiz.korlibs.korte:korte-jvm:4.0.9'
implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-core:1.7.3'
}
implementation 'com.soywiz.korlibs.korte:korte-jvm:3.0.0-Beta5'
implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-core:1.6.3'
}

View File

@@ -1,6 +1,6 @@
package tasks
import korlibs.template.Template
import com.soywiz.korte.Template
import kotlinx.coroutines.runBlocking
import org.gradle.api.DefaultTask
import org.gradle.api.tasks.TaskAction
@@ -42,13 +42,10 @@ open class ReportGenerateTask : DefaultTask() {
val results = LinkedHashMap<String, LinkedHashMap<String, TestCase>>()
val tests = LinkedHashSet<String>()
for (case in testSuite.testCases) {
if (!case.isValid()) {
continue
}
tests.add(case.testName)
val map = results.getOrPut(case.source) { LinkedHashMap() }
val oldValue = map.put(case.testName, case)
check(oldValue == null) { "Check failed: $oldValue" }
check(oldValue == null)
}
val failPercent = (testSuite.failures.toDouble() / testSuite.tests * 100.0).roundToInt()

View File

@@ -19,16 +19,14 @@ class TestCase {
var failure: Failure? = null
val index by lazy {
name.split('|').getOrNull(0)?.toIntOrNull() ?: 0
name.split('|')[0].toInt()
}
val testName by lazy {
name.split('|').getOrNull(1).orEmpty()
name.split('|')[1]
}
val source by lazy {
name.split('|').getOrNull(2).orEmpty()
name.split('|')[2]
}
fun isValid() = name.count { it == '|' } == 2
}
}

View File

@@ -2,15 +2,13 @@
<html lang="en">
<head>
<meta charset="utf-8">
<meta content="width=device-width, initial-scale=1" name="viewport">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>{{ testSuite.name }}</title>
<!-- CSS only -->
<link crossorigin="anonymous" href="https://cdn.jsdelivr.net/npm/bootstrap@5.2.2/dist/css/bootstrap.min.css"
integrity="sha384-Zenh87qX5JnK2Jl0vWa8Ck2rdkQ2Bzep5IDxbcnCeuOxjzrPF/et3URy9Bv1WTRi" rel="stylesheet">
<!-- JavaScript Bundle with Popper -->
<script crossorigin="anonymous"
integrity="sha384-OERcA2EqjJCMA+/3y+gxIOqMEjwtxJY7qPCqsdltbNJuaOe923+mo//f6V8Qbsw3"
src="https://cdn.jsdelivr.net/npm/bootstrap@5.2.2/dist/js/bootstrap.bundle.min.js"></script>
<link href="https://cdn.jsdelivr.net/npm/bootstrap@5.2.0-beta1/dist/css/bootstrap.min.css" rel="stylesheet"
integrity="sha384-0evHe/X+R7YkIZDRvuzKMRqM+OrBnVFBL6DOitfPri4tjfHxaWutUpFmBp4vmVor" crossorigin="anonymous">
<script src="https://cdn.jsdelivr.net/npm/bootstrap@5.2.0-beta1/dist/js/bootstrap.bundle.min.js"
integrity="sha384-pprn3073KE6tl6bjs2QrFaJGz5/SUsLqktiwsUTF55Jfv3qYSDhgCecCxMW52nD2"
crossorigin="anonymous"></script>
<script src="https://cdn.jsdelivr.net/npm/feather-icons/dist/feather.min.js"></script>
</head>
<body class="py-4">
@@ -27,80 +25,67 @@
{{ testSuite.errors }} ({{ error_percent }}%)
</div>
</div>
<table class="table table-hover">
<thead class="sticky-top bg-body">
<tr>
<th scope="col">Source</th>
{% for test in tests %}
<th class="text-center" scope="col" style="min-width: 5em;">{{ test }}</th>
{% endfor %}
</tr>
</thead>
{% for name, cases in results %}
<tr>
<th scope="row">{{ name }}</th>
{% for test in tests %}
{% set case = cases[test] %}
{% if case.failure == null %}
<td class="table-success text-center">
<i data-feather="check"></i>
</td>
{% else %}
{% if case.failure.type == 'java.lang.AssertionError' %}
<td class="table-warning text-center" data-bs-target="#failure_{{ case.hashCode }}"
data-bs-toggle="modal" style="cursor: pointer;">
<i data-feather="alert-triangle"></i>
</td>
{% elseif case.failure.type == 'java.net.SocketTimeoutException' or case.failure.type ==
'java.net.UnknownHostException' %}
<td class="table-secondary text-center" data-bs-target="#failure_{{ case.hashCode }}"
data-bs-toggle="modal" style="cursor: pointer;">
<i data-feather="power"></i>
</td>
{% elseif case.failure.type == 'org.koitharu.kotatsu.parsers.CloudFlareProtectedException' %}
<td class="table-secondary text-center" data-bs-target="#failure_{{ case.hashCode }}"
data-bs-toggle="modal" style="cursor: pointer;">
<i data-feather="shield"></i>
</td>
{% elseif case.failure.type == 'org.koitharu.kotatsu.parsers.exception.AuthRequiredException' %}
<td class="table-secondary text-center" data-bs-target="#failure_{{ case.hashCode }}"
data-bs-toggle="modal" style="cursor: pointer;">
<i data-feather="user-x"></i>
</td>
{% else %}
<td class="table-danger text-center" data-bs-target="#failure_{{ case.hashCode }}"
data-bs-toggle="modal" style="cursor: pointer;">
<i data-feather="x"></i>
</td>
{% endif %}
<!--suppress HtmlUnknownTag -->
<div class="modal fade" id="failure_{{ case.hashCode }}" tabindex="-1">
<div class="modal-dialog modal-lg modal-dialog-scrollable">
<div class="modal-content">
<div class="modal-header">
<h5 class="modal-title">{{ case.testName }} failed</h5>
<button aria-label="Close" class="btn-close" data-bs-dismiss="modal"
type="button"></button>
</div>
<div class="modal-body font-monospace lh-sm bg-light" style="font-size: 0.7em;">
{{ case.failure.textHtml()|raw }}
</div>
<div class="modal-footer">
<button class="btn btn-secondary" data-bs-dismiss="modal" type="button">Close</button>
<div class="table-responsive mt-4">
<table class="table table-hover">
<thead>
<tr>
<th scope="col">Source</th>
{% for test in tests %}
<th scope="col" style="min-width: 5em;">{{ test }}</th>
{% endfor %}
</tr>
</thead>
{% for name, cases in results %}
<tr>
<th scope="row">{{ name }}</th>
{% for test in tests %}
{% set case = cases[test] %}
{% if case.failure == null %}
<td class="table-success text-center">
<i data-feather="check"></i>
</td>
{% else %}
{% if case.failure.type == 'java.lang.AssertionError' %}
<td class="table-warning text-center" style="cursor: pointer;"
data-bs-toggle="modal" data-bs-target="#failure_{{ case.hashCode }}">
<i data-feather="alert-triangle"></i>
</td>
{% else %}
<td class="table-danger text-center" style="cursor: pointer;"
data-bs-toggle="modal" data-bs-target="#failure_{{ case.hashCode }}">
<i data-feather="x"></i>
</td>
{% endif %}
<!--suppress HtmlUnknownTag -->
<div class="modal fade" id="failure_{{ case.hashCode }}" tabindex="-1">
<div class="modal-dialog modal-lg modal-dialog-scrollable">
<div class="modal-content">
<div class="modal-header">
<h5 class="modal-title">{{ case.testName }} failed</h5>
<button type="button" class="btn-close" data-bs-dismiss="modal"
aria-label="Close"></button>
</div>
<div class="modal-body font-monospace lh-sm bg-light" style="font-size: 0.7em;">
{{ case.failure.textHtml()|raw }}
</div>
<div class="modal-footer">
<button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Close</button>
</div>
</div>
</div>
</div>
</div>
{% endif %}
{% endif %}
{% endfor %}
</tr>
{% endfor %}
</tr>
{% endfor %}
</table>
</table>
</div>
</div>
<script>
feather.replace()
</script>
</body>
</html>
</html>

View File

@@ -1,2 +1 @@
kotlin.code.style=official
org.gradle.jvmargs=-Xmx4g -XX:MaxMetaspaceSize=4096m

View File

@@ -1,5 +1,5 @@
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-8.2-bin.zip
distributionUrl=https\://services.gradle.org/distributions/gradle-7.3.3-bin.zip
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists

View File

@@ -3,5 +3,5 @@ plugins {
}
dependencies {
implementation 'com.google.devtools.ksp:symbol-processing-api:1.9.20-1.0.14'
}
implementation 'com.google.devtools.ksp:symbol-processing-api:1.6.21-1.0.5'
}

View File

@@ -70,13 +70,10 @@ class ParserProcessor(
import org.koitharu.kotatsu.parsers.model.MangaSource
@Suppress("DEPRECATION")
@InternalParsersApi
@Deprecated("", replaceWith = ReplaceWith("context.newParserInstance(this)"))
fun MangaSource.newParser(context: MangaLoaderContext): MangaParser = when (this) {
""".trimIndent(),
)
//language=kotlin
sourcesWriter?.write(
"""
package org.koitharu.kotatsu.parsers.model
@@ -84,9 +81,8 @@ class ParserProcessor(
enum class MangaSource(
val title: String,
val locale: String?,
val contentType: ContentType,
) {
LOCAL("Local", null, ContentType.OTHER),
LOCAL("Local", null),
""".trimIndent(),
)
@@ -109,7 +105,7 @@ class ParserProcessor(
)
sourcesWriter?.write(
"""
DUMMY("Dummy", null, ContentType.OTHER),
DUMMY("Dummy", null),
;
}
""".trimIndent(),
@@ -121,8 +117,6 @@ class ParserProcessor(
private val factoryWriter: Writer?,
) : KSVisitorVoid() {
private val titles = HashMap<String, String>()
override fun visitClassDeclaration(classDeclaration: KSClassDeclaration, data: Unit) {
if (classDeclaration.classKind != ClassKind.CLASS || classDeclaration.isAbstract()) {
logger.error("Only non-abstract can be annotated with @MangaSourceParser", classDeclaration)
@@ -132,7 +126,6 @@ class ParserProcessor(
val name = annotation.arguments.single { it.name?.asString() == "name" }.value as String
val title = annotation.arguments.single { it.name?.asString() == "title" }.value as String
val locale = annotation.arguments.single { it.name?.asString() == "locale" }.value as String
val type = annotation.arguments.single { it.name?.asString() == "type" }.value
val localeString = if (locale.isEmpty()) "null" else "\"$locale\""
val localeObj = if (locale.isEmpty()) null else Locale(locale)
val localeTitle = localeObj?.getDisplayLanguage(localeObj)
@@ -151,13 +144,7 @@ class ParserProcessor(
classDeclaration,
)
}
val className = checkNotNull(classDeclaration.qualifiedName?.asString()) { "Class name is null" }
val prevTitleClass = titles.put(title, className)
if (prevTitleClass != null) {
logger.warn("Source title duplication: \"$title\" is assigned to both $prevTitleClass and $className")
}
val className = classDeclaration.qualifiedName?.asString()
factoryWriter?.write("\tMangaSource.$name -> $className(context)\n")
val deprecationString = if (deprecation != null) {
val reason = deprecation.arguments
@@ -165,7 +152,7 @@ class ParserProcessor(
"@Deprecated(\"$reason\") "
} else ""
val localeComment = localeTitle?.toTitleCase(localeObj)?.let { " /* $it */" }.orEmpty()
sourcesWriter?.write("\t$deprecationString$name(\"$title\", $localeString$localeComment, ContentType.$type),\n")
sourcesWriter?.write("\t$deprecationString$name(\"$title\", $localeString$localeComment),\n")
}
}
}
}

View File

@@ -1,16 +1,19 @@
pluginManagement {
plugins {
id 'com.google.devtools.ksp' version '1.6.21-1.0.5'
id 'org.jetbrains.kotlin.jvm' version '1.6.21'
}
repositories {
google()
mavenCentral()
gradlePluginPortal()
google()
}
}
dependencyResolutionManagement {
repositoriesMode.set(RepositoriesMode.FAIL_ON_PROJECT_REPOS)
repositories {
google()
mavenCentral()
gradlePluginPortal()
}
}

View File

@@ -11,4 +11,4 @@ package org.koitharu.kotatsu.parsers
@SinceKotlin("1.3")
@RequiresOptIn
@MustBeDocumented
annotation class InternalParsersApi
annotation class InternalParsersApi()

View File

@@ -1,19 +1,136 @@
package org.koitharu.kotatsu.parsers
import okhttp3.CookieJar
import okhttp3.OkHttpClient
import okhttp3.*
import okhttp3.HttpUrl.Companion.toHttpUrl
import okhttp3.MediaType.Companion.toMediaType
import okhttp3.RequestBody.Companion.toRequestBody
import org.json.JSONObject
import org.jsoup.HttpStatusException
import org.koitharu.kotatsu.parsers.config.MangaSourceConfig
import org.koitharu.kotatsu.parsers.exception.GraphQLException
import org.koitharu.kotatsu.parsers.exception.NotFoundException
import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.util.await
import org.koitharu.kotatsu.parsers.util.parseJson
import java.util.*
abstract class MangaLoaderContext {
abstract val httpClient: OkHttpClient
protected abstract val httpClient: OkHttpClient
abstract val cookieJar: CookieJar
@Suppress("DEPRECATION")
fun newParserInstance(source: MangaSource): MangaParser = source.newParser(this)
/**
* Do a GET http request to specific url
* @param url
* @param headers an additional headers for request, may be null
*/
suspend fun httpGet(url: HttpUrl, headers: Headers? = null): Response {
val request = Request.Builder()
.get()
.url(url)
if (headers != null) {
request.headers(headers)
}
return httpClient.newCall(request.build()).await().ensureSuccess()
}
suspend fun httpGet(url: String, headers: Headers? = null): Response {
return httpGet(url.toHttpUrl(), headers)
}
/**
* Do a HEAD http request to specific url
* @param url
* @param headers an additional headers for request, may be null
*/
suspend fun httpHead(url: String, headers: Headers? = null): Response {
val request = Request.Builder()
.head()
.url(url)
if (headers != null) {
request.headers(headers)
}
return httpClient.newCall(request.build()).await().ensureSuccess()
}
/**
* Do a POST http request to specific url with `multipart/form-data` payload
* @param url
* @param form payload as key=>value map
* @param headers an additional headers for request, may be null
*/
suspend fun httpPost(
url: String,
form: Map<String, String>,
headers: Headers? = null,
): Response {
val body = FormBody.Builder()
form.forEach { (k, v) ->
body.addEncoded(k, v)
}
val request = Request.Builder()
.post(body.build())
.url(url)
if (headers != null) {
request.headers(headers)
}
return httpClient.newCall(request.build()).await().ensureSuccess()
}
/**
* Do a POST http request to specific url with `multipart/form-data` payload
* @param url
* @param payload payload as `key=value` string with `&` separator
* @param headers an additional headers for request, may be null
*/
suspend fun httpPost(
url: String,
payload: String,
headers: Headers?,
): Response {
val body = FormBody.Builder()
payload.split('&').forEach {
val pos = it.indexOf('=')
if (pos != -1) {
val k = it.substring(0, pos)
val v = it.substring(pos + 1)
body.addEncoded(k, v)
}
}
val request = Request.Builder()
.post(body.build())
.url(url)
if (headers != null) {
request.headers(headers)
}
return httpClient.newCall(request.build()).await().ensureSuccess()
}
/**
* Do a GraphQL request to specific url
* @param endpoint an url
* @param query GraphQL request payload
*/
suspend fun graphQLQuery(endpoint: String, query: String): JSONObject {
val body = JSONObject()
body.put("operationName", null as Any?)
body.put("variables", JSONObject())
body.put("query", "{$query}")
val mediaType = "application/json; charset=utf-8".toMediaType()
val requestBody = body.toString().toRequestBody(mediaType)
val request = Request.Builder()
.post(requestBody)
.url(endpoint)
val json = httpClient.newCall(request.build()).await().ensureSuccess().parseJson()
json.optJSONArray("errors")?.let {
if (it.length() != 0) {
throw GraphQLException(it)
}
}
return json
}
open fun encodeBase64(data: ByteArray): String = Base64.getEncoder().encodeToString(data)
@@ -29,4 +146,21 @@ abstract class MangaLoaderContext {
abstract suspend fun evaluateJs(script: String): String?
abstract fun getConfig(source: MangaSource): MangaSourceConfig
}
private fun Response.ensureSuccess(): Response {
val exception: Exception? = when (code) { // Catch some error codes, not all
404 -> NotFoundException(message, request.url.toString())
in 500..599 -> HttpStatusException(message, code, request.url.toString())
else -> null
}
if (exception != null) {
runCatching {
close()
}.onFailure {
exception.addSuppressed(it)
}
throw exception
}
return this
}
}

View File

@@ -1,96 +1,63 @@
package org.koitharu.kotatsu.parsers
import androidx.annotation.CallSuper
import androidx.annotation.VisibleForTesting
import okhttp3.Headers
import okhttp3.HttpUrl
import org.jsoup.nodes.Element
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.exception.ParseException
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.network.OkHttpWebClient
import org.koitharu.kotatsu.parsers.network.UserAgents
import org.koitharu.kotatsu.parsers.network.WebClient
import org.koitharu.kotatsu.parsers.util.FaviconParser
import org.koitharu.kotatsu.parsers.util.RelatedMangaFinder
import org.koitharu.kotatsu.parsers.util.domain
import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl
import java.util.*
abstract class MangaParser @InternalParsersApi constructor(
@property:InternalParsersApi val context: MangaLoaderContext,
val source: MangaSource,
) {
abstract class MangaParser @InternalParsersApi constructor(val source: MangaSource) {
protected abstract val context: MangaLoaderContext
/**
* Supported [SortOrder] variants. Must not be empty.
*
* For better performance use [EnumSet] for more than one item.
*/
abstract val availableSortOrders: Set<SortOrder>
/**
* Supported [MangaState] variants for filtering. May be empty.
*
* For better performance use [EnumSet] for more than one item.
*/
open val availableStates: Set<MangaState>
get() = emptySet()
/**
* Whether parser supports filtering by more than one tag
*/
open val isMultipleTagsSupported: Boolean = true
@Deprecated(
message = "Use availableSortOrders instead",
replaceWith = ReplaceWith("availableSortOrders"),
)
val sortOrders: Set<SortOrder>
get() = availableSortOrders
abstract val sortOrders: Set<SortOrder>
val config by lazy { context.getConfig(source) }
open val sourceLocale: Locale
get() = source.locale?.let { Locale(it) } ?: Locale.ROOT
val isNsfwSource = source.contentType == ContentType.HENTAI
val sourceLocale: Locale?
get() = source.locale?.let { Locale(it) }
/**
* Provide default domain and available alternatives, if any.
*
* Never hardcode domain in requests, use [domain] instead.
* Never hardcode domain in requests, use [getDomain] instead.
*/
@InternalParsersApi
abstract val configKeyDomain: ConfigKey.Domain
protected abstract val configKeyDomain: ConfigKey.Domain
open val headers: Headers = Headers.Builder()
.add("User-Agent", UserAgents.CHROME_MOBILE)
.build()
@VisibleForTesting(otherwise = VisibleForTesting.PROTECTED)
internal open val headers: Headers? = null
/**
* Used as fallback if value of `sortOrder` passed to [getList] is null
*/
protected open val defaultSortOrder: SortOrder
get() {
val supported = availableSortOrders
return SortOrder.entries.first { it in supported }
val supported = sortOrders
return SortOrder.values().first { it in supported }
}
@JvmField
protected val webClient: WebClient = OkHttpWebClient(context.httpClient, source)
/**
* Parse list of manga by specified criteria
*
* @param offset starting from 0 and used for pagination.
* Note than passed value may not be divisible by internal page size, so you should adjust it manually.
* @param query search query, may be null or empty if no search needed
* @param tags genres for filtering, values from [getAvailableTags] and [Manga.tags]. May be null or empty
* @param sortOrder one of [availableSortOrders] or null for default value
* @param tags genres for filtering, values from [getTags] and [Manga.tags]. May be null or empty
* @param sortOrder one of [sortOrders] or null for default value
*/
@JvmSynthetic
@InternalParsersApi
@Deprecated(
"Use getList with filter instead",
replaceWith = ReplaceWith("getList(offset, filter)"),
)
abstract suspend fun getList(
offset: Int,
query: String?,
@@ -104,15 +71,8 @@ abstract class MangaParser @InternalParsersApi constructor(
* @param offset starting from 0 and used for pagination.
* @param query search query
*/
@Deprecated(
"Use getList with filter instead",
ReplaceWith(
"getList(offset, MangaListFilter.Search(query))",
"org.koitharu.kotatsu.parsers.model.MangaListFilter",
),
)
open suspend fun getList(offset: Int, query: String): List<Manga> {
return getList(offset, MangaListFilter.Search(query))
return getList(offset, query, null, defaultSortOrder)
}
/**
@@ -120,29 +80,11 @@ abstract class MangaParser @InternalParsersApi constructor(
*
* @param offset starting from 0 and used for pagination.
* Note than passed value may not be divisible by internal page size, so you should adjust it manually.
* @param tags genres for filtering, values from [getAvailableTags] and [Manga.tags]. May be null or empty
* @param sortOrder one of [availableSortOrders] or null for default value
* @param tags genres for filtering, values from [getTags] and [Manga.tags]. May be null or empty
* @param sortOrder one of [sortOrders] or null for default value
*/
@Deprecated(
"Use getList with filter instead",
ReplaceWith(
"getList(offset, MangaListFilter.Advanced(sortOrder, tags, null, emptySet()))",
"org.koitharu.kotatsu.parsers.model.MangaListFilter",
),
)
open suspend fun getList(offset: Int, tags: Set<MangaTag>?, sortOrder: SortOrder?): List<Manga> {
return getList(
offset,
MangaListFilter.Advanced(sortOrder ?: defaultSortOrder, tags.orEmpty(), null, emptySet()),
)
}
open suspend fun getList(offset: Int, filter: MangaListFilter?): List<Manga> {
return when (filter) {
is MangaListFilter.Advanced -> getList(offset, null, filter.tags, filter.sortOrder)
is MangaListFilter.Search -> getList(offset, filter.query, null, defaultSortOrder)
null -> getList(offset, null, null, defaultSortOrder)
}
return getList(offset, null, tags, sortOrder ?: defaultSortOrder)
}
/**
@@ -161,29 +103,27 @@ abstract class MangaParser @InternalParsersApi constructor(
/**
* Fetch direct link to the page image.
*/
open suspend fun getPageUrl(page: MangaPage): String = page.url.toAbsoluteUrl(domain)
open suspend fun getPageUrl(page: MangaPage): String = page.url.toAbsoluteUrl(getDomain())
/**
* Fetch available tags (genres) for source
*/
abstract suspend fun getAvailableTags(): Set<MangaTag>
abstract suspend fun getTags(): Set<MangaTag>
/**
* Fetch available locales for multilingual sources
* Returns direct link to the website favicon
*/
open suspend fun getAvailableLocales(): Set<Locale> = emptySet()
@Deprecated(
message = "Use getAvailableTags instead",
replaceWith = ReplaceWith("getAvailableTags()"),
message = "Use parseFavicons() to get multiple favicons with different size",
replaceWith = ReplaceWith("parseFavicons()"),
)
suspend fun getTags(): Set<MangaTag> = getAvailableTags()
open fun getFaviconUrl() = "https://${getDomain()}/favicon.ico"
/**
* Parse favicons from the main page of the source`s website
*/
open suspend fun getFavicons(): Favicons {
return FaviconParser(webClient, domain).parseFavicons()
return FaviconParser(context, getDomain(), headers).parseFavicons()
}
@CallSuper
@@ -191,13 +131,70 @@ abstract class MangaParser @InternalParsersApi constructor(
keys.add(configKeyDomain)
}
open suspend fun getRelatedManga(seed: Manga): List<Manga> {
return RelatedMangaFinder(listOf(this)).invoke(seed)
/* Utils */
fun getDomain(): String {
return config[configKeyDomain]
}
protected fun getParser(source: MangaSource) = if (this.source == source) {
this
} else {
context.newParserInstance(source)
fun getDomain(subdomain: String): String {
val domain = getDomain()
return subdomain + "." + domain.removePrefix("www.")
}
}
fun urlBuilder(): HttpUrl.Builder {
return HttpUrl.Builder()
.scheme("https")
.host(getDomain())
}
/**
* Create a unique id for [Manga]/[MangaChapter]/[MangaPage].
* @param url must be relative url, without a domain
* @see [Manga.id]
* @see [MangaChapter.id]
* @see [MangaPage.id]
*/
@InternalParsersApi
protected fun generateUid(url: String): Long {
var h = 1125899906842597L
source.name.forEach { c ->
h = 31 * h + c.code
}
url.forEach { c ->
h = 31 * h + c.code
}
return h
}
/**
* Create a unique id for [Manga]/[MangaChapter]/[MangaPage].
* @param id an internal identifier
* @see [Manga.id]
* @see [MangaChapter.id]
* @see [MangaPage.id]
*/
@InternalParsersApi
protected fun generateUid(id: Long): Long {
var h = 1125899906842597L
source.name.forEach { c ->
h = 31 * h + c.code
}
h = 31 * h + id
return h
}
@InternalParsersApi
protected fun Element.parseFailed(message: String? = null): Nothing {
throw ParseException(message, ownerDocument()?.location() ?: baseUri(), null)
}
@InternalParsersApi
protected fun Set<MangaTag>?.oneOrThrowIfMany(): MangaTag? {
return when {
isNullOrEmpty() -> null
size == 1 -> first()
else -> throw IllegalArgumentException("Multiple genres are not supported by this source")
}
}
}

View File

@@ -1,7 +1,5 @@
package org.koitharu.kotatsu.parsers
import org.koitharu.kotatsu.parsers.model.ContentType
/**
* Annotate each [MangaParser] implementation with this annotation, used by codegen
*/
@@ -13,15 +11,10 @@ annotation class MangaSourceParser(
val name: String,
/**
* User-friendly title of manga source. In most case equals the website name.
* Avoid extra whitespaces between the words if it is not required.
*/
val title: String,
/**
* Language code (for example "en" or "ru") or blank if parser provide manga on different languages.
*/
val locale: String = "",
/**
* Type of content provided by parser. See [ContentType] for more info
*/
val type: ContentType = ContentType.MANGA,
)
)

View File

@@ -1,6 +1,5 @@
package org.koitharu.kotatsu.parsers
import androidx.annotation.RestrictTo
import org.koitharu.kotatsu.parsers.model.Manga
import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.model.MangaTag
@@ -9,16 +8,12 @@ import org.koitharu.kotatsu.parsers.util.Paginator
@InternalParsersApi
abstract class PagedMangaParser(
context: MangaLoaderContext,
source: MangaSource,
@RestrictTo(RestrictTo.Scope.TESTS) @JvmField internal val pageSize: Int,
pageSize: Int,
searchPageSize: Int = pageSize,
) : MangaParser(context, source) {
) : MangaParser(source) {
@JvmField
protected val paginator = Paginator(pageSize)
@JvmField
protected val searchPaginator = Paginator(searchPageSize)
override suspend fun getList(offset: Int, query: String): List<Manga> {
@@ -52,4 +47,4 @@ abstract class PagedMangaParser(
paginator.onListReceived(offset, page, list.size)
return list
}
}
}

View File

@@ -1,28 +1,13 @@
package org.koitharu.kotatsu.parsers.config
sealed class ConfigKey<T>(
@JvmField val key: String,
val key: String,
) {
abstract val defaultValue: T
class Domain(
@JvmField @JvmSuppressWildcards vararg val presetValues: String,
) : ConfigKey<String>("domain") {
init {
require(presetValues.isNotEmpty()) { "You must provide at least one domain" }
}
override val defaultValue: String
get() = presetValues.first()
}
class ShowSuspiciousContent(
override val defaultValue: Boolean,
) : ConfigKey<Boolean>("show_suspicious")
class UserAgent(
override val defaultValue: String,
) : ConfigKey<String>("user_agent")
}
val presetValues: Array<String>?,
) : ConfigKey<String>("domain")
}

View File

@@ -6,7 +6,6 @@ import org.koitharu.kotatsu.parsers.model.MangaSource
/**
* Authorization is required for access to the requested content
*/
class AuthRequiredException @InternalParsersApi @JvmOverloads constructor(
class AuthRequiredException @InternalParsersApi constructor(
val source: MangaSource,
cause: Throwable? = null,
) : RuntimeException("Authorization required", cause)
) : RuntimeException("Authorization required")

View File

@@ -0,0 +1,7 @@
package org.koitharu.kotatsu.parsers.exception
import okio.IOException
class CloudFlareProtectedException(
val url: String,
) : IOException("Protected by CloudFlare: $url")

View File

@@ -1,24 +0,0 @@
package org.koitharu.kotatsu.parsers.model
enum class ContentType {
/**
* Standard manga, manhua, webtoons, etc
*/
MANGA,
/**
* Use this if the source provides mostly nsfw content.
*/
HENTAI,
/**
* Western comics
*/
COMICS,
/**
* Use this type if no other suits your needs. For example, for an indie manga
*/
OTHER,
}

View File

@@ -3,12 +3,11 @@ package org.koitharu.kotatsu.parsers.model
import okhttp3.HttpUrl.Companion.toHttpUrl
class Favicon internal constructor(
@JvmField val url: String,
@JvmField val size: Int,
@JvmField internal val rel: String?,
val url: String,
val size: Int,
internal val rel: String?,
) : Comparable<Favicon> {
@JvmField
val type: String = url.toHttpUrl().pathSegments.last()
.substringAfterLast('.', "").lowercase()
@@ -49,4 +48,4 @@ class Favicon internal constructor(
"mask-icon" -> -1
else -> 0
}
}
}

View File

@@ -2,7 +2,7 @@ package org.koitharu.kotatsu.parsers.model
class Favicons internal constructor(
favicons: Collection<Favicon>,
@JvmField val referer: String,
val referer: String,
) : Collection<Favicon> {
private val icons = favicons.sortedDescending()
@@ -18,11 +18,6 @@ class Favicons internal constructor(
override fun iterator(): Iterator<Favicon> = icons.iterator()
operator fun minus(victim: Favicon): Favicons = Favicons(
favicons = icons.filterNot { it == victim },
referer = referer,
)
/**
* Finds a favicon whose size in pixels is greater than or equal to the specified size.
* If such icon is not available returns the largest icon
@@ -47,4 +42,4 @@ class Favicons internal constructor(
}
return result
}
}
}

View File

@@ -6,67 +6,67 @@ class Manga(
/**
* Unique identifier for manga
*/
@JvmField val id: Long,
val id: Long,
/**
* Manga title, human-readable
*/
@JvmField val title: String,
val title: String,
/**
* Alternative title (for example on other language), may be null
*/
@JvmField val altTitle: String?,
val altTitle: String?,
/**
* Relative url to manga (**without** a domain) or any other uri.
* Used principally in parsers
*/
@JvmField val url: String,
val url: String,
/**
* Absolute url to manga, must be ready to open in browser
*/
@JvmField val publicUrl: String,
val publicUrl: String,
/**
* Normalized manga rating, must be in range of 0..1 or [RATING_UNKNOWN] if rating s unknown
* @see hasRating
*/
@JvmField val rating: Float,
val rating: Float,
/**
* Indicates that manga may contain sensitive information (18+, NSFW)
*/
@JvmField val isNsfw: Boolean,
val isNsfw: Boolean,
/**
* Absolute link to the cover
* @see largeCoverUrl
*/
@JvmField val coverUrl: String,
val coverUrl: String,
/**
* Tags (genres) of the manga
*/
@JvmField val tags: Set<MangaTag>,
val tags: Set<MangaTag>,
/**
* Manga status (ongoing, finished) or null if unknown
*/
@JvmField val state: MangaState?,
val state: MangaState?,
/**
* Author of the manga, may be null
*/
@JvmField val author: String?,
val author: String?,
/**
* Large cover url (absolute), null if is no large cover
* @see coverUrl
*/
@JvmField val largeCoverUrl: String? = null,
val largeCoverUrl: String? = null,
/**
* Manga description, may be html or null
*/
@JvmField val description: String? = null,
val description: String? = null,
/**
* List of chapters
*/
@JvmField val chapters: List<MangaChapter>? = null,
val chapters: List<MangaChapter>? = null,
/**
* Manga source
*/
@JvmField val source: MangaSource,
val source: MangaSource,
) {
/**
@@ -155,8 +155,4 @@ class Manga(
result = 31 * result + source.hashCode()
return result
}
override fun toString(): String {
return "Manga($id - \"$title\" [$url] - $source)"
}
}
}

View File

@@ -4,34 +4,34 @@ class MangaChapter(
/**
* An unique id of chapter
*/
@JvmField val id: Long,
val id: Long,
/**
* User-readable name of chapter
*/
@JvmField val name: String,
val name: String,
/**
* Chapter number starting from 1
*/
@JvmField val number: Int,
val number: Int,
/**
* Relative url to chapter (**without** a domain) or any other uri.
* Used principally in parsers
*/
@JvmField val url: String,
val url: String,
/**
* User-readable name of scanlator (releaser) or null if unknown
*/
@JvmField val scanlator: String?,
val scanlator: String?,
/**
* Chapter upload date in milliseconds
*/
@JvmField val uploadDate: Long,
val uploadDate: Long,
/**
* User-readable name of branch.
* A branch is a group of chapters that overlap (e.g. different languages)
*/
@JvmField val branch: String?,
@JvmField val source: MangaSource,
val branch: String?,
val source: MangaSource,
) : Comparable<MangaChapter> {
override fun compareTo(other: MangaChapter): Int {
@@ -67,8 +67,4 @@ class MangaChapter(
result = 31 * result + source.hashCode()
return result
}
override fun toString(): String {
return "MangaChapter($id - #$number [$url] - $source)"
}
}
}

View File

@@ -1,29 +0,0 @@
package org.koitharu.kotatsu.parsers.model
import java.util.*
sealed interface MangaListFilter {
fun isEmpty(): Boolean
val sortOrder: SortOrder?
data class Search(
@JvmField val query: String,
) : MangaListFilter {
override val sortOrder: SortOrder? = null
override fun isEmpty() = query.isBlank()
}
data class Advanced(
override val sortOrder: SortOrder,
@JvmField val tags: Set<MangaTag>,
@JvmField val locale: Locale?,
@JvmField val states: Set<MangaState>,
) : MangaListFilter {
override fun isEmpty(): Boolean = tags.isEmpty() && locale == null && states.isEmpty()
}
}

View File

@@ -6,19 +6,24 @@ class MangaPage(
/**
* Unique identifier for manga
*/
@JvmField val id: Long,
val id: Long,
/**
* Relative url to page (**without** a domain) or any other uri.
* Used principally in parsers.
* May contain link to image or html page.
* @see MangaParser.getPageUrl
*/
@JvmField val url: String,
val url: String,
/**
* Absolute link to the chapter or website home page.
* Used in Referer header
*/
val referer: String,
/**
* Absolute url of the small page image if exists, null otherwise
*/
@JvmField val preview: String?,
@JvmField val source: MangaSource,
val preview: String?,
val source: MangaSource,
) {
override fun equals(other: Any?): Boolean {
@@ -29,19 +34,19 @@ class MangaPage(
if (id != other.id) return false
if (url != other.url) return false
if (referer != other.referer) return false
if (preview != other.preview) return false
return source == other.source
if (source != other.source) return false
return true
}
override fun hashCode(): Int {
var result = id.hashCode()
result = 31 * result + url.hashCode()
result = 31 * result + referer.hashCode()
result = 31 * result + (preview?.hashCode() ?: 0)
result = 31 * result + source.hashCode()
return result
}
override fun toString(): String {
return "MangaPage($id [$url] - $source)"
}
}
}

View File

@@ -1,5 +1,5 @@
package org.koitharu.kotatsu.parsers.model
enum class MangaState {
ONGOING, FINISHED, ABANDONED, PAUSED
}
ONGOING, FINISHED
}

View File

@@ -6,13 +6,13 @@ class MangaTag(
/**
* User-readable tag title, should be in Title case
*/
@JvmField val title: String,
val title: String,
/**
* Identifier of a tag, must be unique among the source.
* @see MangaParser.getList
*/
@JvmField val key: String,
@JvmField val source: MangaSource,
val key: String,
val source: MangaSource,
) {
override fun equals(other: Any?): Boolean {
@@ -34,8 +34,4 @@ class MangaTag(
result = 31 * result + source.hashCode()
return result
}
override fun toString(): String {
return "MangaTag($key \"$title\" - $source)"
}
}
}

View File

@@ -1,133 +0,0 @@
package org.koitharu.kotatsu.parsers.network
import okhttp3.*
import okhttp3.MediaType.Companion.toMediaType
import okhttp3.RequestBody.Companion.toRequestBody
import org.json.JSONObject
import org.jsoup.HttpStatusException
import org.koitharu.kotatsu.parsers.exception.AuthRequiredException
import org.koitharu.kotatsu.parsers.exception.GraphQLException
import org.koitharu.kotatsu.parsers.exception.NotFoundException
import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.util.await
import org.koitharu.kotatsu.parsers.util.parseJson
import java.net.HttpURLConnection
class OkHttpWebClient(
private val httpClient: OkHttpClient,
private val mangaSource: MangaSource,
) : WebClient {
override suspend fun httpGet(url: HttpUrl, extraHeaders: Headers?): Response {
val request = Request.Builder()
.get()
.url(url)
.addTags()
.addExtraHeaders(extraHeaders)
return httpClient.newCall(request.build()).await().ensureSuccess()
}
override suspend fun httpHead(url: HttpUrl): Response {
val request = Request.Builder()
.head()
.url(url)
.addTags()
return httpClient.newCall(request.build()).await().ensureSuccess()
}
override suspend fun httpPost(url: HttpUrl, form: Map<String, String>, extraHeaders: Headers?): Response {
val body = FormBody.Builder()
form.forEach { (k, v) ->
body.addEncoded(k, v)
}
val request = Request.Builder()
.post(body.build())
.url(url)
.addTags()
.addExtraHeaders(extraHeaders)
return httpClient.newCall(request.build()).await().ensureSuccess()
}
override suspend fun httpPost(url: HttpUrl, payload: String, extraHeaders: Headers?): Response {
val body = FormBody.Builder()
payload.split('&').forEach {
val pos = it.indexOf('=')
if (pos != -1) {
val k = it.substring(0, pos)
val v = it.substring(pos + 1)
body.addEncoded(k, v)
}
}
val request = Request.Builder()
.post(body.build())
.url(url)
.addTags()
.addExtraHeaders(extraHeaders)
return httpClient.newCall(request.build()).await().ensureSuccess()
}
override suspend fun httpPost(url: HttpUrl, body: JSONObject, extraHeaders: Headers?): Response {
val mediaType = "application/json; charset=utf-8".toMediaType()
val requestBody = body.toString().toRequestBody(mediaType)
val request = Request.Builder()
.post(requestBody)
.url(url)
.addTags()
.addExtraHeaders(extraHeaders)
return httpClient.newCall(request.build()).await().ensureSuccess()
}
override suspend fun graphQLQuery(endpoint: String, query: String): JSONObject {
val body = JSONObject()
body.put("operationName", null as Any?)
body.put("variables", JSONObject())
body.put("query", "{$query}")
val mediaType = "application/json; charset=utf-8".toMediaType()
val requestBody = body.toString().toRequestBody(mediaType)
val request = Request.Builder()
.post(requestBody)
.url(endpoint)
.addTags()
val json = httpClient.newCall(request.build()).await().parseJson()
json.optJSONArray("errors")?.let {
if (it.length() != 0) {
throw GraphQLException(it)
}
}
return json
}
private fun Request.Builder.addTags(): Request.Builder {
tag(MangaSource::class.java, mangaSource)
return this
}
private fun Request.Builder.addExtraHeaders(headers: Headers?): Request.Builder {
if (headers != null) {
headers(headers)
}
return this
}
private fun Response.ensureSuccess(): Response {
val exception: Exception? = when (code) { // Catch some error codes, not all
HttpURLConnection.HTTP_NOT_FOUND -> NotFoundException(message, request.url.toString())
HttpURLConnection.HTTP_UNAUTHORIZED -> request.tag(MangaSource::class.java)?.let {
AuthRequiredException(it)
} ?: HttpStatusException(message, code, request.url.toString())
in 400..599 -> HttpStatusException(message, code, request.url.toString())
else -> null
}
if (exception != null) {
runCatching {
close()
}.onFailure {
exception.addSuppressed(it)
}
throw exception
}
return this
}
}

View File

@@ -1,14 +0,0 @@
package org.koitharu.kotatsu.parsers.network
object UserAgents {
const val CHROME_MOBILE =
"Mozilla/5.0 (Linux; Android 13) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.5735.196 Mobile Safari/537.36"
const val CHROME_DESKTOP =
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"
const val FIREFOX_DESKTOP = "Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/116.0"
const val KOTATSU = "Kotatsu/5.3 (Android 13;;; en)"
}

View File

@@ -1,116 +0,0 @@
package org.koitharu.kotatsu.parsers.network
import okhttp3.Headers
import okhttp3.HttpUrl
import okhttp3.HttpUrl.Companion.toHttpUrl
import okhttp3.Response
import org.json.JSONObject
interface WebClient {
/**
* Do a GET http request to specific url
* @param url
*/
suspend fun httpGet(url: String): Response = httpGet(url.toHttpUrl())
suspend fun httpGet(url: String, extraHeaders: Headers?): Response = httpGet(url.toHttpUrl(), extraHeaders)
/**
* Do a GET http request to specific url
* @param url
*/
suspend fun httpGet(url: HttpUrl): Response = httpGet(url, null)
/**
* Do a GET http request to specific url
* @param url
* @param extraHeaders additional HTTP headers for request
*/
suspend fun httpGet(url: HttpUrl, extraHeaders: Headers?): Response
/**
* Do a HEAD http request to specific url
* @param url
*/
suspend fun httpHead(url: String): Response = httpHead(url.toHttpUrl())
/**
* Do a HEAD http request to specific url
* @param url
*/
suspend fun httpHead(url: HttpUrl): Response
/**
* Do a POST http request to specific url with `multipart/form-data` payload
* @param url
* @param form payload as key=>value map
*/
suspend fun httpPost(url: String, form: Map<String, String>): Response = httpPost(url.toHttpUrl(), form, null)
/**
* Do a POST http request to specific url with `multipart/form-data` payload
* @param url
* @param form payload as key=>value map
*/
suspend fun httpPost(url: HttpUrl, form: Map<String, String>): Response = httpPost(url, form, null)
/**
* Do a POST http request to specific url with `multipart/form-data` payload
* @param url
* @param form payload as key=>value map
* @param extraHeaders additional HTTP headers for request
*/
suspend fun httpPost(url: HttpUrl, form: Map<String, String>, extraHeaders: Headers?): Response
/**
* Do a POST http request to specific url with `multipart/form-data` payload
* @param url
* @param payload payload as `key=value` string with `&` separator
*/
suspend fun httpPost(url: String, payload: String): Response = httpPost(url.toHttpUrl(), payload, null)
/**
* Do a POST http request to specific url with `multipart/form-data` payload
* @param url
* @param payload payload as `key=value` string with `&` separator
*/
suspend fun httpPost(url: HttpUrl, payload: String): Response = httpPost(url, payload, null)
/**
* Do a POST http request to specific url with `multipart/form-data` payload
* @param url
* @param payload payload as `key=value` string with `&` separator
* @param extraHeaders additional HTTP headers for request
*/
suspend fun httpPost(url: HttpUrl, payload: String, extraHeaders: Headers?): Response
/**
* Do a POST http request to specific url with json payload
* @param url
* @param body
*/
suspend fun httpPost(url: String, body: JSONObject): Response = httpPost(url.toHttpUrl(), body, null)
/**
* Do a POST http request to specific url with json payload
* @param url
* @param body
*/
suspend fun httpPost(url: HttpUrl, body: JSONObject): Response = httpPost(url, body, null)
/**
* Do a POST http request to specific url with json payload
* @param url
* @param body
* @param extraHeaders additional HTTP headers for request
*/
suspend fun httpPost(url: HttpUrl, body: JSONObject, extraHeaders: Headers?): Response
/**
* Do a GraphQL request to specific url
* @param endpoint an url
* @param query GraphQL request payload
*/
suspend fun graphQLQuery(endpoint: String, query: String): JSONObject
}

View File

@@ -1,4 +1,4 @@
package org.koitharu.kotatsu.parsers.site.be
package org.koitharu.kotatsu.parsers.site
import androidx.collection.ArraySet
import org.json.JSONArray
@@ -8,25 +8,25 @@ import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.domain
import org.koitharu.kotatsu.parsers.util.generateUid
import org.koitharu.kotatsu.parsers.util.getDomain
import org.koitharu.kotatsu.parsers.util.json.mapJSON
import org.koitharu.kotatsu.parsers.util.json.mapJSONIndexed
import org.koitharu.kotatsu.parsers.util.json.mapJSONNotNull
import org.koitharu.kotatsu.parsers.util.json.stringIterator
import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl
import java.util.*
@MangaSourceParser("ANIBEL", "Anibel", "be")
internal class AnibelParser(context: MangaLoaderContext) : MangaParser(context, MangaSource.ANIBEL) {
internal class AnibelParser(override val context: MangaLoaderContext) : MangaParser(MangaSource.ANIBEL) {
override val configKeyDomain = ConfigKey.Domain("anibel.net")
override val configKeyDomain = ConfigKey.Domain("anibel.net", null)
override val availableSortOrders: Set<SortOrder> = EnumSet.of(
override val sortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.NEWEST,
)
override fun getFaviconUrl(): String {
return "https://cdn.${getDomain()}/favicons/favicon.png"
}
override suspend fun getList(
offset: Int,
query: String?,
@@ -73,12 +73,12 @@ internal class AnibelParser(context: MangaLoaderContext) : MangaParser(context,
title = title.getString("be"),
coverUrl = jo.getString("poster").removePrefix("/cdn")
.toAbsoluteUrl(getDomain("cdn")) + "?width=200&height=280",
altTitle = title.optJSONArray("alt")?.optString(0)?.takeUnless(String::isEmpty),
altTitle = title.getString("alt").takeUnless(String::isEmpty),
author = null,
isNsfw = false,
rating = jo.getDouble("rating").toFloat() / 10f,
url = href,
publicUrl = "https://${domain}/$href",
publicUrl = "https://${getDomain()}/$href",
tags = jo.getJSONArray("genres").mapToTags(),
state = when (jo.getString("status")) {
"ongoing" -> MangaState.ONGOING
@@ -111,21 +111,20 @@ internal class AnibelParser(context: MangaLoaderContext) : MangaParser(context,
""".trimIndent(),
).getJSONObject("media")
val title = details.getJSONObject("title")
val poster = details.getString("poster").removePrefix("/cdn").toAbsoluteUrl(getDomain("cdn"))
val poster = details.getString("poster").removePrefix("/cdn")
.toAbsoluteUrl(getDomain("cdn"))
val chapters = apiCall(
"""
chapters(mediaId: "${details.getString("mediaId")}", offset: 0, limit: 99999) {
docs {
id
chapter
released
}
chapters(mediaId: "${details.getString("mediaId")}") {
id
chapter
released
}
""".trimIndent(),
).getJSONObject("chapters").getJSONArray("docs")
).getJSONArray("chapters")
return manga.copy(
title = title.getString("be"),
altTitle = title.optJSONArray("alt")?.optString(0)?.takeUnless(String::isEmpty),
altTitle = title.getString("alt"),
coverUrl = "$poster?width=200&height=280",
largeCoverUrl = poster,
description = details.getJSONObject("description").getString("be"),
@@ -166,17 +165,19 @@ internal class AnibelParser(context: MangaLoaderContext) : MangaParser(context,
""".trimIndent(),
).getJSONObject("chapter")
val pages = chapterJson.getJSONArray("images")
val chapterUrl = "https://${getDomain()}/${chapter.url}"
return pages.mapJSONIndexed { i, jo ->
MangaPage(
id = generateUid("${chapter.url}/$i"),
url = jo.getString("large"),
referer = chapterUrl,
preview = jo.getString("thumbnail"),
source = source,
)
}
}
override suspend fun getAvailableTags(): Set<MangaTag> {
override suspend fun getTags(): Set<MangaTag> {
val json = apiCall(
"""
getFilters(mediaType: manga) {
@@ -191,29 +192,23 @@ internal class AnibelParser(context: MangaLoaderContext) : MangaParser(context,
private suspend fun search(query: String): List<Manga> {
val json = apiCall(
"""
search(query: "$query", limit: 60) {
mediaId
search(query: "$query", limit: 40) {
id
title {
be
en
}
poster
status
slug
mediaType
genres
url
type
}
""".trimIndent(),
)
val array = json.getJSONArray("search")
return array.mapJSONNotNull { jo ->
val type = jo.getString("mediaType").lowercase()
if (type != "manga") {
return@mapJSONNotNull null
}
val mediaId = jo.getString("mediaId")
return array.mapJSON { jo ->
val mediaId = jo.getString("id")
val title = jo.getJSONObject("title")
val href = "$type/${jo.getString("slug")}"
val href = "${jo.getString("type").lowercase()}/${jo.getString("url")}"
Manga(
id = generateUid(mediaId),
title = title.getString("be"),
@@ -224,20 +219,17 @@ internal class AnibelParser(context: MangaLoaderContext) : MangaParser(context,
isNsfw = false,
rating = RATING_UNKNOWN,
url = href,
publicUrl = "https://${domain}/$href",
tags = jo.getJSONArray("genres").mapToTags(),
state = when (jo.getString("status")) {
"ongoing" -> MangaState.ONGOING
"finished" -> MangaState.FINISHED
else -> null
},
publicUrl = "https://${getDomain()}/$href",
tags = emptySet(),
state = null,
source = source,
)
}
}
private suspend fun apiCall(request: String): JSONObject {
return webClient.graphQLQuery("https://${domain}/graphql", request).getJSONObject("data")
return context.graphQLQuery("https://api.${getDomain()}/graphql", request)
.getJSONObject("data")
}
private fun JSONArray.mapToTags(): Set<MangaTag> {
@@ -250,7 +242,6 @@ internal class AnibelParser(context: MangaLoaderContext) : MangaParser(context,
c == '-' -> {
builder.setCharAt(i, ' ')
}
capitalize -> {
builder.setCharAt(i, c.uppercaseChar())
capitalize = false
@@ -272,4 +263,4 @@ internal class AnibelParser(context: MangaLoaderContext) : MangaParser(context,
}
return result
}
}
}

View File

@@ -1,4 +1,4 @@
package org.koitharu.kotatsu.parsers.site.all
package org.koitharu.kotatsu.parsers.site
import androidx.collection.ArraySet
import org.json.JSONArray
@@ -19,14 +19,13 @@ import javax.crypto.spec.IvParameterSpec
import javax.crypto.spec.SecretKeySpec
@MangaSourceParser("BATOTO", "Bato.To")
internal class BatoToParser(context: MangaLoaderContext) : PagedMangaParser(
context = context,
internal class BatoToParser(override val context: MangaLoaderContext) : PagedMangaParser(
source = MangaSource.BATOTO,
pageSize = 60,
searchPageSize = 20,
) {
override val availableSortOrders: Set<SortOrder> = EnumSet.of(
override val sortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.NEWEST,
SortOrder.UPDATED,
SortOrder.POPULARITY,
@@ -35,30 +34,7 @@ internal class BatoToParser(context: MangaLoaderContext) : PagedMangaParser(
override val configKeyDomain = ConfigKey.Domain(
"bato.to",
"batocomic.com",
"batocomic.net",
"batocomic.org",
"batotoo.com",
"batotwo.com",
"battwo.com",
"comiko.net",
"comiko.org",
"mangatoto.com",
"mangatoto.net",
"mangatoto.org",
"readtoto.com",
"readtoto.net",
"readtoto.org",
"dto.to",
"hto.to",
"mto.to",
"wto.to",
"xbato.com",
"xbato.net",
"xbato.org",
"zbato.com",
"zbato.net",
"zbato.org",
arrayOf("bato.to", "mto.to", "mangatoto.com", "battwo.com", "batotwo.com", "comiko.net", "batotoo.com"),
)
override suspend fun getListPage(
@@ -73,7 +49,7 @@ internal class BatoToParser(context: MangaLoaderContext) : PagedMangaParser(
@Suppress("NON_EXHAUSTIVE_WHEN_STATEMENT")
val url = buildString {
append("https://")
append(domain)
append(getDomain())
append("/browse?sort=")
when (sortOrder) {
SortOrder.UPDATED,
@@ -82,7 +58,6 @@ internal class BatoToParser(context: MangaLoaderContext) : PagedMangaParser(
SortOrder.POPULARITY -> append("views_a.za")
SortOrder.NEWEST -> append("create.za")
SortOrder.ALPHABETICAL -> append("title.az")
SortOrder.RATING -> Unit
}
if (!tags.isNullOrEmpty()) {
append("&genres=")
@@ -95,7 +70,7 @@ internal class BatoToParser(context: MangaLoaderContext) : PagedMangaParser(
}
override suspend fun getDetails(manga: Manga): Manga {
val root = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml()
val root = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml()
.requireElementById("mainer")
val details = root.selectFirstOrThrow(".detail-set")
val attrs = details.selectFirst(".attr-main")?.select(".attr-item")?.associate {
@@ -118,15 +93,16 @@ internal class BatoToParser(context: MangaLoaderContext) : PagedMangaParser(
chapters = root.selectFirst(".episode-list")
?.selectFirst(".main")
?.children()
?.mapChapters(reversed = true) { i, div ->
?.reversed()
?.mapChapters { i, div ->
div.parseChapter(i)
}.orEmpty(),
)
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(domain)
val scripts = webClient.httpGet(fullUrl).parseHtml().select("script")
val fullUrl = chapter.url.toAbsoluteUrl(getDomain())
val scripts = context.httpGet(fullUrl).parseHtml().select("script")
for (script in scripts) {
val scriptSrc = script.html()
val p = scriptSrc.indexOf("const imgHttpLis =")
@@ -150,6 +126,7 @@ internal class BatoToParser(context: MangaLoaderContext) : PagedMangaParser(
result += MangaPage(
id = generateUid(url),
url = url + "?" + args.getString(i),
referer = fullUrl,
preview = null,
source = source,
)
@@ -159,9 +136,9 @@ internal class BatoToParser(context: MangaLoaderContext) : PagedMangaParser(
throw ParseException("Cannot find images list", fullUrl)
}
override suspend fun getAvailableTags(): Set<MangaTag> {
val scripts = webClient.httpGet(
"https://${domain}/browse",
override suspend fun getTags(): Set<MangaTag> {
val scripts = context.httpGet(
"https://${getDomain()}/browse",
).parseHtml().selectOrThrow("script")
for (script in scripts) {
val genres = script.html().substringBetweenFirst("const _genres =", ";") ?: continue
@@ -180,10 +157,12 @@ internal class BatoToParser(context: MangaLoaderContext) : PagedMangaParser(
throw ParseException("Cannot find gernes list", scripts[0].baseUri())
}
override fun getFaviconUrl(): String = "https://styles.amarkcdn.com/img/batoto/favicon.ico?v0"
private suspend fun search(page: Int, query: String): List<Manga> {
val url = buildString {
append("https://")
append(domain)
append(getDomain())
append("/search?word=")
append(query.replace(' ', '+'))
append("&page=")
@@ -198,7 +177,7 @@ internal class BatoToParser(context: MangaLoaderContext) : PagedMangaParser(
?.toIntOrNull() ?: body.parseFailed("Cannot determine current page")
private suspend fun parseList(url: String, page: Int): List<Manga> {
val body = webClient.httpGet(url).parseHtml().body()
val body = context.httpGet(url).parseHtml().body()
if (body.selectFirst(".browse-no-matches") != null) {
return emptyList()
}
@@ -328,4 +307,4 @@ internal class BatoToParser(context: MangaLoaderContext) : PagedMangaParser(
} else byteArrayOf(),
)
}
}
}

View File

@@ -1,7 +1,8 @@
package org.koitharu.kotatsu.parsers.site.vi
package org.koitharu.kotatsu.parsers.site
import androidx.collection.ArrayMap
import okhttp3.Headers
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
import org.json.JSONArray
import org.jsoup.nodes.Document
import org.jsoup.nodes.Element
@@ -10,30 +11,27 @@ import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.network.UserAgents
import org.koitharu.kotatsu.parsers.util.*
import java.text.SimpleDateFormat
import java.util.*
import kotlin.collections.HashSet
@MangaSourceParser("BLOGTRUYEN", "BlogTruyen", "vi")
class BlogTruyenParser(context: MangaLoaderContext) :
PagedMangaParser(context, MangaSource.BLOGTRUYEN, pageSize = 20) {
class BlogTruyenParser(override val context: MangaLoaderContext) :
PagedMangaParser(MangaSource.BLOGTRUYEN, pageSize = 20) {
override val configKeyDomain: ConfigKey.Domain
get() = ConfigKey.Domain("blogtruyenmoi.com")
get() = ConfigKey.Domain("blogtruyen.vn", null)
override val availableSortOrders: Set<SortOrder>
override val sortOrders: Set<SortOrder>
get() = EnumSet.of(SortOrder.UPDATED)
override val headers: Headers = Headers.Builder()
.add("User-Agent", UserAgents.CHROME_DESKTOP)
.build()
private val mutex = Mutex()
private val dateFormat = SimpleDateFormat("dd/MM/yyyy HH:mm", Locale.US)
private var cacheTags = SuspendLazy(::fetchTags)
private var cacheTags: ArrayMap<String, MangaTag>? = null
override suspend fun getDetails(manga: Manga): Manga {
val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml()
val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml()
val descriptionElement = doc.selectFirstOrThrow("div.description")
val statusText = descriptionElement
.selectFirst("p:contains(Trạng thái) > span.color-red")
@@ -58,28 +56,27 @@ class BlogTruyenParser(context: MangaLoaderContext) :
}
}
val tags = cacheTags.tryGet().getOrNull()?.let { tagMap ->
descriptionElement.select("p > span.category").mapNotNullToSet {
val tagName = it.selectFirst("a")?.text()?.trim() ?: return@mapNotNullToSet null
tagMap[tagName]
}
val tagMap = getOrCreateTagMap()
val tags = descriptionElement.select("p > span.category").mapNotNullToSet {
val tagName = it.selectFirst("a")?.text()?.trim() ?: return@mapNotNullToSet null
tagMap[tagName]
}
return manga.copy(
tags = tags ?: emptySet(),
tags = tags,
author = descriptionElement.selectFirst("p:contains(Tác giả) > a")?.text(),
description = doc.selectFirst(".detail .content")?.html(),
chapters = parseChapterList(doc),
largeCoverUrl = doc.selectLast("div.thumbnail > img")?.imageUrl(),
largeCoverUrl = doc.selectLast("div.thumbnail > img")?.attrAsAbsoluteUrlOrNull("src"),
state = state,
rating = rating ?: RATING_UNKNOWN,
isNsfw = doc.getElementById("warningCategory") != null,
isNsfw = doc.getElementById("warningCategory") != null
)
}
private fun parseChapterList(doc: Document): List<MangaChapter> {
val chapterList = doc.select("#list-chapters > p")
return chapterList.mapChapters(reversed = true) { index, element ->
return chapterList.asReversed().mapChapters { index, element ->
val titleElement = element.selectFirst("span.title > a") ?: return@mapChapters null
val name = titleElement.text()
val relativeUrl = titleElement.attrAsRelativeUrl("href")
@@ -93,7 +90,7 @@ class BlogTruyenParser(context: MangaLoaderContext) :
scanlator = null,
uploadDate = uploadDate,
branch = null,
source = source,
source = source
)
}
}
@@ -106,49 +103,46 @@ class BlogTruyenParser(context: MangaLoaderContext) :
): List<Manga> {
return when {
!query.isNullOrEmpty() -> {
val searchUrl = "https://${domain}/timkiem/nangcao/1/0/-1/-1?txt=${query.urlEncoded()}&p=$page"
val searchContent = webClient.httpGet(searchUrl).parseHtml()
val searchUrl = "https://${getDomain()}/timkiem/nangcao/1/0/-1/-1?txt=${query.urlEncoded()}&p=$page"
val searchContent = context.httpGet(searchUrl).parseHtml()
.selectFirst("section.list-manga-bycate > div.list")
parseMangaList(searchContent)
}
!tags.isNullOrEmpty() -> {
val tag = tags.oneOrThrowIfMany()!!
val categoryAjax =
"https://${domain}/ajax/Category/AjaxLoadMangaByCategory?id=${tag.key}&orderBy=5&p=$page"
val listContent = webClient.httpGet(categoryAjax).parseHtml().selectFirst("div.list")
val categoryAjax = "https://${getDomain()}/ajax/Category/AjaxLoadMangaByCategory?id=${tag.key}&orderBy=5&p=$page"
val listContent = context.httpGet(categoryAjax).parseHtml().selectFirst("div.list")
parseMangaList(listContent)
}
else -> getNormalList(page)
}
}
private suspend fun getNormalList(page: Int): List<Manga> {
val pageLink = "https://${domain}/page-$page"
val doc = webClient.httpGet(pageLink).parseHtml()
val pageLink = "https://${getDomain()}/page-$page"
val doc = context.httpGet(pageLink).parseHtml()
val listElements = doc.selectFirstOrThrow("section.list-mainpage.listview")
.select("div.bg-white.storyitem")
return listElements.mapNotNull { el ->
val linkTag = el.selectFirst("div.fl-l > a") ?: return@mapNotNull null
return listElements.mapNotNull {
val linkTag = it.selectFirst("div.fl-l > a") ?: return@mapNotNull null
val relativeUrl = linkTag.attrAsRelativeUrl("href")
val tags = cacheTags.tryGet().getOrNull()?.let { tagMap ->
el.select("footer > div.category > a").mapNotNullToSet { a ->
tagMap[a.text()]
}
val tagMap = getOrCreateTagMap()
val tags = it.select("footer > div.category > a").mapNotNullToSet { a ->
tagMap[a.text()]
}
Manga(
id = generateUid(relativeUrl),
title = linkTag.attr("title"),
altTitle = null,
description = el.selectFirst("p.al-j.break.line-height-15")?.text(),
description = it.selectFirst("p.al-j.break.line-height-15")?.text(),
url = relativeUrl,
publicUrl = relativeUrl.toAbsoluteUrl(domain),
coverUrl = linkTag.selectLast("img")?.imageUrl().orEmpty(),
publicUrl = relativeUrl.toAbsoluteUrl(getDomain()),
coverUrl = linkTag.selectLast("img")?.attr("src").orEmpty(),
source = source,
tags = tags ?: emptySet(),
tags = tags,
isNsfw = false,
rating = RATING_UNKNOWN,
author = null,
@@ -170,8 +164,8 @@ class BlogTruyenParser(context: MangaLoaderContext) :
altTitle = null,
description = mangaInfo.select("div.al-j.fs-12").text(),
url = relativeUrl,
publicUrl = relativeUrl.toAbsoluteUrl(domain),
coverUrl = mangaInfo.selectFirst("div > img.img")?.imageUrl().orEmpty(),
publicUrl = relativeUrl.toAbsoluteUrl(getDomain()),
coverUrl = mangaInfo.selectFirst("div > img.img")?.absUrl("src").orEmpty(),
isNsfw = false,
rating = RATING_UNKNOWN,
tags = emptySet(),
@@ -185,16 +179,19 @@ class BlogTruyenParser(context: MangaLoaderContext) :
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
fun generateImageId(index: Int) = generateUid("${chapter.url}/$index")
val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml()
val doc = context.httpGet(chapter.url.toAbsoluteUrl(getDomain())).parseHtml()
val pages = ArrayList<MangaPage>()
val referer = chapter.url.toAbsoluteUrl(getDomain())
doc.select("#content > img").forEach { img ->
val url = img.attrAsRelativeUrl("src")
pages.add(
MangaPage(
id = generateImageId(pages.size),
url = img.imageUrl(),
id = generateImageId(pages.lastIndex),
url = url,
referer = referer,
preview = null,
source = source,
),
)
)
}
@@ -207,11 +204,12 @@ class BlogTruyenParser(context: MangaLoaderContext) :
val imageUrl = imageArr.getJSONObject(i).getString("url")
pages.add(
MangaPage(
id = generateImageId(pages.size),
id = generateImageId(pages.lastIndex),
url = imageUrl,
referer = referer,
preview = null,
source = source,
),
source = source
)
)
}
}
@@ -219,13 +217,20 @@ class BlogTruyenParser(context: MangaLoaderContext) :
return pages
}
override suspend fun getAvailableTags(): Set<MangaTag> {
return cacheTags.get().values.toSet()
override suspend fun getTags(): Set<MangaTag> {
val map = getOrCreateTagMap()
val tags = HashSet<MangaTag>(map.size)
for (entry in map) {
tags.add(entry.value)
}
return tags
}
private suspend fun fetchTags(): Map<String, MangaTag> {
val doc = webClient.httpGet("/timkiem/nangcao".toAbsoluteUrl(domain)).parseHtml()
private suspend fun getOrCreateTagMap(): ArrayMap<String, MangaTag> = mutex.withLock {
cacheTags?.let { return@withLock it }
val doc = context.httpGet("/timkiem/nangcao".toAbsoluteUrl(getDomain())).parseHtml()
val tagItems = doc.select("li[data-id]")
val tagMap = ArrayMap<String, MangaTag>(tagItems.size)
for (tag in tagItems) {
@@ -233,15 +238,11 @@ class BlogTruyenParser(context: MangaLoaderContext) :
tagMap[tag.text().trim()] = MangaTag(
title = title,
key = tag.attr("data-id"),
source = source,
source = source
)
}
return tagMap
}
private fun Element.imageUrl(): String {
return attrAsAbsoluteUrlOrNull("src")
?: attrAsAbsoluteUrlOrNull("data-cfsrc")
?: ""
cacheTags = tagMap
tagMap
}
}

View File

@@ -1,39 +1,47 @@
package org.koitharu.kotatsu.parsers.site.en
package org.koitharu.kotatsu.parsers.site
import org.koitharu.kotatsu.parsers.InternalParsersApi
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.*
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import java.util.*
@MangaSourceParser("CLONEMANGA", "CloneManga", "en")
internal class CloneMangaParser(context: MangaLoaderContext) : MangaParser(context, MangaSource.CLONEMANGA) {
internal class CloneMangaParser(override val context: MangaLoaderContext) : PagedMangaParser(
MangaSource.CLONEMANGA,
pageSize = 1,
) {
override val availableSortOrders: Set<SortOrder> = Collections.singleton(
override val sortOrders: Set<SortOrder> = Collections.singleton(
SortOrder.POPULARITY,
)
override val configKeyDomain = ConfigKey.Domain("manga.clone-army.org")
override val configKeyDomain = ConfigKey.Domain("manga.clone-army.org", null)
@InternalParsersApi
override suspend fun getList(offset: Int, query: String?, tags: Set<MangaTag>?, sortOrder: SortOrder): List<Manga> {
if (query != null || offset > 0) {
override fun getFaviconUrl(): String {
return "https://pbs.twimg.com/profile_images/458758466346029056/Ys93EANp_400x400.png"
}
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
if (query != null || page > 1) {
return emptyList()
}
val link = "https://$domain/viewer_landing.php"
val doc = webClient.httpGet(link).parseHtml()
val link = "https://${getDomain()}/viewer_landing.php"
val doc = context.httpGet(link).parseHtml()
val mangas = doc.getElementsByClass("comicPreviewContainer")
return mangas.mapNotNull { item ->
val background = item.selectFirstOrThrow(".comicPreview").styleValueOrNull("background")
val href = item.selectFirst("a")?.attrAsRelativeUrl("href") ?: return@mapNotNull null
val cover = background?.substring(background.indexOf("site/themes"), background.indexOf(")"))
val attr = item.getElementsByClass("comicPreview").attr("style")
val href = item.selectFirst("a")?.attrAsAbsoluteUrl("href") ?: return@mapNotNull null
val cover = attr.substring(attr.indexOf("site/themes"), attr.indexOf(")"))
Manga(
id = generateUid(href),
title = item.selectFirst("h3")?.text() ?: return@mapNotNull null,
coverUrl = "https://$domain/$cover",
coverUrl = "https://${getDomain()}/$cover",
altTitle = null,
author = "Dan Kim",
rating = RATING_UNKNOWN,
@@ -41,14 +49,14 @@ internal class CloneMangaParser(context: MangaLoaderContext) : MangaParser(conte
isNsfw = false,
tags = emptySet(),
state = null,
publicUrl = href.toAbsoluteUrl(domain),
publicUrl = href.toAbsoluteUrl(getDomain()),
source = source,
)
}
}
override suspend fun getDetails(manga: Manga): Manga {
val doc = webClient.httpGet(manga.publicUrl).parseHtml()
val doc = context.httpGet(manga.publicUrl).parseHtml()
val series = doc.location()
val numChapters = Regex(
pattern = "&page=(.*)&lang=",
@@ -67,7 +75,7 @@ internal class CloneMangaParser(context: MangaLoaderContext) : MangaParser(conte
scanlator = null,
branch = null,
uploadDate = 0L,
source = source,
source = MangaSource.DUMMY,
)
chapters.add(chapter)
}
@@ -75,7 +83,7 @@ internal class CloneMangaParser(context: MangaLoaderContext) : MangaParser(conte
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml()
val doc = context.httpGet(chapter.url.toAbsoluteUrl(getDomain())).parseHtml()
val imgUrl = doc.getElementsByClass("subsectionContainer")[0]
.selectFirst("img")
?.attrAsAbsoluteUrlOrNull("src") ?: doc.parseFailed("Something broken")
@@ -83,11 +91,14 @@ internal class CloneMangaParser(context: MangaLoaderContext) : MangaParser(conte
MangaPage(
id = generateUid(imgUrl),
url = imgUrl,
referer = imgUrl,
preview = null,
source = source,
source = MangaSource.DUMMY,
),
)
}
override suspend fun getAvailableTags(): Set<MangaTag> = emptySet()
}
override suspend fun getTags(): Set<MangaTag> {
return emptySet()
}
}

View File

@@ -1,7 +1,8 @@
package org.koitharu.kotatsu.parsers.site.all
package org.koitharu.kotatsu.parsers.site
import androidx.collection.ArraySet
import androidx.collection.SparseArrayCompat
import org.json.JSONArray
import org.json.JSONObject
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser
@@ -21,11 +22,11 @@ private const val PAGE_SIZE = 20
private const val CHAPTERS_LIMIT = 99999
@MangaSourceParser("COMICK_FUN", "ComicK")
internal class ComickFunParser(context: MangaLoaderContext) : MangaParser(context, MangaSource.COMICK_FUN) {
internal class ComickFunParser(override val context: MangaLoaderContext) : MangaParser(MangaSource.COMICK_FUN) {
override val configKeyDomain = ConfigKey.Domain("comick.app")
override val configKeyDomain = ConfigKey.Domain("comick.fun", null)
override val availableSortOrders: Set<SortOrder> = EnumSet.of(
override val sortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.POPULARITY,
SortOrder.UPDATED,
SortOrder.RATING,
@@ -40,11 +41,11 @@ internal class ComickFunParser(context: MangaLoaderContext) : MangaParser(contex
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val domain = domain
val domain = getDomain()
val url = buildString {
append("https://api.")
append(domain)
append("/v1.0/search?tachiyomi=true")
append("/search?tachiyomi=true")
if (!query.isNullOrEmpty()) {
if (offset > 0) {
return emptyList()
@@ -70,7 +71,7 @@ internal class ComickFunParser(context: MangaLoaderContext) : MangaParser(contex
)
}
}
val ja = webClient.httpGet(url).parseJsonArray()
val ja = context.httpGet(url).parseJsonArray()
val tagsMap = cachedTags ?: loadTags()
return ja.mapJSON { jo ->
val slug = jo.getString("slug")
@@ -85,7 +86,7 @@ internal class ComickFunParser(context: MangaLoaderContext) : MangaParser(contex
coverUrl = jo.getString("cover_url"),
largeCoverUrl = null,
description = jo.getStringOrNull("desc"),
tags = jo.selectGenres(tagsMap),
tags = jo.selectGenres("genres", tagsMap),
state = runCatching {
if (jo.getBoolean("translation_completed")) {
MangaState.FINISHED
@@ -100,44 +101,45 @@ internal class ComickFunParser(context: MangaLoaderContext) : MangaParser(contex
}
override suspend fun getDetails(manga: Manga): Manga {
val domain = domain
val domain = getDomain()
val url = "https://api.$domain/comic/${manga.url}?tachiyomi=true"
val jo = webClient.httpGet(url).parseJson()
val jo = context.httpGet(url).parseJson()
val comic = jo.getJSONObject("comic")
return manga.copy(
title = comic.getString("title"),
altTitle = null, // TODO
isNsfw = jo.getBoolean("matureContent") || comic.getBoolean("hentai"),
description = comic.getStringOrNull("parsed") ?: comic.getString("desc"),
tags = manga.tags + comic.getJSONArray("md_comic_md_genres").mapJSONToSet {
val g = it.getJSONObject("md_genres")
tags = manga.tags + jo.getJSONArray("genres").mapJSONToSet {
MangaTag(
title = g.getString("name"),
key = g.getString("slug"),
title = it.getString("name"),
key = it.getString("slug"),
source = source,
)
},
author = jo.getJSONArray("artists").optJSONObject(0)?.getString("name"),
chapters = getChapters(comic.getString("hid")),
chapters = getChapters(comic.getLong("id")),
)
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val jo = webClient.httpGet(
"https://api.${domain}/chapter/${chapter.url}?tachiyomi=true",
val jo = context.httpGet(
"https://api.${getDomain()}/chapter/${chapter.url}?tachiyomi=true",
).parseJson().getJSONObject("chapter")
val referer = "https://${getDomain()}/"
return jo.getJSONArray("images").mapJSON {
val url = it.getString("url")
MangaPage(
id = generateUid(url),
url = url,
referer = referer,
preview = null,
source = source,
)
}
}
override suspend fun getAvailableTags(): Set<MangaTag> {
override suspend fun getTags(): Set<MangaTag> {
val sparseArray = cachedTags ?: loadTags()
val set = ArraySet<MangaTag>(sparseArray.size())
for (i in 0 until sparseArray.size()) {
@@ -147,7 +149,7 @@ internal class ComickFunParser(context: MangaLoaderContext) : MangaParser(contex
}
private suspend fun loadTags(): SparseArrayCompat<MangaTag> {
val ja = webClient.httpGet("https://api.${domain}/genre").parseJsonArray()
val ja = context.httpGet("https://api.${getDomain()}/genre").parseJsonArray()
val tags = SparseArrayCompat<MangaTag>(ja.length())
for (jo in ja.JSONIterator()) {
tags.append(
@@ -163,49 +165,46 @@ internal class ComickFunParser(context: MangaLoaderContext) : MangaParser(contex
return tags
}
private suspend fun getChapters(hid: String): List<MangaChapter> {
val ja = webClient.httpGet(
url = "https://api.${domain}/comic/$hid/chapters?limit=$CHAPTERS_LIMIT",
private suspend fun getChapters(id: Long): List<MangaChapter> {
val ja = context.httpGet(
url = "https://api.${getDomain()}/comic/$id/chapter?tachiyomi=true&limit=$CHAPTERS_LIMIT",
).parseJson().getJSONArray("chapters")
val dateFormat = SimpleDateFormat("yyyy-MM-dd")
val list = ja.toJSONList().reversed()
val chaptersBuilder = ChaptersListBuilder(list.size)
val branchedChapters = HashMap<String?, HashMap<Pair<String?, String?>, MangaChapter>>()
for (jo in list) {
val vol = jo.getStringOrNull("vol")
val chap = jo.getStringOrNull("chap")
val volChap = vol to chap
val counters = HashMap<Locale, Int>()
return ja.mapReversed { jo ->
val locale = Locale.forLanguageTag(jo.getString("lang"))
val lc = locale.getDisplayName(locale).toTitleCase(locale)
val branch = (list.indices).firstNotNullOf { i ->
val b = if (i == 0) lc else "$lc ($i)"
if (branchedChapters[b]?.get(volChap) == null) b else null
}
val chapter = MangaChapter(
var number = counters[locale] ?: 0
number++
counters[locale] = number
MangaChapter(
id = generateUid(jo.getLong("id")),
name = buildString {
vol?.let { append("Vol ").append(it).append(' ') }
chap?.let { append("Chap ").append(it) }
jo.getStringOrNull("vol")?.let { append("Vol ").append(it).append(' ') }
jo.getStringOrNull("chap")?.let { append("Chap ").append(it) }
jo.getStringOrNull("title")?.let { append(": ").append(it) }
},
number = branchedChapters[branch]?.size?.plus(1) ?: 1,
number = number,
url = jo.getString("hid"),
scanlator = jo.optJSONArray("group_name")?.asIterable<String>()?.joinToString()
?.takeUnless { it.isBlank() },
scanlator = jo.optJSONArray("group_name")?.optString(0),
uploadDate = dateFormat.tryParse(jo.getString("created_at").substringBefore('T')),
branch = branch,
branch = locale.getDisplayName(locale).toTitleCase(locale),
source = source,
)
if (chaptersBuilder.add(chapter)) {
branchedChapters.getOrPut(branch, ::HashMap)[volChap] = chapter
}
}
return chaptersBuilder.toList()
}
private fun JSONObject.selectGenres(tags: SparseArrayCompat<MangaTag>): Set<MangaTag> {
val array = optJSONArray("genres") ?: return emptySet()
private inline fun <R> JSONArray.mapReversed(block: (JSONObject) -> R): List<R> {
val len = length()
val destination = ArrayList<R>(len)
for (i in (0 until len).reversed()) {
val jo = getJSONObject(i)
destination.add(block(jo))
}
return destination
}
private fun JSONObject.selectGenres(name: String, tags: SparseArrayCompat<MangaTag>): Set<MangaTag> {
val array = optJSONArray(name) ?: return emptySet()
val res = ArraySet<MangaTag>(array.length())
for (i in 0 until array.length()) {
val id = array.getInt(i)
@@ -214,4 +213,4 @@ internal class ComickFunParser(context: MangaLoaderContext) : MangaParser(contex
}
return res
}
}
}

View File

@@ -1,38 +1,29 @@
package org.koitharu.kotatsu.parsers.site.ru
package org.koitharu.kotatsu.parsers.site
import androidx.collection.ArrayMap
import okhttp3.Headers
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.exception.ParseException
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.network.UserAgents
import org.koitharu.kotatsu.parsers.util.*
import org.koitharu.kotatsu.parsers.util.json.mapJSON
import org.koitharu.kotatsu.parsers.util.json.mapJSONIndexed
import org.koitharu.kotatsu.parsers.util.json.mapJSONToSet
import java.util.*
@MangaSourceParser("DESUME", "Desu.Me", "ru")
internal class DesuMeParser(context: MangaLoaderContext) : PagedMangaParser(context, MangaSource.DESUME, 20) {
@MangaSourceParser("DESUME", "Desu.me", "ru")
internal class DesuMeParser(override val context: MangaLoaderContext) : PagedMangaParser(MangaSource.DESUME, 20) {
override val configKeyDomain = ConfigKey.Domain("desu.me", "desu.win")
override val configKeyDomain = ConfigKey.Domain("desu.me", null)
override val availableSortOrders: Set<SortOrder> = EnumSet.of(
override val sortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.UPDATED,
SortOrder.POPULARITY,
SortOrder.NEWEST,
SortOrder.ALPHABETICAL,
)
override val headers: Headers = Headers.Builder()
.add("User-Agent", UserAgents.KOTATSU)
.build()
private val tagsCache = SuspendLazy(::fetchTags)
override suspend fun getListPage(
page: Int,
query: String?,
@@ -42,7 +33,7 @@ internal class DesuMeParser(context: MangaLoaderContext) : PagedMangaParser(cont
if (query != null && page != searchPaginator.firstPage) {
return emptyList()
}
val domain = domain
val domain = getDomain()
val url = buildString {
append("https://")
append(domain)
@@ -59,16 +50,14 @@ internal class DesuMeParser(context: MangaLoaderContext) : PagedMangaParser(cont
append(query)
}
}
val json = webClient.httpGet(url).parseJson().getJSONArray("response")
val json = context.httpGet(url).parseJson().getJSONArray("response")
?: throw ParseException("Invalid response", url)
val total = json.length()
val list = ArrayList<Manga>(total)
val tagsMap = tagsCache.tryGet().getOrNull()
for (i in 0 until total) {
val jo = json.getJSONObject(i)
val cover = jo.getJSONObject("image")
val id = jo.getLong("id")
val genres = jo.getString("genres").split(',')
list += Manga(
url = "/manga/api/$id",
publicUrl = jo.getString("url"),
@@ -84,13 +73,7 @@ internal class DesuMeParser(context: MangaLoaderContext) : PagedMangaParser(cont
rating = jo.getDouble("score").toFloat().coerceIn(0f, 1f),
id = generateUid(id),
isNsfw = false,
tags = if (!tagsMap.isNullOrEmpty()) {
genres.mapNotNullToSet { g ->
tagsMap[g.trim().toTitleCase()]
}
} else {
emptySet()
},
tags = emptySet(),
author = null,
description = jo.getString("description"),
)
@@ -99,8 +82,8 @@ internal class DesuMeParser(context: MangaLoaderContext) : PagedMangaParser(cont
}
override suspend fun getDetails(manga: Manga): Manga {
val url = manga.url.toAbsoluteUrl(domain)
val json = webClient.httpGet(url).parseJson().getJSONObject("response")
val url = manga.url.toAbsoluteUrl(getDomain())
val json = context.httpGet(url).parseJson().getJSONObject("response")
?: throw ParseException("Invalid response", url)
val baseChapterUrl = manga.url + "/chapter/"
val chaptersList = json.getJSONObject("chapters").getJSONArray("list")
@@ -134,13 +117,14 @@ internal class DesuMeParser(context: MangaLoaderContext) : PagedMangaParser(cont
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(domain)
val json = webClient.httpGet(fullUrl)
val fullUrl = chapter.url.toAbsoluteUrl(getDomain())
val json = context.httpGet(fullUrl)
.parseJson()
.getJSONObject("response") ?: throw ParseException("Invalid response", fullUrl)
return json.getJSONObject("pages").getJSONArray("list").mapJSON { jo ->
MangaPage(
id = generateUid(jo.getLong("id")),
referer = fullUrl,
preview = null,
source = chapter.source,
url = jo.getString("img"),
@@ -148,8 +132,22 @@ internal class DesuMeParser(context: MangaLoaderContext) : PagedMangaParser(cont
}
}
override suspend fun getAvailableTags(): Set<MangaTag> {
return tagsCache.get().values.toSet()
override suspend fun getTags(): Set<MangaTag> {
val doc = context.httpGet("https://${getDomain()}/manga/").parseHtml()
val root = doc.body().requireElementById("animeFilter")
.selectFirstOrThrow(".catalog-genres")
return root.select("li").mapToSet {
val input = it.selectFirstOrThrow("input")
MangaTag(
source = source,
key = input.attr("data-genre-slug").ifEmpty {
it.parseFailed("data-genre-slug is empty")
},
title = input.attr("data-genre-name").toTitleCase().ifEmpty {
it.parseFailed("data-genre-name is empty")
},
)
}
}
private fun getSortKey(sortOrder: SortOrder) =
@@ -160,26 +158,4 @@ internal class DesuMeParser(context: MangaLoaderContext) : PagedMangaParser(cont
SortOrder.NEWEST -> "id"
else -> "updated"
}
private suspend fun fetchTags(): Map<String, MangaTag> {
val doc = webClient.httpGet("https://$domain/manga/").parseHtml()
val root = doc.body().requireElementById("animeFilter")
.selectFirstOrThrow(".catalog-genres")
val li = root.select("li")
val result = ArrayMap<String, MangaTag>(li.size)
li.forEach {
val input = it.selectFirstOrThrow("input")
val tag = MangaTag(
source = source,
key = input.attr("data-genre-slug").ifEmpty {
it.parseFailed("data-genre-slug is empty")
},
title = input.attr("data-genre-name").toTitleCase().ifEmpty {
it.parseFailed("data-genre-name is empty")
},
)
result[tag.title] = tag
}
return result
}
}
}

View File

@@ -1,8 +1,5 @@
package org.koitharu.kotatsu.parsers.site.all
package org.koitharu.kotatsu.parsers.site
import androidx.collection.SparseArrayCompat
import androidx.collection.set
import okhttp3.HttpUrl.Companion.toHttpUrlOrNull
import org.jsoup.nodes.Element
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParserAuthProvider
@@ -18,29 +15,24 @@ import kotlin.math.pow
private const val DOMAIN_UNAUTHORIZED = "e-hentai.org"
private const val DOMAIN_AUTHORIZED = "exhentai.org"
@MangaSourceParser("EXHENTAI", "ExHentai", type = ContentType.HENTAI)
@MangaSourceParser("EXHENTAI", "ExHentai")
internal class ExHentaiParser(
context: MangaLoaderContext,
) : PagedMangaParser(context, MangaSource.EXHENTAI, pageSize = 25), MangaParserAuthProvider {
override val context: MangaLoaderContext,
) : PagedMangaParser(MangaSource.EXHENTAI, pageSize = 25), MangaParserAuthProvider {
override val availableSortOrders: Set<SortOrder> = Collections.singleton(
override val sortOrders: Set<SortOrder> = Collections.singleton(
SortOrder.NEWEST,
)
override val configKeyDomain: ConfigKey.Domain
get() = ConfigKey.Domain(
if (isAuthorized) DOMAIN_AUTHORIZED else DOMAIN_UNAUTHORIZED,
if (isAuthorized) DOMAIN_UNAUTHORIZED else DOMAIN_AUTHORIZED,
)
get() = ConfigKey.Domain(if (isAuthorized) DOMAIN_AUTHORIZED else DOMAIN_UNAUTHORIZED, null)
override val authUrl: String
get() = "https://${domain}/bounce_login.php"
get() = "https://${getDomain()}/bounce_login.php"
private val ratingPattern = Regex("-?[0-9]+px")
private val authCookies = arrayOf("ipb_member_id", "ipb_pass_hash")
private var updateDm = false
private val nextPages = SparseArrayCompat<Long>()
private val suspiciousContentKey = ConfigKey.ShowSuspiciousContent(true)
override val isAuthorized: Boolean
get() {
@@ -62,7 +54,6 @@ internal class ExHentaiParser(
init {
context.cookieJar.insertCookies(DOMAIN_AUTHORIZED, "nw=1", "sl=dm_2")
context.cookieJar.insertCookies(DOMAIN_UNAUTHORIZED, "nw=1", "sl=dm_2")
paginator.firstPage = 0
}
override suspend fun getListPage(
@@ -72,16 +63,11 @@ internal class ExHentaiParser(
sortOrder: SortOrder,
): List<Manga> {
var search = query?.urlEncoded().orEmpty()
val next = nextPages.get(page, 0L)
if (page > 0 && next == 0L) {
assert(false) { "Page timestamp not found" }
return emptyList()
}
val url = buildString {
append("https://")
append(domain)
append("/?next=")
append(next)
append(getDomain())
append("/?page=")
append(page)
if (!tags.isNullOrEmpty()) {
var fCats = 0
for (tag in tags) {
@@ -102,12 +88,8 @@ internal class ExHentaiParser(
if (updateDm) {
append("&inline_set=dm_e")
}
append("&advsearch=1")
if (config[suspiciousContentKey]) {
append("&f_sh=on")
}
}
val body = webClient.httpGet(url).parseHtml().body()
val body = context.httpGet(url).parseHtml().body()
val root = body.selectFirst("table.itg")
?.selectFirst("tbody")
?: if (updateDm) {
@@ -117,7 +99,6 @@ internal class ExHentaiParser(
return getListPage(page, query, tags, sortOrder)
}
updateDm = false
nextPages[page + 1] = getNextTimestamp(body)
return root.children().mapNotNull { tr ->
if (tr.childrenSize() != 2) return@mapNotNull null
val (td1, td2) = tr.children()
@@ -151,7 +132,7 @@ internal class ExHentaiParser(
}
override suspend fun getDetails(manga: Manga): Manga {
val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml()
val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml()
val root = doc.body().selectFirstOrThrow("div.gm")
val cover = root.getElementById("gd1")?.children()?.first()
val title = root.getElementById("gd2")
@@ -195,13 +176,14 @@ internal class ExHentaiParser(
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml()
val doc = context.httpGet(chapter.url.toAbsoluteUrl(getDomain())).parseHtml()
val root = doc.body().requireElementById("gdt")
return root.select("a").map { a ->
val url = a.attrAsRelativeUrl("href")
MangaPage(
id = generateUid(url),
url = url,
referer = a.absUrl("href"),
preview = null,
source = source,
)
@@ -209,12 +191,12 @@ internal class ExHentaiParser(
}
override suspend fun getPageUrl(page: MangaPage): String {
val doc = webClient.httpGet(page.url.toAbsoluteUrl(domain)).parseHtml()
val doc = context.httpGet(page.url.toAbsoluteUrl(getDomain())).parseHtml()
return doc.body().requireElementById("img").attrAsAbsoluteUrl("src")
}
override suspend fun getAvailableTags(): Set<MangaTag> {
val doc = webClient.httpGet("https://${domain}").parseHtml()
override suspend fun getTags(): Set<MangaTag> {
val doc = context.httpGet("https://${getDomain()}").parseHtml()
val root = doc.body().requireElementById("searchbox").selectFirstOrThrow("table")
return root.select("div.cs").mapNotNullToSet { div ->
val id = div.id().substringAfterLast('_').toIntOrNull()
@@ -228,7 +210,7 @@ internal class ExHentaiParser(
}
override suspend fun getUsername(): String {
val doc = webClient.httpGet("https://forums.$DOMAIN_UNAUTHORIZED/").parseHtml().body()
val doc = context.httpGet("https://forums.$DOMAIN_UNAUTHORIZED/").parseHtml().body()
val username = doc.getElementById("userlinks")
?.getElementsByAttributeValueContaining("href", "showuser=")
?.firstOrNull()
@@ -241,11 +223,6 @@ internal class ExHentaiParser(
return username
}
override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) {
super.onCreateConfig(keys)
keys.add(suspiciousContentKey)
}
private fun isAuthorized(domain: String): Boolean {
val cookies = context.cookieJar.getCookies(domain).mapToSet { x -> x.name }
return authCookies.all { it in cookies }
@@ -299,12 +276,4 @@ internal class ExHentaiParser(
val num = className.drop(2).toIntOrNull(16) ?: return null
return 2.0.pow(num).toInt().toString()
}
private fun getNextTimestamp(root: Element): Long {
return root.getElementById("unext")
?.attrAsAbsoluteUrlOrNull("href")
?.toHttpUrlOrNull()
?.queryParameter("next")
?.toLongOrNull() ?: 1
}
}
}

View File

@@ -1,4 +1,4 @@
package org.koitharu.kotatsu.parsers.site.all
package org.koitharu.kotatsu.parsers.site
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.async
@@ -20,17 +20,16 @@ private const val PAGE_SIZE = 20
private const val CHAPTERS_FIRST_PAGE_SIZE = 120
private const val CHAPTERS_MAX_PAGE_SIZE = 500
private const val CHAPTERS_PARALLELISM = 3
private const val CHAPTERS_MAX_COUNT = 10_000 // strange api behavior, looks like a bug
private const val CONTENT_RATING =
"contentRating[]=safe&contentRating[]=suggestive&contentRating[]=erotica&contentRating[]=pornographic"
private const val LOCALE_FALLBACK = "en"
@MangaSourceParser("MANGADEX", "MangaDex")
internal class MangaDexParser(context: MangaLoaderContext) : MangaParser(context, MangaSource.MANGADEX) {
internal class MangaDexParser(override val context: MangaLoaderContext) : MangaParser(MangaSource.MANGADEX) {
override val configKeyDomain = ConfigKey.Domain("mangadex.org")
override val configKeyDomain = ConfigKey.Domain("mangadex.org", null)
override val availableSortOrders: EnumSet<SortOrder> = EnumSet.of(
override val sortOrders: EnumSet<SortOrder> = EnumSet.of(
SortOrder.UPDATED,
SortOrder.ALPHABETICAL,
SortOrder.NEWEST,
@@ -43,7 +42,7 @@ internal class MangaDexParser(context: MangaLoaderContext) : MangaParser(context
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val domain = domain
val domain = getDomain()
val url = buildString {
append("https://api.")
append(domain)
@@ -76,7 +75,7 @@ internal class MangaDexParser(context: MangaLoaderContext) : MangaParser(context
},
)
}
val json = webClient.httpGet(url).parseJson().getJSONArray("data")
val json = context.httpGet(url).parseJson().getJSONArray("data")
return json.mapJSON { jo ->
val id = jo.getString("id")
val attrs = jo.getJSONObject("attributes")
@@ -124,43 +123,70 @@ internal class MangaDexParser(context: MangaLoaderContext) : MangaParser(context
}
override suspend fun getDetails(manga: Manga): Manga = coroutineScope {
val domain = domain
val domain = getDomain()
val mangaId = manga.url.removePrefix("/")
val attrsDeferred = async {
webClient.httpGet(
context.httpGet(
"https://api.$domain/manga/${mangaId}?includes[]=artist&includes[]=author&includes[]=cover_art",
).parseJson().getJSONObject("data").getJSONObject("attributes")
}
val feedDeferred = async { loadChapters(mangaId) }
val mangaAttrs = attrsDeferred.await()
val feed = feedDeferred.await()
// 2022-01-02T00:27:11+00:00
val dateFormat = SimpleDateFormat(
"yyyy-MM-dd'T'HH:mm:ss'+00:00'",
Locale.ROOT,
)
manga.copy(
description = mangaAttrs.optJSONObject("description")?.selectByLocale()
description = mangaAttrs.getJSONObject("description").selectByLocale()
?: manga.description,
chapters = mapChapters(feed),
chapters = feed.mapChapters { _, jo ->
val id = jo.getString("id")
val attrs = jo.getJSONObject("attributes")
if (!attrs.isNull("externalUrl")) {
return@mapChapters null
}
val locale = attrs.getStringOrNull("translatedLanguage")?.let { Locale.forLanguageTag(it) }
val relations = jo.getJSONArray("relationships").associateByKey("type")
val number = attrs.getIntOrDefault("chapter", 0)
MangaChapter(
id = generateUid(id),
name = attrs.getStringOrNull("title")?.takeUnless(String::isEmpty)
?: "Chapter #$number",
number = number,
url = id,
scanlator = relations["scanlation_group"]?.getStringOrNull("name"),
uploadDate = dateFormat.tryParse(attrs.getString("publishAt")),
branch = locale?.getDisplayName(locale)?.toTitleCase(locale),
source = source,
)
},
)
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val domain = domain
val chapterJson = webClient.httpGet("https://api.$domain/at-home/server/${chapter.url}?forcePort443=false")
val domain = getDomain()
val chapterJson = context.httpGet("https://api.$domain/at-home/server/${chapter.url}?forcePort443=false")
.parseJson()
.getJSONObject("chapter")
val pages = chapterJson.getJSONArray("data")
val prefix = "https://uploads.$domain/data/${chapterJson.getString("hash")}/"
val referer = "https://$domain/"
return List(pages.length()) { i ->
val url = prefix + pages.getString(i)
MangaPage(
id = generateUid(url),
url = url,
referer = referer,
preview = null, // TODO prefix + dataSaver.getString(i),
source = source,
)
}
}
override suspend fun getAvailableTags(): Set<MangaTag> {
val tags = webClient.httpGet("https://api.${domain}/manga/tag").parseJson()
override suspend fun getTags(): Set<MangaTag> {
val tags = context.httpGet("https://api.${getDomain()}/manga/tag").parseJson()
.getJSONArray("data")
return tags.mapJSONToSet { jo ->
MangaTag(
@@ -188,7 +214,7 @@ internal class MangaDexParser(context: MangaLoaderContext) : MangaParser(context
return firstPage.data
}
val tail = coroutineScope {
val leftCount = firstPage.total.coerceAtMost(CHAPTERS_MAX_COUNT) - firstPage.size
val leftCount = firstPage.total - firstPage.size
val pages = (leftCount / CHAPTERS_MAX_PAGE_SIZE.toFloat()).toIntUp()
val dispatcher = Dispatchers.Default.limitedParallelism(CHAPTERS_PARALLELISM)
List(pages) { page ->
@@ -205,25 +231,20 @@ internal class MangaDexParser(context: MangaLoaderContext) : MangaParser(context
}
private suspend fun loadChapters(mangaId: String, offset: Int, limit: Int): Chapters {
val limitedLimit = when {
offset >= CHAPTERS_MAX_COUNT -> return Chapters(emptyList(), CHAPTERS_MAX_COUNT)
offset + limit > CHAPTERS_MAX_COUNT -> CHAPTERS_MAX_COUNT - offset
else -> limit
}
val url = buildString {
append("https://api.")
append(domain)
append(getDomain())
append("/manga/")
append(mangaId)
append("/feed")
append("?limit=")
append(limitedLimit)
append(limit)
append("&includes[]=scanlation_group&order[volume]=asc&order[chapter]=asc&offset=")
append(offset)
append('&')
append(CONTENT_RATING)
}
val json = webClient.httpGet(url).parseJson()
val json = context.httpGet(url).parseJson()
if (json.getString("result") == "ok") {
return Chapters(
data = json.optJSONArray("data")?.toJSONList().orEmpty(),
@@ -237,48 +258,6 @@ internal class MangaDexParser(context: MangaLoaderContext) : MangaParser(context
}
}
private fun mapChapters(list: List<JSONObject>): List<MangaChapter> {
// 2022-01-02T00:27:11+00:00
val dateFormat = SimpleDateFormat(
"yyyy-MM-dd'T'HH:mm:ss'+00:00'",
Locale.ROOT,
)
val chaptersBuilder = ChaptersListBuilder(list.size)
val branchedChapters = HashMap<String?, HashMap<Float, MangaChapter>>()
for (jo in list) {
val id = jo.getString("id")
val attrs = jo.getJSONObject("attributes")
if (!attrs.isNull("externalUrl")) {
continue
}
val number = jo.getJSONObject("attributes").getFloatOrDefault("chapter", 0f)
val locale = attrs.getStringOrNull("translatedLanguage")?.let { Locale.forLanguageTag(it) }
val lc = locale?.getDisplayName(locale)?.toTitleCase(locale)
val relations = jo.getJSONArray("relationships").associateByKey("type")
val team = relations["scanlation_group"]?.getJSONObject("attributes")?.getStringOrNull("name")
?.takeUnless { it.isBlank() }
val branch = (list.indices).firstNotNullOf { i ->
val b = if (i == 0) lc else "$lc ($i)"
if (branchedChapters[b]?.get(number) == null) b else null
}
val chapter = MangaChapter(
id = generateUid(id),
name = attrs.getStringOrNull("title")?.takeUnless(String::isEmpty)
?: "Chapter #${number.toString().removeSuffix(".0")}",
number = if (number <= 0f) (branchedChapters[branch]?.size?.plus(1) ?: 0) else number.toInt(),
url = id,
scanlator = team,
uploadDate = dateFormat.tryParse(attrs.getString("publishAt")),
branch = branch,
source = source,
)
if (chaptersBuilder.add(chapter)) {
branchedChapters.getOrPut(branch, ::HashMap)[number] = chapter
}
}
return chaptersBuilder.toList()
}
private class Chapters(
val data: List<JSONObject>,
val total: Int,
@@ -287,4 +266,4 @@ internal class MangaDexParser(context: MangaLoaderContext) : MangaParser(context
val size: Int
get() = data.size
}
}
}

View File

@@ -1,4 +1,4 @@
package org.koitharu.kotatsu.parsers.site.uk
package org.koitharu.kotatsu.parsers.site
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
@@ -12,20 +12,16 @@ import java.util.*
private const val DEF_BRANCH_NAME = "Основний переклад"
@MangaSourceParser("MANGAINUA", "MANGA/in/UA", "uk")
class MangaInUaParser(context: MangaLoaderContext) : PagedMangaParser(
context = context,
class MangaInUaParser(override val context: MangaLoaderContext) : PagedMangaParser(
source = MangaSource.MANGAINUA,
pageSize = 24,
searchPageSize = 10,
) {
override val availableSortOrders: Set<SortOrder> = setOf(SortOrder.UPDATED)
override val sortOrders: Set<SortOrder>
get() = Collections.singleton(SortOrder.UPDATED)
override val configKeyDomain: ConfigKey.Domain = ConfigKey.Domain("manga.in.ua")
private val userHashRegex by lazy {
Regex("site_login_hash\\s*=\\s*\'([^\']+)\'", RegexOption.IGNORE_CASE)
}
override val configKeyDomain: ConfigKey.Domain = ConfigKey.Domain("manga.in.ua", null)
override suspend fun getListPage(
page: Int,
@@ -34,17 +30,22 @@ class MangaInUaParser(context: MangaLoaderContext) : PagedMangaParser(
sortOrder: SortOrder,
): List<Manga> {
val url = when {
!query.isNullOrEmpty() -> ("/index.php?do=search&subaction=search&search_start=$page&full_search=1&story=$query&titleonly=3").toAbsoluteUrl(
domain,
)
!query.isNullOrEmpty() -> (
"/index.php?do=search" +
"&subaction=search" +
"&search_start=$page" +
"&full_search=1" +
"&story=$query" +
"&titleonly=3"
).toAbsoluteUrl(getDomain())
tags.isNullOrEmpty() -> "/mangas/page/$page".toAbsoluteUrl(domain)
tags.isNullOrEmpty() -> "/mangas/page/$page".toAbsoluteUrl(getDomain())
tags.size == 1 -> "${tags.first().key}/page/$page"
tags.size > 1 -> throw IllegalArgumentException("This source supports only 1 genre")
else -> "/mangas/page/$page".toAbsoluteUrl(domain)
else -> "/mangas/page/$page".toAbsoluteUrl(getDomain())
}
val doc = webClient.httpGet(url).parseHtml()
val container = doc.body().requireElementById("site-content")
val doc = context.httpGet(url).parseHtml()
val container = doc.body().requireElementById("dle-content")
val items = container.select("div.col-6")
return items.mapNotNull { item ->
val href = item.selectFirst("a")?.attrAsRelativeUrl("href") ?: return@mapNotNull null
@@ -56,44 +57,33 @@ class MangaInUaParser(context: MangaLoaderContext) : PagedMangaParser(
}.orEmpty(),
altTitle = null,
author = null,
rating = item.selectFirst("div.card__short-rate--num")?.text()?.toFloatOrNull()?.div(10F)
?: RATING_UNKNOWN,
rating = item.selectFirst("div.card__short-rate--num")
?.text()
?.toFloatOrNull()
?.div(10F) ?: RATING_UNKNOWN,
url = href,
isNsfw = item.selectFirst("ul.card__list")?.select("li")?.lastOrNull()?.text() == "18+",
tags = runCatching {
item.selectFirst("div.card__category")?.select("a")?.mapToSet {
MangaTag(
title = it.ownText(),
key = it.attrOrThrow("href").removeSuffix("/"),
key = it.attr("href").removeSuffix("/"),
source = source,
)
}
}.getOrNull().orEmpty(),
state = null,
publicUrl = href.toAbsoluteUrl(container.host ?: domain),
publicUrl = href.toAbsoluteUrl(container.host ?: getDomain()),
source = source,
)
}
}
override suspend fun getDetails(manga: Manga): Manga {
val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml()
val root = doc.body().requireElementById("site-content")
val linkToComics = root.requireElementById("linkstocomics")
val userHash = doc.select("script").firstNotNullOf { script ->
userHashRegex.find(script.html())?.groupValues?.getOrNull(1)
}
val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml()
val root = doc.body().requireElementById("dle-content")
val dateFormat = SimpleDateFormat("dd.MM.yyyy", Locale.US)
val chapterNodes = webClient.httpPost(
"https://$domain/engine/ajax/controller.php?mod=load_chapters",
mapOf(
"action" to "show",
"news_id" to linkToComics.attrOrThrow("data-news_id"),
"news_category" to linkToComics.attrOrThrow("data-news_category"),
"this_link" to "",
"user_hash" to userHash,
),
).parseHtml().select(".ltcitems")
val chapterNodes = root.selectFirstOrThrow(".linkstocomics").select(".ltcitems")
var prevChapterName: String? = null
var i = 0
return manga.copy(
@@ -101,7 +91,8 @@ class MangaInUaParser(context: MangaLoaderContext) : PagedMangaParser(
largeCoverUrl = root.selectFirst("div.item__full-sidebar--poster")?.selectFirst("img")
?.attrAsAbsoluteUrlOrNull("src"),
chapters = chapterNodes.mapChapters { _, item ->
val href = item?.selectFirst("a")?.attrAsRelativeUrlOrNull("href") ?: return@mapChapters null
val href = item?.selectFirst("a")?.attrAsRelativeUrlOrNull("href")
?: return@mapChapters null
val isAlternative = item.styleValueOrNull("background") != null
val name = item.selectFirst("a")?.text().orEmpty()
if (!isAlternative) i++
@@ -129,16 +120,9 @@ class MangaInUaParser(context: MangaLoaderContext) : PagedMangaParser(
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(domain)
val doc = webClient.httpGet(fullUrl).parseHtml()
val userHash = doc.select("script").firstNotNullOf { script ->
userHashRegex.find(script.html())?.groupValues?.getOrNull(1)
}
val ajaxUrl = urlBuilder().addPathSegment("engine").addPathSegment("ajax").addPathSegment("controller.php")
.addEncodedQueryParameter("mod", "load_chapters_image")
.addQueryParameter("news_id", doc.requireElementById("comics").attrOrThrow("data-news_id"))
.addEncodedQueryParameter("action", "show").addQueryParameter("user_hash", userHash).build()
val root = webClient.httpGet(ajaxUrl).parseHtml().root()
val fullUrl = chapter.url.toAbsoluteUrl(getDomain())
val doc = context.httpGet(fullUrl).parseHtml()
val root = doc.body().requireElementById("comics").selectFirstOrThrow("ul.xfieldimagegallery")
return root.select("li").map { ul ->
val img = ul.selectFirstOrThrow("img")
val url = img.attrAsAbsoluteUrl("data-src")
@@ -146,22 +130,23 @@ class MangaInUaParser(context: MangaLoaderContext) : PagedMangaParser(
id = generateUid(url),
url = url,
preview = null,
referer = fullUrl,
source = source,
)
}
}
override suspend fun getAvailableTags(): Set<MangaTag> {
val domain = domain
val doc = webClient.httpGet("https://$domain/mangas").parseHtml()
override suspend fun getTags(): Set<MangaTag> {
val domain = getDomain()
val doc = context.httpGet("https://$domain/mangas").parseHtml()
val root = doc.body().requireElementById("menu_1").selectFirstOrThrow("div.menu__wrapper")
return root.select("li").mapNotNullToSet { li ->
val a = li.selectFirst("a") ?: return@mapNotNullToSet null
MangaTag(
title = a.ownText(),
key = a.attrOrThrow("href").removeSuffix("/"),
key = a.attr("href").removeSuffix("/"),
source = source,
)
}
}
}
}

View File

@@ -0,0 +1,187 @@
package org.koitharu.kotatsu.parsers.site
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import java.text.SimpleDateFormat
import java.util.*
@Deprecated("Permanently shutdown")
@MangaSourceParser("MANGAOWL", "MangaOwl", "en")
internal class MangaOwlParser(override val context: MangaLoaderContext) : MangaParser(MangaSource.MANGAOWL) {
override val configKeyDomain = ConfigKey.Domain("mangaowls.com", null)
override val sortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.POPULARITY,
SortOrder.NEWEST,
SortOrder.UPDATED,
)
private val regexNsfw = Regex("(yaoi)|(yuri)|(smut)|(mature)|(adult)", RegexOption.IGNORE_CASE)
override suspend fun getList(
offset: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val page = (offset / 36f).toIntUp().inc()
val link = buildString {
append("https://")
append(getDomain())
when {
!query.isNullOrEmpty() -> {
append("/search/$page?search=")
append(query.urlEncoded())
}
!tags.isNullOrEmpty() -> {
for (tag in tags) {
append(tag.key)
}
append("/$page?type=${getAlternativeSortKey(sortOrder)}")
}
else -> {
append("/${getSortKey(sortOrder)}/$page")
}
}
}
val doc = context.httpGet(link).parseHtml()
val slides = doc.body().selectOrThrow("ul.slides")
val items = slides.select("div.col-md-2")
return items.mapNotNull { item ->
val href = item.selectFirst("h6 a")?.attrAsRelativeUrlOrNull("href") ?: return@mapNotNull null
Manga(
id = generateUid(href),
title = item.selectFirst("h6 a")?.text() ?: return@mapNotNull null,
coverUrl = item.select("div.img-responsive").attr("abs:data-background-image"),
altTitle = null,
author = null,
rating = runCatching {
item.selectFirst("div.block-stars")
?.text()
?.toFloatOrNull()
?.div(10f)
}.getOrNull() ?: RATING_UNKNOWN,
url = href,
isNsfw = false,
tags = emptySet(),
state = null,
publicUrl = href.toAbsoluteUrl(getDomain()),
source = source,
)
}
}
override suspend fun getDetails(manga: Manga): Manga {
val doc = context.httpGet(manga.publicUrl).parseHtml()
val info = doc.body().selectFirstOrThrow("div.single_detail")
val table = doc.body().selectFirstOrThrow("div.single-grid-right")
val dateFormat = SimpleDateFormat("MM/dd/yyyy", Locale.US)
val trRegex = "window\\['tr'] = '([^']*)';".toRegex(RegexOption.IGNORE_CASE)
val trElement = doc.getElementsByTag("script").find { trRegex.find(it.data()) != null }
?: doc.parseFailed("Oops, tr not found")
val tr = trRegex.find(trElement.data())!!.groups[1]!!.value
val s = context.encodeBase64(getDomain().toByteArray())
var isNsfw = manga.isNsfw
val parsedTags = info.select("div.col-xs-12.col-md-8.single-right-grid-right > p > a[href*=genres]")
.mapNotNullToSet {
val a = it.selectFirst("a") ?: return@mapNotNullToSet null
val name = a.text()
if (!isNsfw && isNsfwGenre(name)) {
isNsfw = true
}
MangaTag(
title = name.toTitleCase(),
key = a.attr("href"),
source = source,
)
}
return manga.copy(
description = info.selectFirst(".description")?.html(),
largeCoverUrl = info.select("img").first()?.let { img ->
if (img.hasAttr("data-src")) img.attr("abs:data-src") else img.attr("abs:src")
},
isNsfw = isNsfw,
author = info.selectFirst("p.fexi_header_para a.author_link")?.text(),
state = parseStatus(info.select("p.fexi_header_para:contains(status)").first()?.ownText()),
tags = manga.tags + parsedTags,
chapters = table.select("div.table.table-chapter-list").select("li.list-group-item.chapter_list")
.asReversed().mapChapters { i, li ->
val a = li.select("a")
val href = a.attr("data-href").ifEmpty {
li.parseFailed("Link is missing")
}
MangaChapter(
id = generateUid(href),
name = a.select("label").text(),
number = i + 1,
url = "$href?tr=$tr&s=$s",
scanlator = null,
branch = null,
uploadDate = dateFormat.tryParse(li.selectFirst("small:last-of-type")?.text()),
source = MangaSource.MANGAOWL,
)
},
)
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(getDomain())
val doc = context.httpGet(fullUrl).parseHtml()
val root = doc.body().selectOrThrow("div.item img.owl-lazy")
return root.map { div ->
val url = div?.attrAsRelativeUrlOrNull("data-src") ?: doc.parseFailed("Page image not found")
MangaPage(
id = generateUid(url),
url = url,
preview = null,
referer = url,
source = MangaSource.MANGAOWL,
)
}
}
private fun parseStatus(status: String?) = when {
status == null -> null
status.contains("Ongoing") -> MangaState.ONGOING
status.contains("Completed") -> MangaState.FINISHED
else -> null
}
override suspend fun getTags(): Set<MangaTag> {
val doc = context.httpGet("https://${getDomain()}/").parseHtml()
val root = doc.body().select("ul.dropdown-menu.multi-column.columns-3").select("li")
return root.mapToSet { p ->
val a = p.selectFirstOrThrow("a")
MangaTag(
title = a.text().toTitleCase(),
key = a.attr("href"),
source = source,
)
}
}
private fun getSortKey(sortOrder: SortOrder) =
when (sortOrder) {
SortOrder.POPULARITY -> "popular"
SortOrder.NEWEST -> "new_release"
SortOrder.UPDATED -> "lastest"
else -> "lastest"
}
private fun getAlternativeSortKey(sortOrder: SortOrder) =
when (sortOrder) {
SortOrder.POPULARITY -> "0"
SortOrder.NEWEST -> "2"
SortOrder.UPDATED -> "3"
else -> "3"
}
private fun isNsfwGenre(name: String): Boolean = regexNsfw.containsMatchIn(name)
}

View File

@@ -1,4 +1,4 @@
package org.koitharu.kotatsu.parsers.site.en
package org.koitharu.kotatsu.parsers.site
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser
@@ -11,11 +11,11 @@ import java.text.SimpleDateFormat
import java.util.*
@MangaSourceParser("MANGATOWN", "MangaTown", "en")
internal class MangaTownParser(context: MangaLoaderContext) : MangaParser(context, MangaSource.MANGATOWN) {
internal class MangaTownParser(override val context: MangaLoaderContext) : MangaParser(MangaSource.MANGATOWN) {
override val configKeyDomain = ConfigKey.Domain("www.mangatown.com")
override val configKeyDomain = ConfigKey.Domain("www.mangatown.com", null)
override val availableSortOrders: Set<SortOrder> = EnumSet.of(
override val sortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.ALPHABETICAL,
SortOrder.RATING,
SortOrder.POPULARITY,
@@ -42,18 +42,18 @@ internal class MangaTownParser(context: MangaLoaderContext) : MangaParser(contex
if (offset != 0) {
return emptyList()
}
"/search?name=${query.urlEncoded()}".toAbsoluteUrl(domain)
"/search?name=${query.urlEncoded()}".toAbsoluteUrl(getDomain())
}
tags.isNullOrEmpty() -> "/directory/$page.htm$sortKey".toAbsoluteUrl(domain)
tags.size == 1 -> "/directory/${tags.first().key}/$page.htm$sortKey".toAbsoluteUrl(domain)
tags.isNullOrEmpty() -> "/directory/$page.htm$sortKey".toAbsoluteUrl(getDomain())
tags.size == 1 -> "/directory/${tags.first().key}/$page.htm$sortKey".toAbsoluteUrl(getDomain())
else -> tags.joinToString(
prefix = "/search?page=$page".toAbsoluteUrl(domain),
prefix = "/search?page=$page".toAbsoluteUrl(getDomain()),
) { tag ->
"&genres[${tag.key}]=1"
}
}
val doc = webClient.httpGet(url).parseHtml()
val doc = context.httpGet(url).parseHtml()
val root = doc.body().selectFirstOrThrow("ul.manga_pic_list")
return root.select("li").mapNotNull { li ->
val a = li.selectFirst("a.manga_cover")
@@ -87,13 +87,13 @@ internal class MangaTownParser(context: MangaLoaderContext) : MangaParser(contex
}.orEmpty(),
url = href,
isNsfw = false,
publicUrl = href.toAbsoluteUrl(a.host ?: domain),
publicUrl = href.toAbsoluteUrl(a.host ?: getDomain()),
)
}
}
override suspend fun getDetails(manga: Manga): Manga {
val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml()
val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml()
val root = doc.body().selectFirstOrThrow("section.main")
.selectFirstOrThrow("div.article_content")
val info = root.selectFirst("div.detail_info")?.selectFirst("ul")
@@ -135,8 +135,8 @@ internal class MangaTownParser(context: MangaLoaderContext) : MangaParser(contex
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(domain)
val doc = webClient.httpGet(fullUrl).parseHtml()
val fullUrl = chapter.url.toAbsoluteUrl(getDomain())
val doc = context.httpGet(fullUrl).parseHtml()
val root = doc.body().selectFirstOrThrow("div.page_select")
return root.selectFirstOrThrow("select").selectOrThrow("option").mapNotNull {
val href = it.attrAsRelativeUrlOrNull("value")
@@ -147,18 +147,19 @@ internal class MangaTownParser(context: MangaLoaderContext) : MangaParser(contex
id = generateUid(href),
url = href,
preview = null,
referer = fullUrl,
source = MangaSource.MANGATOWN,
)
}
}
override suspend fun getPageUrl(page: MangaPage): String {
val doc = webClient.httpGet(page.url.toAbsoluteUrl(domain)).parseHtml()
val doc = context.httpGet(page.url.toAbsoluteUrl(getDomain())).parseHtml()
return doc.requireElementById("image").attrAsAbsoluteUrl("src")
}
override suspend fun getAvailableTags(): Set<MangaTag> {
val doc = webClient.httpGet("/directory/".toAbsoluteUrl(domain)).parseHtml()
override suspend fun getTags(): Set<MangaTag> {
val doc = context.httpGet("/directory/".toAbsoluteUrl(getDomain())).parseHtml()
val root = doc.body().selectFirst("aside.right")
?.getElementsContainingOwnText("Genres")
?.first()
@@ -187,7 +188,7 @@ internal class MangaTownParser(context: MangaLoaderContext) : MangaParser(contex
}
private suspend fun bypassLicensedChapters(manga: Manga): List<MangaChapter> {
val doc = webClient.httpGet(manga.url.toAbsoluteUrl(getDomain("m"))).parseHtml()
val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain("m"))).parseHtml()
val list = doc.body().selectFirst("ul.detail-ch-list") ?: return emptyList()
val dateFormat = SimpleDateFormat("MMM dd,yyyy", Locale.US)
return list.select("li").asReversed().mapIndexedNotNull { i, li ->
@@ -213,4 +214,4 @@ internal class MangaTownParser(context: MangaLoaderContext) : MangaParser(contex
}
private fun String.parseTagKey() = split('/').findLast { regexTag matches it }
}
}

View File

@@ -0,0 +1,188 @@
package org.koitharu.kotatsu.parsers.site
import androidx.collection.ArraySet
import kotlinx.coroutines.async
import kotlinx.coroutines.awaitAll
import kotlinx.coroutines.coroutineScope
import org.jsoup.nodes.Element
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import java.text.SimpleDateFormat
import java.util.*
@MangaSourceParser("NHENTAI", "N-Hentai")
class NHentaiParser(override val context: MangaLoaderContext) : PagedMangaParser(MangaSource.NHENTAI, pageSize = 25) {
override val configKeyDomain: ConfigKey.Domain
get() = ConfigKey.Domain("nhentai.net", null)
override val sortOrders: Set<SortOrder>
get() = EnumSet.of(SortOrder.NEWEST, SortOrder.POPULARITY)
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
if (query.isNullOrEmpty() && tags != null && tags.size > 1) {
return getListPage(page, buildQuery(tags), emptySet(), sortOrder)
}
val domain = getDomain()
val url = buildString {
append("https://")
append(domain)
if (!query.isNullOrEmpty()) {
append("/search/?q=")
append(query.urlEncoded())
append("&page=")
append(page)
if (sortOrder == SortOrder.POPULARITY) {
append("&sort=popular")
}
} else {
append('/')
if (!tags.isNullOrEmpty()) {
val tag = tags.single()
append("tag/")
append(tag.key)
append('/')
if (sortOrder == SortOrder.POPULARITY) {
append("popular")
}
append("?page=")
append(page)
} else {
if (sortOrder == SortOrder.POPULARITY) {
append("?sort=popular&page=")
} else {
append("?page=")
}
append(page)
}
}
}
val root = context.httpGet(url).parseHtml().body().requireElementById("content")
.selectLastOrThrow("div.index-container")
val regexBrackets = Regex("\\[[^]]+]|\\([^)]+\\)")
val regexSpaces = Regex("\\s+")
return root.select(".gallery").map { div ->
val a = div.selectFirstOrThrow("a.cover")
val href = a.attrAsRelativeUrl("href")
val img = div.selectFirstOrThrow("img")
val title = div.selectFirstOrThrow(".caption").text()
Manga(
id = generateUid(href),
title = title.replace(regexBrackets, "")
.replace(regexSpaces, " ")
.trim(),
altTitle = null,
url = href,
publicUrl = href.toAbsoluteUrl(domain),
rating = RATING_UNKNOWN,
isNsfw = true,
coverUrl = img.attrAsAbsoluteUrlOrNull("data-src")
?: img.attrAsAbsoluteUrl("src"),
tags = setOf(),
state = null,
author = null,
largeCoverUrl = null,
description = null,
chapters = listOf(),
source = source,
)
}
}
override suspend fun getDetails(manga: Manga): Manga {
val root = context.httpGet(
url = manga.url.toAbsoluteUrl(getDomain()),
).parseHtml().body().requireElementById("bigcontainer")
val img = root.requireElementById("cover").selectFirstOrThrow("img")
val tagContainers = root.requireElementById("tags").select(".tag-container")
val dateFormat = SimpleDateFormat(
"yyyy-MM-dd'T'HH:mm:ss.SSSSSS'+00:00'",
Locale.ROOT,
)
return manga.copy(
tags = tagContainers.find { x -> x.ownText() == "Tags:" }?.parseTags() ?: manga.tags,
author = tagContainers.find { x -> x.ownText() == "Artists:" }
?.selectFirst("span.name")?.text()?.toCamelCase(),
largeCoverUrl = img.attrAsAbsoluteUrlOrNull("data-src")
?: img.attrAsAbsoluteUrl("src"),
description = null,
chapters = listOf(
MangaChapter(
id = manga.id,
name = manga.title,
number = 1,
url = manga.url,
scanlator = null,
uploadDate = dateFormat.tryParse(
tagContainers.find { x -> x.ownText() == "Uploaded:" }
?.selectFirst("time")
?.attr("datetime"),
),
branch = null,
source = source,
),
),
)
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val url = chapter.url.toAbsoluteUrl(getDomain())
val root = context.httpGet(url).parseHtml().requireElementById("thumbnail-container")
return root.select(".thumb-container").map { div ->
val a = div.selectFirstOrThrow("a")
val img = div.selectFirstOrThrow("img")
val href = a.attrAsRelativeUrl("href")
MangaPage(
id = generateUid(href),
url = href,
referer = url,
preview = img.attrAsAbsoluteUrlOrNull("data-src")
?: img.attrAsAbsoluteUrl("src"),
source = source,
)
}
}
override suspend fun getPageUrl(page: MangaPage): String {
val root = context.httpGet(page.url.toAbsoluteUrl(getDomain())).parseHtml().body()
.requireElementById("image-container")
return root.selectFirstOrThrow("img").attrAsAbsoluteUrl("src")
}
override suspend fun getTags(): Set<MangaTag> {
return coroutineScope {
// parse first 3 pages of tags
(1..3).map { page ->
async { getTags(page) }
}
}.awaitAll().flattenTo(ArraySet(360))
}
private suspend fun getTags(page: Int): Set<MangaTag> {
val root = context.httpGet("https://${getDomain()}/tags/popular?page=$page").parseHtml().body()
.getElementById("tag-container")
return root?.parseTags().orEmpty()
}
private fun Element.parseTags() = select("a.tag").mapToSet { a ->
val href = a.attr("href").removeSuffix('/')
MangaTag(
title = a.selectFirstOrThrow(".name").text().toTitleCase(),
key = href.substringAfterLast('/'),
source = source,
)
}
private fun buildQuery(tags: Collection<MangaTag>) = tags.joinToString(separator = " ") { tag ->
"tag:\"${tag.key}\""
}
}

View File

@@ -1,12 +1,10 @@
package org.koitharu.kotatsu.parsers.site.vi
package org.koitharu.kotatsu.parsers.site
import androidx.collection.ArrayMap
import androidx.collection.ArraySet
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.*
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.exception.NotFoundException
import org.koitharu.kotatsu.parsers.model.*
@@ -15,18 +13,13 @@ import java.text.SimpleDateFormat
import java.util.*
@MangaSourceParser("NETTRUYEN", "NetTruyen", "vi")
class NetTruyenParser(context: MangaLoaderContext) :
PagedMangaParser(context, MangaSource.NETTRUYEN, pageSize = 36) {
class NetTruyenParser(override val context: MangaLoaderContext) :
PagedMangaParser(MangaSource.NETTRUYEN, pageSize = 36) {
override val configKeyDomain: ConfigKey.Domain = ConfigKey.Domain(
"www.nettruyenlive.com",
"www.nettruyenio.com",
"www.nettruyento.com",
"nettruyento.com",
"nettruyenin.com",
)
override val configKeyDomain: ConfigKey.Domain
get() = ConfigKey.Domain("www.nettruyenme.com", null)
override val availableSortOrders: Set<SortOrder>
override val sortOrders: Set<SortOrder>
get() = EnumSet.of(SortOrder.UPDATED, SortOrder.POPULARITY, SortOrder.NEWEST, SortOrder.RATING)
private val mutex = Mutex()
@@ -34,13 +27,13 @@ class NetTruyenParser(context: MangaLoaderContext) :
private var tagCache: ArrayMap<String, MangaTag>? = null
override suspend fun getDetails(manga: Manga): Manga {
val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml()
val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml()
val rating = doc.selectFirst("span[itemprop=ratingValue]")
?.ownText()
?.toFloatOrNull() ?: 0f
val chapterElements = doc.getElementById("nt_listchapter")?.select("ul > li") ?: doc.parseFailed()
val chapters = chapterElements.mapChapters(reversed = true) { index, element ->
val chapters = chapterElements.asReversed().mapChapters { index, element ->
val a = element.selectFirst("div.chapter > a") ?: return@mapChapters null
val relativeUrl = a.attrAsRelativeUrlOrNull("href") ?: return@mapChapters null
val timeText = element.selectFirst("div.col-xs-4.text-center.no-wrap.small")?.text()
@@ -96,7 +89,6 @@ class NetTruyenParser(context: MangaLoaderContext) :
val currentYear = calendar.get(Calendar.YEAR).toString().takeLast(2)
"$relativeDate/$currentYear"
}
3 -> relativeDate
else -> return 0L
}
@@ -117,7 +109,7 @@ class NetTruyenParser(context: MangaLoaderContext) :
val isSearching = !query.isNullOrEmpty()
val url = buildString {
append("https://")
append(domain)
append(getDomain())
if (isSearching) {
append("/tim-truyen?keyword=")
append(query!!.urlEncoded())
@@ -132,7 +124,7 @@ class NetTruyenParser(context: MangaLoaderContext) :
}
val response = if (isSearching) {
val result = runCatchingCancellable { webClient.httpGet(url) }
val result = runCatching { context.httpGet(url) }
val exception = result.exceptionOrNull()
if (exception is NotFoundException) {
return emptyList()
@@ -140,7 +132,7 @@ class NetTruyenParser(context: MangaLoaderContext) :
result.getOrThrow()
} else {
webClient.httpGet(url)
context.httpGet(url)
}
val itemsElements = response.parseHtml()
@@ -163,7 +155,7 @@ class NetTruyenParser(context: MangaLoaderContext) :
id = generateUid(slug),
title = tooltipElement.selectFirst("div.title")?.text().orEmpty(),
altTitle = null,
url = absUrl.toRelativeUrl(domain),
url = absUrl.toRelativeUrl(getDomain()),
publicUrl = absUrl,
rating = RATING_UNKNOWN,
isNsfw = false,
@@ -181,20 +173,21 @@ class NetTruyenParser(context: MangaLoaderContext) :
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val pageElements = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml()
val pageElements = context.httpGet(chapter.url.toAbsoluteUrl(getDomain())).parseHtml()
.select("div.reading-detail.box_doc > div img")
return pageElements.map { element ->
val url = element.attrAsAbsoluteUrl("data-original")
MangaPage(
id = generateUid(url),
url = url,
referer = getDomain(),
preview = null,
source = source,
)
}
}
override suspend fun getAvailableTags(): Set<MangaTag> {
override suspend fun getTags(): Set<MangaTag> {
val map = getOrCreateTagMap()
val tagSet = ArraySet<MangaTag>(map.size)
for (entry in map) {
@@ -206,7 +199,7 @@ class NetTruyenParser(context: MangaLoaderContext) :
private suspend fun getOrCreateTagMap(): ArrayMap<String, MangaTag> = mutex.withLock {
tagCache?.let { return@withLock it }
val doc = webClient.httpGet("/tim-truyen-nang-cao".toAbsoluteUrl(domain)).parseHtml()
val doc = context.httpGet("/tim-truyen-nang-cao".toAbsoluteUrl(getDomain())).parseHtml()
val tagItems = doc.select("div.genre-item")
val result = ArrayMap<String, MangaTag>(tagItems.size)
for (item in tagItems) {

View File

@@ -1,26 +1,20 @@
package org.koitharu.kotatsu.parsers.site.ja
package org.koitharu.kotatsu.parsers.site
import okhttp3.Headers
import org.koitharu.kotatsu.parsers.*
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.exception.AuthRequiredException
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.network.UserAgents
import org.koitharu.kotatsu.parsers.util.*
import java.util.*
private const val STATUS_ONGOING = "連載"
private const val STATUS_FINISHED = "完結"
@MangaSourceParser("NICOVIDEO_SEIGA", "NicoVideo Seiga", "ja")
class NicovideoSeigaParser(context: MangaLoaderContext) :
MangaParser(context, MangaSource.NICOVIDEO_SEIGA),
@MangaSourceParser("NICOVIDEO_SEIGA", "Nicovideo Seiga", "ja")
class NicovideoSeigaParser(override val context: MangaLoaderContext) :
MangaParser(MangaSource.NICOVIDEO_SEIGA),
MangaParserAuthProvider {
override val headers: Headers = Headers.Builder()
.add("User-Agent", UserAgents.CHROME_DESKTOP)
.build()
override val authUrl: String
get() = "https://${getDomain("account")}/login?site=seiga"
@@ -30,16 +24,16 @@ class NicovideoSeigaParser(context: MangaLoaderContext) :
}
override suspend fun getUsername(): String {
val body = webClient.httpGet("https://${getDomain("app")}/my/apps").parseHtml().body()
val body = context.httpGet("https://${getDomain("app")}/my/apps").parseHtml().body()
return body.selectFirst("#userinfo > div > div > strong")?.text() ?: throw AuthRequiredException(source)
}
override val availableSortOrders: Set<SortOrder> = EnumSet.of(
override val sortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.UPDATED,
SortOrder.POPULARITY,
)
override val configKeyDomain: ConfigKey.Domain = ConfigKey.Domain("nicovideo.jp")
override val configKeyDomain: ConfigKey.Domain = ConfigKey.Domain("nicovideo.jp", null)
@InternalParsersApi
override suspend fun getList(
@@ -54,12 +48,12 @@ class NicovideoSeigaParser(context: MangaLoaderContext) :
!query.isNullOrEmpty() -> return if (offset == 0) getSearchList(query, page) else emptyList()
tags.isNullOrEmpty() -> "https://$domain/manga/list?page=$page&sort=${getSortKey(sortOrder)}"
tags.size == 1 -> "https://$domain/manga/list?category=${tags.first().key}&page=$page" +
"&sort=${getSortKey(sortOrder)}"
"&sort=${getSortKey(sortOrder)}"
tags.size > 1 -> throw IllegalArgumentException("This source supports only 1 category")
else -> "https://$domain/manga/list?page=$page&sort=${getSortKey(sortOrder)}"
}
val doc = webClient.httpGet(url).parseHtml()
val doc = context.httpGet(url).parseHtml()
val comicList = doc.body().select("#comic_list > ul > li") ?: doc.parseFailed("Container not found")
val items = comicList.select("div > .description > div > div")
return items.mapNotNull { item ->
@@ -94,8 +88,8 @@ class NicovideoSeigaParser(context: MangaLoaderContext) :
}
override suspend fun getDetails(manga: Manga): Manga {
val doc = webClient.httpGet(manga.url.toAbsoluteUrl(getDomain("seiga"))).parseHtml()
val contents = doc.body().requireElementById("contents")
val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain("seiga"))).parseHtml()
val contents = doc.body().selectFirstOrThrow("#contents")
val statusText = contents
.select("div.mg_work_detail > div > div:nth-child(2) > div.tip.content_status.status_series > span")
.text()
@@ -128,7 +122,7 @@ class NicovideoSeigaParser(context: MangaLoaderContext) :
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(getDomain("seiga"))
val doc = webClient.httpGet(fullUrl).parseHtml()
val doc = context.httpGet(fullUrl).parseHtml()
if (!doc.select("#login_manga").isEmpty())
throw AuthRequiredException(source)
val root = doc.body().select("#page_contents > li")
@@ -138,13 +132,14 @@ class NicovideoSeigaParser(context: MangaLoaderContext) :
id = generateUid(url),
url = url,
preview = null,
referer = fullUrl,
source = source,
)
}
}
override suspend fun getAvailableTags(): Set<MangaTag> {
val doc = webClient.httpGet("https://${getDomain("seiga")}/manga/list").parseHtml()
override suspend fun getTags(): Set<MangaTag> {
val doc = context.httpGet("https://${getDomain("seiga")}/manga/list").parseHtml()
val root = doc.body().selectOrThrow("#mg_category_list > ul > li")
return root.mapToSet { li ->
val a = li.selectFirstOrThrow("a")
@@ -158,7 +153,7 @@ class NicovideoSeigaParser(context: MangaLoaderContext) :
private suspend fun getSearchList(query: String, page: Int): List<Manga> {
val domain = getDomain("seiga")
val doc = webClient.httpGet("https://$domain/manga/search/?q=$query&page=$page&sort=score").parseHtml()
val doc = context.httpGet("https://$domain/manga/search/?q=$query&page=$page&sort=score").parseHtml()
val root = doc.body().select(".search_result__item")
return root.mapNotNull { item ->
val href = item.selectFirst(".search_result__item__thumbnail > a")
@@ -187,4 +182,4 @@ class NicovideoSeigaParser(context: MangaLoaderContext) :
SortOrder.UPDATED -> "manga_updated"
else -> "manga_view"
}
}
}

View File

@@ -1,11 +1,7 @@
package org.koitharu.kotatsu.parsers.site.all
package org.koitharu.kotatsu.parsers.site
import androidx.collection.ArrayMap
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
import okhttp3.Headers
import okhttp3.HttpUrl.Companion.toHttpUrl
import okhttp3.Interceptor
import okhttp3.Response
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
@@ -16,35 +12,25 @@ import java.text.SimpleDateFormat
import java.util.*
internal abstract class NineMangaParser(
context: MangaLoaderContext,
final override val context: MangaLoaderContext,
source: MangaSource,
defaultDomain: String,
) : PagedMangaParser(context, source, pageSize = 26), Interceptor {
) : PagedMangaParser(source, pageSize = 26) {
override val configKeyDomain = ConfigKey.Domain(defaultDomain)
override val configKeyDomain = ConfigKey.Domain(defaultDomain, null)
init {
context.cookieJar.insertCookies(domain, "ninemanga_template_desk=yes")
context.cookieJar.insertCookies(getDomain(), "ninemanga_template_desk=yes")
}
override val headers = super.headers.newBuilder()
override val headers = Headers.Builder()
.add("Accept-Language", "en-US;q=0.7,en;q=0.3")
.build()
override val availableSortOrders: Set<SortOrder> = Collections.singleton(
override val sortOrders: Set<SortOrder> = Collections.singleton(
SortOrder.POPULARITY,
)
override fun intercept(chain: Interceptor.Chain): Response {
val request = chain.request()
val newRequest = if (request.url.host == domain) {
request.newBuilder().removeHeader("Referer").build()
} else {
request
}
return chain.proceed(newRequest)
}
override suspend fun getListPage(
page: Int,
query: String?,
@@ -53,7 +39,7 @@ internal abstract class NineMangaParser(
): List<Manga> {
val url = buildString {
append("https://")
append(domain)
append(getDomain())
when {
!query.isNullOrEmpty() -> {
append("/search/?name_sel=&wd=")
@@ -77,11 +63,13 @@ internal abstract class NineMangaParser(
append(page)
append(".html")
}
val doc = webClient.httpGet(url).parseHtml()
val root = doc.body().selectFirst("ul.direlist") ?: doc.parseFailed("Cannot find root")
val doc = context.httpGet(url, headers).parseHtml()
val root = doc.body().selectFirst("ul.direlist")
?: doc.parseFailed("Cannot find root")
val baseHost = root.baseUri().toHttpUrl().host
return root.select("li").map { node ->
val href = node.selectFirst("a")?.absUrl("href") ?: node.parseFailed("Link not found")
val href = node.selectFirst("a")?.absUrl("href")
?: node.parseFailed("Link not found")
val relUrl = href.toRelativeUrl(baseHost)
val dd = node.selectFirst("dd")
Manga(
@@ -103,25 +91,30 @@ internal abstract class NineMangaParser(
}
override suspend fun getDetails(manga: Manga): Manga {
val doc = webClient.httpGet(
manga.url.toAbsoluteUrl(domain) + "?waring=1",
val doc = context.httpGet(
manga.url.toAbsoluteUrl(getDomain()) + "?waring=1",
headers,
).parseHtml()
val root = doc.body().selectFirstOrThrow("div.manga")
val infoRoot = root.selectFirstOrThrow("div.bookintro")
val tagMap = getOrCreateTagMap()
val selectTag = infoRoot.getElementsByAttributeValue("itemprop", "genre").first()?.select("a")
val tags = selectTag?.mapNotNullToSet { tagMap[it.text()] }
return manga.copy(
tags = tags.orEmpty(),
tags = infoRoot.getElementsByAttributeValue("itemprop", "genre").first()
?.select("a")?.mapToSet { a ->
MangaTag(
title = a.text().toTitleCase(),
key = a.attr("href").substringBetween("/", "."),
source = source,
)
}.orEmpty(),
author = infoRoot.getElementsByAttributeValue("itemprop", "author").first()?.text(),
state = parseStatus(infoRoot.select("li a.red").text()),
description = infoRoot.getElementsByAttributeValue("itemprop", "description").first()?.html()
?.substringAfter("</b>"),
description = infoRoot.getElementsByAttributeValue("itemprop", "description").first()
?.html()?.substringAfter("</b>"),
chapters = root.selectFirst("div.chapterbox")?.select("ul.sub_vol_ul > li")
?.mapChapters(reversed = true) { i, li ->
?.asReversed()?.mapChapters { i, li ->
val a = li.selectFirst("a.chapter_list_a")
val href =
a?.attrAsRelativeUrlOrNull("href")?.replace("%20", " ") ?: li.parseFailed("Link not found")
val href = a?.attrAsRelativeUrlOrNull("href")
?.replace("%20", " ") ?: li.parseFailed("Link not found")
MangaChapter(
id = generateUid(href),
name = a.text(),
@@ -137,12 +130,13 @@ internal abstract class NineMangaParser(
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml()
val doc = context.httpGet(chapter.url.toAbsoluteUrl(getDomain()), headers).parseHtml()
return doc.body().getElementById("page")?.select("option")?.map { option ->
val url = option.attr("value")
MangaPage(
id = generateUid(url),
url = url,
referer = chapter.url.toAbsoluteUrl(getDomain()),
preview = null,
source = source,
)
@@ -150,58 +144,30 @@ internal abstract class NineMangaParser(
}
override suspend fun getPageUrl(page: MangaPage): String {
val doc = webClient.httpGet(page.url.toAbsoluteUrl(domain)).parseHtml()
val doc = context.httpGet(page.url.toAbsoluteUrl(getDomain()), headers).parseHtml()
val root = doc.body()
return root.selectFirst("a.pic_download")?.absUrl("href") ?: doc.parseFailed("Page image not found")
return root.selectFirst("a.pic_download")?.absUrl("href")
?: doc.parseFailed("Page image not found")
}
private var tagCache: ArrayMap<String, MangaTag>? = null
private val mutex = Mutex()
override suspend fun getAvailableTags(): Set<MangaTag> {
return getOrCreateTagMap().values.toSet()
}
private suspend fun getOrCreateTagMap(): Map<String, MangaTag> = mutex.withLock {
tagCache?.let { return@withLock it }
val tagMap = ArrayMap<String, MangaTag>()
val tagElements = webClient.httpGet("https://${domain}/search/?type=high").parseHtml().select("li.cate_list")
for (el in tagElements) {
if (el.text().isEmpty()) continue
val cateId = el.attr("cate_id")
val a = el.selectFirstOrThrow("a")
tagMap[el.text()] = MangaTag(
override suspend fun getTags(): Set<MangaTag> {
val doc = context.httpGet("https://${getDomain()}/search/?type=high", headers)
.parseHtml()
val root = doc.body().getElementById("search_form")
return root?.select("li.cate_list")?.mapNotNullToSet { li ->
val cateId = li.attr("cate_id") ?: return@mapNotNullToSet null
val a = li.selectFirst("a") ?: return@mapNotNullToSet null
MangaTag(
title = a.text().toTitleCase(),
key = cateId,
source = source,
)
}
tagCache = tagMap
return@withLock tagMap
} ?: doc.parseFailed("Root not found")
}
private fun parseStatus(status: String) = when {
// en
status.contains("Ongoing") -> MangaState.ONGOING
status.contains("Completed") -> MangaState.FINISHED
//es
status.contains("En curso") -> MangaState.ONGOING
status.contains("Completado") -> MangaState.FINISHED
//ru
status.contains("постоянный") -> MangaState.ONGOING
status.contains("завершенный") -> MangaState.FINISHED
//de
status.contains("Laufende") -> MangaState.ONGOING
status.contains("Abgeschlossen") -> MangaState.FINISHED
//pt
status.contains("Completo") -> MangaState.ONGOING
status.contains("Em tradução") -> MangaState.FINISHED
//it
status.contains("In corso") -> MangaState.ONGOING
status.contains("Completato") -> MangaState.FINISHED
//fr
status.contains("En cours") -> MangaState.ONGOING
status.contains("Complété") -> MangaState.FINISHED
else -> null
}
@@ -291,4 +257,4 @@ internal abstract class NineMangaParser(
MangaSource.NINEMANGA_FR,
"fr.ninemanga.com",
)
}
}

View File

@@ -0,0 +1,228 @@
package org.koitharu.kotatsu.parsers.site
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.MangaParserAuthProvider
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.exception.AuthRequiredException
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import java.text.SimpleDateFormat
import java.util.*
private const val MAX_THUMB_INDEX = 19
@MangaSourceParser("NUDEMOON", "Nude-Moon", "ru")
internal class NudeMoonParser(
override val context: MangaLoaderContext,
) : MangaParser(MangaSource.NUDEMOON), MangaParserAuthProvider {
override val configKeyDomain = ConfigKey.Domain("nude-moon.net", null)
override val authUrl: String
get() = "https://${getDomain()}/index.php"
override val isAuthorized: Boolean
get() {
return context.cookieJar.getCookies(getDomain()).any {
it.name == "fusion_user"
}
}
override val sortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.NEWEST,
SortOrder.POPULARITY,
SortOrder.RATING,
)
init {
context.cookieJar.insertCookies(
getDomain(),
"NMfYa=1;",
"nm_mobile=0;",
)
}
override suspend fun getList(
offset: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val domain = getDomain()
val url = when {
!query.isNullOrEmpty() -> "https://$domain/search?stext=${query.urlEncoded()}&rowstart=$offset"
!tags.isNullOrEmpty() -> tags.joinToString(
separator = "_",
prefix = "https://$domain/tags/",
postfix = "&rowstart=$offset",
transform = { it.key.urlEncoded() },
)
else -> "https://$domain/all_manga?${getSortKey(sortOrder)}&rowstart=$offset"
}
val doc = context.httpGet(url).parseHtml()
val root = doc.body().run {
selectFirst("td.main-bg") ?: selectFirst("td.main-body")
} ?: doc.parseFailed("Cannot find root")
return root.select("table.news_pic2").mapNotNull { row ->
val a = row.selectFirst("td.bg_style1")?.selectFirst("a")
?: return@mapNotNull null
val href = a.attrAsRelativeUrl("href")
val title = a.selectFirst("h2")?.text().orEmpty()
val info = row.selectFirst("td[width=100%]") ?: return@mapNotNull null
Manga(
id = generateUid(href),
url = href,
title = title.substringAfter(" / "),
altTitle = title.substringBefore(" / ", "")
.takeUnless { it.isBlank() },
author = info.getElementsContainingOwnText("Автор:").firstOrNull()
?.nextElementSibling()?.ownText(),
coverUrl = row.selectFirst("img.news_pic2")?.absUrl("data-src")
.orEmpty(),
tags = row.selectFirst("span.tag-links")?.select("a")
?.mapToSet {
MangaTag(
title = it.text().toTitleCase(),
key = it.attr("href").substringAfterLast('/'),
source = source,
)
}.orEmpty(),
source = source,
publicUrl = a.absUrl("href"),
rating = RATING_UNKNOWN,
isNsfw = true,
description = row.selectFirst("div.description")?.html(),
state = null,
)
}
}
override suspend fun getDetails(manga: Manga): Manga {
val body = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml().body()
val root = body.selectFirst("table.shoutbox")
?: body.parseFailed("Cannot find root")
val info = root.select("div.tbl2")
val lastInfo = info.last()
return manga.copy(
largeCoverUrl = body.selectFirst("img.news_pic2")?.absUrl("src"),
description = info.select("div.blockquote").lastOrNull()?.html() ?: manga.description,
tags = info.select("span.tag-links").firstOrNull()?.select("a")?.mapToSet {
MangaTag(
title = it.text().toTitleCase(),
key = it.attr("href").substringAfterLast('/'),
source = source,
)
}?.plus(manga.tags) ?: manga.tags,
author = lastInfo?.getElementsByAttributeValueContaining("href", "mangaka/")?.text()
?: manga.author,
chapters = listOf(
MangaChapter(
id = manga.id,
url = getReadLink(manga.url),
source = source,
number = 1,
name = manga.title,
scanlator = lastInfo?.getElementsByAttributeValueContaining("href", "perevod/")?.text(),
uploadDate = lastInfo?.getElementsContainingOwnText("Дата:")
?.firstOrNull()
?.html()
?.parseDate() ?: 0L,
branch = null,
),
),
)
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(getDomain())
val doc = context.httpGet(fullUrl).parseHtml()
val mangaId = chapter.url.substringAfterLast('/').substringBefore('-').toIntOrNull()
val script = doc.select("script").firstNotNullOfOrNull {
it.html().takeIf { x -> x.contains(" images = new ") }
} ?: if (isAuthorized) {
doc.parseFailed("Cannot find pages list")
} else {
throw AuthRequiredException(source)
}
val pagesRegex = Regex("images\\[(\\d+)].src\\s*=\\s*'([^']+)'", RegexOption.MULTILINE)
return pagesRegex.findAll(script).map { match ->
val i = match.groupValues[1].toInt()
val url = match.groupValues[2]
MangaPage(
id = generateUid(url),
url = url,
referer = fullUrl,
preview = if (i <= MAX_THUMB_INDEX && mangaId != null) {
val part2 = url.substringBeforeLast('/')
val part3 = url.substringAfterLast('/')
val part1 = part2.substringBeforeLast('/')
"$part1/thumb/$mangaId/thumb_$part3"
} else {
null
},
source = source,
)
}.toList()
}
override suspend fun getTags(): Set<MangaTag> {
val domain = getDomain()
val doc = context.httpGet("https://$domain/all_manga").parseHtml()
val root = doc.body().getElementsContainingOwnText("Поиск манги по тегам")
.firstOrNull()?.parents()?.find { it.tag().normalName() == "tbody" }
?.selectFirst("td.textbox")?.selectFirst("td.small")
?: doc.parseFailed("Tags root not found")
return root.select("a").mapToSet {
MangaTag(
title = it.text().toTitleCase(),
key = it.attr("href").substringAfterLast('/')
.removeSuffix("+"),
source = source,
)
}
}
override suspend fun getUsername(): String {
val body = context.httpGet("https://${getDomain()}/").parseHtml()
.body()
return body
.getElementsContainingOwnText("Профиль")
.firstOrNull()
?.attr("href")
?.substringAfterLast('/')
?: run {
throw if (body.selectFirst("form[name=\"loginform\"]") != null) {
AuthRequiredException(source)
} else {
body.parseFailed("Cannot find username")
}
}
}
override fun getFaviconUrl(): String {
return "https://${getDomain()}/favicon.jpg"
}
private fun getSortKey(sortOrder: SortOrder) =
when (sortOrder) {
SortOrder.POPULARITY -> "views"
SortOrder.NEWEST -> "date"
SortOrder.RATING -> "like"
else -> "like"
}
private fun String.parseDate(): Long {
val dateString = substringBetweenFirst("Дата:", "<")?.trim() ?: return 0
val dateFormat = SimpleDateFormat("d MMMM yyyy", Locale("ru"))
return dateFormat.tryParse(dateString)
}
private fun getReadLink(url: String): String {
val prefix = url.substringBefore('-', "")
val suffix = url.substringAfter('-').trimStart('-')
return "$prefix-online-$suffix"
}
}

View File

@@ -1,9 +1,7 @@
package org.koitharu.kotatsu.parsers.site.ru
package org.koitharu.kotatsu.parsers.site
import okhttp3.Headers
import okhttp3.Interceptor
import okhttp3.Response
import okhttp3.internal.closeQuietly
import org.json.JSONArray
import org.json.JSONException
import org.json.JSONObject
@@ -12,12 +10,13 @@ import org.koitharu.kotatsu.parsers.MangaParserAuthProvider
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.exception.AuthRequiredException
import org.koitharu.kotatsu.parsers.exception.ContentUnavailableException
import org.koitharu.kotatsu.parsers.exception.ParseException
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.network.UserAgents
import org.koitharu.kotatsu.parsers.util.*
import org.koitharu.kotatsu.parsers.util.json.*
import java.net.HttpURLConnection
import java.net.URLDecoder
import java.text.DateFormat
import java.text.SimpleDateFormat
@@ -26,28 +25,17 @@ import java.util.*
private const val PAGE_SIZE = 30
private const val STATUS_ONGOING = 1
private const val STATUS_FINISHED = 0
private const val TOO_MANY_REQUESTS = 429
@MangaSourceParser("REMANGA", "ReManga", "ru")
@MangaSourceParser("REMANGA", "Remanga", "ru")
internal class RemangaParser(
context: MangaLoaderContext,
) : PagedMangaParser(context, MangaSource.REMANGA, PAGE_SIZE), MangaParserAuthProvider, Interceptor {
override val context: MangaLoaderContext,
) : PagedMangaParser(MangaSource.REMANGA, PAGE_SIZE), MangaParserAuthProvider {
private val userAgentKey = ConfigKey.UserAgent(UserAgents.CHROME_MOBILE)
private val baseHeaders: Headers
get() = Headers.Builder()
.add("User-Agent", config[userAgentKey])
.build()
override val headers
get() = getApiHeaders()
override val configKeyDomain = ConfigKey.Domain("remanga.org", "реманга.орг")
override val configKeyDomain = ConfigKey.Domain("remanga.org", null)
override val authUrl: String
get() = "https://${domain}/user/login"
get() = "https://${getDomain()}/user/login"
override val availableSortOrders: Set<SortOrder> = EnumSet.of(
override val sortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.UPDATED,
SortOrder.POPULARITY,
SortOrder.RATING,
@@ -56,23 +44,13 @@ internal class RemangaParser(
override val isAuthorized: Boolean
get() {
return context.cookieJar.getCookies(domain).any {
return context.cookieJar.getCookies(getDomain()).any {
it.name == "user"
}
}
private val regexLastUrlPath = Regex("/[^/]+/?$")
override fun intercept(chain: Interceptor.Chain): Response {
val response = chain.proceed(chain.request())
if (response.code == TOO_MANY_REQUESTS) {
response.closeQuietly()
Thread.sleep(1000)
return chain.proceed(chain.request().newBuilder().build())
}
return response
}
override suspend fun getListPage(
page: Int,
query: String?,
@@ -80,7 +58,7 @@ internal class RemangaParser(
sortOrder: SortOrder,
): List<Manga> {
copyCookies()
val domain = domain
val domain = getDomain()
val urlBuilder = StringBuilder()
.append("https://api.")
.append(domain)
@@ -100,7 +78,7 @@ internal class RemangaParser(
.append(page)
.append("&count=")
.append(PAGE_SIZE)
val content = webClient.httpGet(urlBuilder.toString()).parseJson()
val content = context.httpGet(urlBuilder.toString(), getApiHeaders()).parseJson()
.getJSONArray("content")
return content.mapJSON { jo ->
val url = "/manga/${jo.getString("dir")}"
@@ -131,12 +109,13 @@ internal class RemangaParser(
override suspend fun getDetails(manga: Manga): Manga {
copyCookies()
val domain = domain
val domain = getDomain()
val slug = manga.url.find(regexLastUrlPath)
?: throw ParseException("Cannot obtain slug from ${manga.url}", manga.publicUrl)
val data = webClient.httpGet(
val data = context.httpGet(
url = "https://api.$domain/api/titles$slug/",
).parseJson()
headers = getApiHeaders(),
).handle401().parseJson()
val content = try {
data.getJSONObject("content")
} catch (e: JSONException) {
@@ -161,10 +140,7 @@ internal class RemangaParser(
)
},
chapters = chapters.mapChapters { i, jo ->
if (
jo.getBooleanOrDefault("is_paid", false) &&
!jo.getBooleanOrDefault("is_bought", false)
) {
if (jo.getBooleanOrDefault("is_paid", false)) {
return@mapChapters null
}
val id = jo.getLong("id")
@@ -173,7 +149,7 @@ internal class RemangaParser(
MangaChapter(
id = generateUid(id),
url = "/api/titles/chapters/$id/",
number = jo.getIntOrDefault("index", chapters.size - i),
number = chapters.size - i,
name = buildString {
append("Том ")
append(jo.optString("tome", "0"))
@@ -195,7 +171,9 @@ internal class RemangaParser(
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val content = webClient.httpGet(chapter.url.toAbsoluteUrl(getDomain("api")))
val referer = "https://${getDomain()}/"
val content = context.httpGet(chapter.url.toAbsoluteUrl(getDomain("api")), getApiHeaders())
.handle401()
.parseJson()
.getJSONObject("content")
val pages = content.optJSONArray("pages")
@@ -213,17 +191,17 @@ internal class RemangaParser(
val result = ArrayList<MangaPage>(pages.length())
for (i in 0 until pages.length()) {
when (val item = pages.get(i)) {
is JSONObject -> result += parsePage(item)
is JSONArray -> item.mapJSONTo(result) { parsePage(it) }
is JSONObject -> result += parsePage(item, referer)
is JSONArray -> item.mapJSONTo(result) { parsePage(it, referer) }
else -> throw ParseException("Unknown json item $item", chapter.url)
}
}
return result
}
override suspend fun getAvailableTags(): Set<MangaTag> {
val domain = domain
val content = webClient.httpGet("https://api.$domain/api/forms/titles/?get=genres")
override suspend fun getTags(): Set<MangaTag> {
val domain = getDomain()
val content = context.httpGet("https://api.$domain/api/forms/titles/?get=genres", getApiHeaders())
.parseJson().getJSONObject("content").getJSONArray("genres")
return content.mapJSONToSet { jo ->
MangaTag(
@@ -235,28 +213,24 @@ internal class RemangaParser(
}
override suspend fun getUsername(): String {
val jo = webClient.httpGet(
url = "https://api.${domain}/api/users/current/",
).parseJson()
val jo = context.httpGet(
url = "https://api.${getDomain()}/api/users/current/",
headers = getApiHeaders(),
).handle401().parseJson()
return jo.getJSONObject("content").getString("username")
}
override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) {
super.onCreateConfig(keys)
keys.add(userAgentKey)
}
private fun getApiHeaders(): Headers {
val userCookie = context.cookieJar.getCookies(domain).find {
private fun getApiHeaders(): Headers? {
val userCookie = context.cookieJar.getCookies(getDomain()).find {
it.name == "user"
} ?: return baseHeaders
} ?: return null
val jo = JSONObject(URLDecoder.decode(userCookie.value, Charsets.UTF_8.name()))
val accessToken = jo.getStringOrNull("access_token") ?: return baseHeaders
return baseHeaders.newBuilder().add("authorization", "bearer $accessToken").build()
val accessToken = jo.getStringOrNull("access_token") ?: return null
return Headers.headersOf("authorization", "bearer $accessToken")
}
private fun copyCookies() {
val domain = domain
val domain = getDomain()
context.cookieJar.copyCookies(domain, "api.$domain")
}
@@ -268,10 +242,11 @@ internal class RemangaParser(
else -> "-chapter_date"
}
private fun parsePage(jo: JSONObject) = MangaPage(
private fun parsePage(jo: JSONObject, referer: String) = MangaPage(
id = generateUid(jo.getLong("id")),
url = jo.getString("link"),
preview = null,
referer = referer,
source = source,
)
@@ -279,9 +254,10 @@ internal class RemangaParser(
val result = ArrayList<JSONObject>(100)
var page = 1
while (true) {
val content = webClient.httpGet(
url = "https://api.$domain/api/titles/chapters/?branch_id=$branchId&page=$page&count=500",
).parseJson().getJSONArray("content")
val content = context.httpGet(
url = "https://api.$domain/api/titles/chapters/?branch_id=$branchId&page=$page&count=100",
headers = getApiHeaders(),
).handle401().parseJson().getJSONArray("content")
val len = content.length()
if (len == 0) {
break
@@ -294,4 +270,10 @@ internal class RemangaParser(
}
return result
}
}
private fun Response.handle401() = apply {
if (code == HttpURLConnection.HTTP_UNAUTHORIZED) {
throw AuthRequiredException(source)
}
}
}

View File

@@ -0,0 +1,177 @@
package org.koitharu.kotatsu.parsers.site
import org.jsoup.nodes.Element
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import org.koitharu.kotatsu.parsers.util.json.getStringOrNull
import org.koitharu.kotatsu.parsers.util.json.mapJSON
import java.text.SimpleDateFormat
import java.util.*
@MangaSourceParser("UNION_MANGAS", "Union Mangás", "pt")
class UnionMangasParser(override val context: MangaLoaderContext) : PagedMangaParser(MangaSource.UNION_MANGAS, 40) {
override val sortOrders = EnumSet.of(
SortOrder.ALPHABETICAL,
SortOrder.POPULARITY,
)
override val configKeyDomain = ConfigKey.Domain("unionleitor.top", emptyArray())
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
if (!query.isNullOrEmpty()) {
return if (page == searchPaginator.firstPage) {
search(query)
} else {
emptyList()
}
}
val tag = tags.oneOrThrowIfMany()
val url = urlBuilder()
.addPathSegment("lista-mangas")
.addPathSegment(
when {
tag != null -> tag.key
sortOrder == SortOrder.ALPHABETICAL -> "a-z"
else -> "visualizacoes"
},
).addPathSegment(page.toString())
val doc = context.httpGet(url.build()).parseHtml()
val root = doc.selectFirstOrThrow("div.tamanho-bloco-perfil")
return root.select(".lista-mangas-novos").map { div ->
val a = div.selectFirstOrThrow("a")
val img = div.selectFirstOrThrow("img")
val href = a.attrAsRelativeUrl("href")
Manga(
id = generateUid(href),
url = href,
publicUrl = a.attrAsAbsoluteUrl("href"),
title = div.selectLastOrThrow("a").text(),
coverUrl = img.attrAsAbsoluteUrl("src"),
altTitle = null,
rating = RATING_UNKNOWN,
tags = emptySet(),
description = div.selectLast("div")?.ownText(),
state = null,
author = null,
isNsfw = false,
source = source,
)
}
}
override suspend fun getDetails(manga: Manga): Manga {
val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml()
val root = doc.selectFirstOrThrow(".perfil-manga")
val dateFormat = SimpleDateFormat("dd/MM/yyyy", Locale.ROOT)
return manga.copy(
rating = root.select("h2")
.find { it.ownText().startsWith('#') }
?.ownText()?.drop(1)?.toFloatOrNull()?.div(10f) ?: manga.rating,
largeCoverUrl = root.selectFirst("img.img-thumbnail")?.attrAsAbsoluteUrlOrNull("src"),
description = root.selectFirst(".panel-default")?.selectFirst(".panel-body")?.html(),
author = root.tableValue("Autor")?.ownText(),
altTitle = root.tableValue("Título(s) Alternativo(s)")?.ownText(),
state = when (root.tableValue("Status")?.selectLast(".label")?.text()) {
"Completo" -> MangaState.FINISHED
"Ativo" -> MangaState.ONGOING
else -> null
},
tags = root.tableValue("Gênero(s)")?.select("a")?.mapToSet {
it.toMangaTag()
} ?: manga.tags,
isNsfw = root.selectFirst(".alert-danger")?.html()?.contains("18 anos") == true,
chapters = root.select("div.row.capitulos").asReversed().mapChapters { i, div ->
val a = div.selectFirstOrThrow("a")
val href = a.attrAsRelativeUrl("href")
val title = a.text()
MangaChapter(
id = generateUid(href),
name = title,
number = i + 1,
url = href,
scanlator = div.selectLast("a")?.text()?.takeUnless { it == title },
uploadDate = dateFormat.tryParse(
a.nextElementSibling()?.text()?.removeSurrounding("(", ")"),
),
branch = null,
source = source,
)
},
)
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(getDomain())
val doc = context.httpGet(fullUrl).parseHtml()
val root = doc.body().selectFirstOrThrow("article")
return root.selectOrThrow("img[pag]").mapNotNull { img ->
val href = img.attrAsRelativeUrl("src")
if (href.startsWith("/images/banner")) {
return@mapNotNull null
}
MangaPage(
id = generateUid(href),
url = href,
referer = fullUrl,
preview = null,
source = source,
)
}
}
override suspend fun getTags(): Set<MangaTag> {
val doc = context.httpGet(urlBuilder().addPathSegment("lista-mangas").build()).parseHtml()
val ul = doc.body().selectFirstOrThrow(".nav-tabs").selectFirstOrThrow("ul.dropdown-menu")
return ul.select("li").mapToSet { li ->
li.selectFirstOrThrow("a").toMangaTag()
}
}
private suspend fun search(query: String): List<Manga> {
val domain = getDomain()
val json = context.httpGet(
urlBuilder()
.addPathSegments("assets/busca.php")
.addQueryParameter("nomeManga", query)
.build(),
).parseJson()
return json.getJSONArray("items").mapJSON { jo ->
val href = "/pagina-manga/" + jo.getString("url")
Manga(
id = generateUid(href),
url = href,
publicUrl = href.toAbsoluteUrl(domain),
title = jo.getString("titulo"),
rating = RATING_UNKNOWN,
tags = emptySet(),
author = jo.getStringOrNull("autor"),
coverUrl = jo.getString("imagem"),
state = null,
isNsfw = false,
altTitle = null,
source = source,
)
}
}
private fun Element.tableValue(title: String): Element? {
return select("h4.media-heading")
.find { it.selectFirst("label.subtit-manga")?.text()?.contains(title, ignoreCase = true) == true }
}
private fun Element.toMangaTag() = MangaTag(
title = text().toTitleCase(sourceLocale ?: Locale.ROOT),
key = attr("href").removeSuffix('/').substringAfterLast('/'),
source = source,
)
}

View File

@@ -1,182 +0,0 @@
package org.koitharu.kotatsu.parsers.site.all
import androidx.collection.ArraySet
import kotlinx.coroutines.async
import kotlinx.coroutines.awaitAll
import kotlinx.coroutines.coroutineScope
import org.jsoup.nodes.Element
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import java.util.*
@MangaSourceParser("IMHENTAI", "ImHentai", type = ContentType.HENTAI)
internal class ImHentai(context: MangaLoaderContext) :
PagedMangaParser(context, MangaSource.IMHENTAI, pageSize = 20) {
override val availableSortOrders: Set<SortOrder> =
EnumSet.of(SortOrder.UPDATED, SortOrder.POPULARITY, SortOrder.RATING)
override val configKeyDomain = ConfigKey.Domain("imhentai.xxx")
override val isMultipleTagsSupported = false
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val tag = tags.oneOrThrowIfMany()
val url = buildString {
append("https://")
append(domain)
if (!query.isNullOrEmpty()) {
append("/search/?key=")
append(query.urlEncoded())
append("&page=")
append(page)
} else if (!tags.isNullOrEmpty()) {
append("/tag/")
append(tag?.key.orEmpty())
append("/")
when (sortOrder) {
SortOrder.UPDATED -> append("")
SortOrder.POPULARITY -> append("popular/")
else -> append("")
}
append("?page=")
append(page)
} else {
append("/search/?page=")
append(page)
when (sortOrder) {
SortOrder.UPDATED -> append("&lt=1&pp=0")
SortOrder.POPULARITY -> append("&lt=0&pp=1")
SortOrder.RATING -> append("&lt=0&pp=0")
else -> append("&lt=1&pp=0")
}
}
}
val doc = webClient.httpGet(url).parseHtml()
return doc.select("div.galleries div.thumb").map { div ->
val a = div.selectFirstOrThrow(".inner_thumb a")
val href = a.attrAsRelativeUrl("href")
Manga(
id = generateUid(href),
url = href,
publicUrl = href.toAbsoluteUrl(domain),
coverUrl = a.selectFirst("img")?.src().orEmpty(),
title = div.selectFirst(".caption")?.text().orEmpty(),
altTitle = null,
rating = RATING_UNKNOWN,
tags = emptySet(),
author = null,
state = null,
source = source,
isNsfw = isNsfwSource,
)
}
}
//Tags are deliberately reduced because there are too many and this slows down the application.
//only the most popular ones are taken.
override suspend fun getAvailableTags(): Set<MangaTag> {
return coroutineScope {
(1..3).map { page ->
async { getTags(page) }
}
}.awaitAll().flattenTo(ArraySet(360))
}
private suspend fun getTags(page: Int): Set<MangaTag> {
val url = "https://$domain/tags/popular/?page=$page"
val root = webClient.httpGet(url).parseHtml()
return root.parseTags()
}
private fun Element.parseTags() = select("div.stags a.tag_btn").mapToSet {
val href = it.attr("href").substringAfterLast("tag/").substringBeforeLast('/')
MangaTag(
key = href,
title = it.selectFirstOrThrow("h3.list_tag").text(),
source = source,
)
}
override suspend fun getDetails(manga: Manga): Manga = coroutineScope {
val fullUrl = manga.url.toAbsoluteUrl(domain)
val doc = webClient.httpGet(fullUrl).parseHtml()
manga.copy(
tags = doc.body().select("li:contains(Tags) a.tag").mapNotNullToSet {
val href = it.attr("href").substringAfterLast("tag/").substringBeforeLast('/')
val name = it.html().substringBeforeLast("<span")
MangaTag(
key = href,
title = name,
source = source,
)
},
author = doc.selectFirst("li:contains(Artists) a.tag")?.html()?.substringBefore("<span"),
chapters = listOf(
MangaChapter(
id = manga.id,
name = manga.title,
number = 1,
url = manga.url,
scanlator = null,
uploadDate = 0,
branch = doc.selectFirst("li:contains(Language) a.tag")?.html()?.substringBeforeLast("<span"),
source = source,
),
),
)
}
override suspend fun getRelatedManga(seed: Manga): List<Manga> {
val doc = webClient.httpGet(seed.url.toAbsoluteUrl(domain)).parseHtml()
val root = doc.body().selectFirstOrThrow("div.related")
return root.select("div.thumb").mapNotNull { div ->
val a = div.selectFirstOrThrow(".inner_thumb a")
val href = a.attrAsRelativeUrl("href")
Manga(
id = generateUid(href),
url = href,
publicUrl = href.toAbsoluteUrl(domain),
coverUrl = a.selectFirst("img")?.src().orEmpty(),
title = div.selectFirst(".caption")?.text().orEmpty(),
altTitle = null,
rating = RATING_UNKNOWN,
tags = emptySet(),
author = null,
state = null,
source = source,
isNsfw = false,
)
}
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(domain)
val doc = webClient.httpGet(fullUrl).parseHtml()
val totalPages = doc.selectFirstOrThrow(".pages").text().replace("Pages: ", "").toInt() + 1
val domainImg = doc.requireElementById("append_thumbs").selectFirstOrThrow("img").src()?.replace("1t.jpg", "")
val pages = ArrayList<MangaPage>(totalPages)
for (i in 1 until totalPages) {
val url = "$domainImg$i.jpg"
pages.add(
MangaPage(
id = generateUid(url),
url = url,
preview = null,
source = source,
),
)
}
return pages
}
}

View File

@@ -1,320 +0,0 @@
package org.koitharu.kotatsu.parsers.site.all
import kotlinx.coroutines.async
import kotlinx.coroutines.coroutineScope
import okhttp3.Headers
import okhttp3.HttpUrl
import okhttp3.HttpUrl.Companion.toHttpUrl
import org.json.JSONObject
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.exception.NotFoundException
import org.koitharu.kotatsu.parsers.exception.ParseException
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import org.koitharu.kotatsu.parsers.util.json.*
import java.util.*
import javax.crypto.Mac
import javax.crypto.spec.SecretKeySpec
internal abstract class LineWebtoonsParser(
context: MangaLoaderContext,
source: MangaSource,
) : MangaParser(context, source) {
override val isMultipleTagsSupported = false
private val signer by lazy {
WebtoonsUrlSigner("gUtPzJFZch4ZyAGviiyH94P99lQ3pFdRTwpJWDlSGFfwgpr6ses5ALOxWHOIT7R1")
}
// we don't __really__ support changing this domain because:
// 1. I don't think other websites have this exact API
// 2. most communication is done with other domains (hosting API and static content), which are not configurable
// 3. we rely on the HTTP client setting the referer header to webtoons.com
//
// This effectively means that changing the domain will break the source. Yikes
override val configKeyDomain = ConfigKey.Domain("webtoons.com")
private val apiDomain = "global.apis.naver.com"
private val staticDomain = "webtoon-phinf.pstatic.net"
override val availableSortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.POPULARITY,
// doesn't actually sort by rating, but by likes
// this should be fine though
SortOrder.RATING,
SortOrder.UPDATED,
)
override val headers: Headers
get() = Headers.Builder()
.add("User-Agent", "nApps (Android 12;; linewebtoon; 3.1.0)")
.build()
override suspend fun getPageUrl(page: MangaPage): String {
return page.url.toAbsoluteUrl(staticDomain)
}
// some language tags do not map perfectly to the ones used by the API
private val languageCode: String
get() = when (val tag = sourceLocale.toLanguageTag()) {
"in" -> "id"
"zh" -> "zh-hant"
else -> tag
}
private suspend fun getChapters(titleNo: Long): List<MangaChapter> {
val firstResult = makeRequest(
url = "/lineWebtoon/webtoon/challengeEpisodeList.json?v=2&titleNo=$titleNo&startIndex=0&pageSize=30",
)
val totalEpisodeCount = firstResult
.getJSONObject("episodeList")
.getInt("totalServiceEpisodeCount")
val episodes = firstResult
.getJSONObject("episodeList")
.getJSONArray("episode")
.toJSONList()
.toMutableList()
while (episodes.count() < totalEpisodeCount) {
val page = makeRequest(
url = "/lineWebtoon/webtoon/challengeEpisodeList.json?v=2&titleNo=$titleNo&startIndex=${episodes.count()}&pageSize=30",
).getJSONObject("episodeList")
.getJSONArray("episode")
.toJSONList()
episodes.addAll(page)
}
return episodes.mapChapters { i, jo ->
MangaChapter(
id = generateUid("$titleNo-$i"),
name = jo.getString("episodeTitle"),
number = jo.getInt("episodeSeq"),
url = "$titleNo-${jo.get("episodeNo")}",
uploadDate = jo.getLong("modifyYmdt"),
branch = null,
scanlator = null,
source = source,
)
}.sortedBy { it.number }
}
override suspend fun getDetails(manga: Manga): Manga = coroutineScope {
val titleNo = manga.url.toLong()
val chaptersDeferred = async { getChapters(titleNo) }
makeRequest("/lineWebtoon/webtoon/challengeTitleInfo.json?v=2&titleNo=${titleNo}")
.getJSONObject("titleInfo")
.let { jo ->
Manga(
id = generateUid(titleNo),
title = jo.getString("title"),
altTitle = null,
url = "$titleNo",
publicUrl = "https://$domain/$languageCode/canvas/a/list?title_no=${titleNo}",
rating = jo.getFloatOrDefault("starScoreAverage", -10f) / 10f,
isNsfw = jo.getBooleanOrDefault("ageGradeNotice", isNsfwSource),
coverUrl = jo.getString("thumbnail").toAbsoluteUrl(staticDomain),
largeCoverUrl = jo.getStringOrNull("thumbnailVertical")?.toAbsoluteUrl(staticDomain),
tags = setOf(parseTag(jo.getJSONObject("genreInfo"))),
author = jo.getStringOrNull("writingAuthorName"),
description = jo.getString("synopsis"),
// I don't think the API provides this info
state = null,
chapters = chaptersDeferred.await(),
source = source,
)
}
}
override suspend fun getList(
offset: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val genre = tags.oneOrThrowIfMany()?.key ?: "ALL"
val sortOrderStr = when (sortOrder) {
SortOrder.UPDATED -> "UPDATE"
SortOrder.POPULARITY -> "READ_COUNT"
SortOrder.RATING -> "LIKEIT"
else -> throw IllegalArgumentException("Unsupported sort order: $sortOrder")
}
val manga = if (query != null) {
if (!tags.isNullOrEmpty()) {
throw IllegalArgumentException("This source does not support search with tags")
}
makeRequest("/lineWebtoon/webtoon/searchChallenge?query=${query.urlEncoded()}&startIndex=${offset + 1}&pageSize=20")
.getJSONObject("challengeSearch")
.getJSONArray("titleList")
.mapJSON { jo ->
val titleNo = jo.getLong("titleNo")
Manga(
id = generateUid(titleNo),
title = jo.getString("title"),
altTitle = null,
url = titleNo.toString(),
publicUrl = "https://$domain/$languageCode/canvas/a/list?title_no=$titleNo",
rating = RATING_UNKNOWN,
isNsfw = isNsfwSource,
coverUrl = jo.getString("thumbnail").toAbsoluteUrl(staticDomain),
largeCoverUrl = null,
tags = emptySet(),
author = jo.getStringOrNull("writingAuthorName"),
description = null,
state = null,
source = source,
)
}
} else {
val result =
makeRequest("/lineWebtoon/webtoon/challengeGenreTitleList.json?genre=$genre&sortOrder=$sortOrderStr&startIndex=${offset + 1}&pageSize=20")
val genres = result.getJSONObject("genreList")
.getJSONArray("challengeGenres")
.mapJSON { jo -> parseTag(jo) }
.associateBy { tag -> tag.key }
result
.getJSONObject("titleList")
.getJSONArray("titles")
.mapJSON { jo ->
val titleNo = jo.getLong("titleNo")
Manga(
id = generateUid(titleNo),
title = jo.getString("title"),
altTitle = null,
url = titleNo.toString(),
publicUrl = "https://$domain/$languageCode/canvas/a/list?title_no=$titleNo",
rating = jo.getFloatOrDefault("starScoreAverage", -10f) / 10f,
isNsfw = jo.getBooleanOrDefault("ageGradeNotice", isNsfwSource),
coverUrl = jo.getString("thumbnail").toAbsoluteUrl(staticDomain),
largeCoverUrl = jo.getStringOrNull("thumbnailVertical")?.toAbsoluteUrl(staticDomain),
tags = setOfNotNull(genres[jo.getString("representGenre")]),
author = jo.getStringOrNull("writingAuthorName"),
description = jo.getString("synopsis"),
// I don't think the API provides this info
state = null,
source = source,
)
}
}
return manga
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val (titleNo, episodeNo) = requireNotNull(chapter.url.splitTwoParts('-'))
return makeRequest("/lineWebtoon/webtoon/challengeEpisodeInfo.json?v=2&titleNo=$titleNo&episodeNo=$episodeNo")
.getJSONObject("episodeInfo")
.getJSONArray("imageInfo")
.mapJSONIndexed { i, jo ->
MangaPage(
id = generateUid("$titleNo-$episodeNo-$i"),
url = jo.getString("url"),
preview = null,
source = source,
)
}
}
private fun parseTag(jo: JSONObject): MangaTag {
return MangaTag(
title = jo.getString("name"),
key = jo.getString("code"),
source = source,
)
}
override suspend fun getAvailableTags(): Set<MangaTag> {
return makeRequest("/lineWebtoon/webtoon/challengeGenreList.json")
.getJSONObject("genreList")
.getJSONArray("challengeGenres")
.mapJSONToSet { jo -> parseTag(jo) }
}
private suspend fun makeRequest(url: String): JSONObject {
val resp = webClient.httpGet(finalizeUrl(url))
val message: JSONObject? = resp.parseJson().optJSONObject("message")
return when (resp.code) {
in 200..299 -> checkNotNull(message).getJSONObject("result")
404 -> throw NotFoundException(message?.getStringOrNull("message").orEmpty(), url)
else -> {
val code = message?.getIntOrDefault("code", 0)
val errorMessage = message?.getStringOrNull("message")
throw ParseException("Api error (code=$code): $errorMessage", url)
}
}
}
private fun finalizeUrl(url: String): HttpUrl {
val httpUrl = url.toAbsoluteUrl(apiDomain).toHttpUrl()
val builder = httpUrl.newBuilder()
.addQueryParameter("serviceZone", "GLOBAL")
if (httpUrl.queryParameter("v") == null) {
builder.addQueryParameter("v", "1")
}
builder.addQueryParameter("language", languageCode)
.addQueryParameter("locale", "languageCode")
.addQueryParameter("platform", "APP_ANDROID")
signer.makeEncryptUrl(builder)
return builder.build()
}
@MangaSourceParser("LINEWEBTOONS_EN", "LineWebtoons English", "en", type = ContentType.MANGA)
class English(context: MangaLoaderContext) : LineWebtoonsParser(context, MangaSource.LINEWEBTOONS_EN)
@MangaSourceParser("LINEWEBTOONS_ZH", "LineWebtoons Chinese", "zh", type = ContentType.MANGA)
class Chinese(context: MangaLoaderContext) : LineWebtoonsParser(context, MangaSource.LINEWEBTOONS_ZH)
@MangaSourceParser("LINEWEBTOONS_TH", "LineWebtoons Thai", "th", type = ContentType.MANGA)
class Thai(context: MangaLoaderContext) : LineWebtoonsParser(context, MangaSource.LINEWEBTOONS_TH)
@MangaSourceParser("LINEWEBTOONS_ID", "LineWebtoons Indonesian", "id", type = ContentType.MANGA)
class Indonesian(context: MangaLoaderContext) : LineWebtoonsParser(context, MangaSource.LINEWEBTOONS_ID)
@MangaSourceParser("LINEWEBTOONS_ES", "LineWebtoons Spanish", "es", type = ContentType.MANGA)
class Spanish(context: MangaLoaderContext) : LineWebtoonsParser(context, MangaSource.LINEWEBTOONS_ES)
@MangaSourceParser("LINEWEBTOONS_FR", "LineWebtoons French", "fr", type = ContentType.MANGA)
class French(context: MangaLoaderContext) : LineWebtoonsParser(context, MangaSource.LINEWEBTOONS_FR)
@MangaSourceParser("LINEWEBTOONS_DE", "LineWebtoons German", "de", type = ContentType.MANGA)
class German(context: MangaLoaderContext) : LineWebtoonsParser(context, MangaSource.LINEWEBTOONS_DE)
private inner class WebtoonsUrlSigner(private val secret: String) {
private val mac = Mac.getInstance("HmacSHA1").apply {
this.init(SecretKeySpec(secret.encodeToByteArray(), "HmacSHA1"))
}
private fun getMessage(url: String, msgpad: String): String {
return url.substring(0, 0xFF.coerceAtMost(url.length)) + msgpad
}
private fun getMessageDigest(s: String): String {
val signedMessage = synchronized(mac) { mac.doFinal(s.toByteArray()) }
return context.encodeBase64(signedMessage)
}
fun makeEncryptUrl(urlBuilder: HttpUrl.Builder) {
val msgpad = Calendar.getInstance().timeInMillis.toString()
val digest = getMessageDigest(getMessage(urlBuilder.build().toString(), msgpad))
urlBuilder
.addQueryParameter("msgpad", msgpad)
.addQueryParameter("md", digest)
// .addEncodedQueryParameter("md", digest.urlEncoded())
}
}
}

View File

@@ -1,201 +0,0 @@
package org.koitharu.kotatsu.parsers.site.animebootstrap
import kotlinx.coroutines.async
import kotlinx.coroutines.coroutineScope
import org.json.JSONArray
import org.jsoup.nodes.Document
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import java.util.*
// see https://themewagon.com/themes/free-bootstrap-4-html5-gaming-anime-website-template-anime/
internal abstract class AnimeBootstrapParser(
context: MangaLoaderContext,
source: MangaSource,
domain: String,
pageSize: Int = 24,
) : PagedMangaParser(context, source, pageSize) {
override val configKeyDomain = ConfigKey.Domain(domain)
override val isMultipleTagsSupported = false
override val availableSortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.UPDATED,
SortOrder.POPULARITY,
SortOrder.ALPHABETICAL,
SortOrder.NEWEST,
)
protected open val listUrl = "/manga"
protected open val datePattern = "dd MMM. yyyy"
init {
paginator.firstPage = 1
searchPaginator.firstPage = 1
}
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val tag = tags.oneOrThrowIfMany()
val url = buildString {
append("https://")
append(domain)
append(listUrl)
append("?page=")
append(page.toString())
append("&type=all")
if (!query.isNullOrEmpty()) {
append("&search=")
append(query.urlEncoded())
}
if (!tags.isNullOrEmpty()) {
append("&categorie=")
append(tag?.key.orEmpty())
}
append("&sort=")
when (sortOrder) {
SortOrder.POPULARITY -> append("view")
SortOrder.UPDATED -> append("updated")
SortOrder.ALPHABETICAL -> append("default")
SortOrder.NEWEST -> append("published")
else -> append("updated")
}
}
val doc = webClient.httpGet(url).parseHtml()
return doc.select("div.col-6 div.product__item").map { div ->
val href = div.selectFirstOrThrow("a").attrAsRelativeUrl("href")
Manga(
id = generateUid(href),
url = href,
publicUrl = href.toAbsoluteUrl(div.host ?: domain),
coverUrl = div.selectFirstOrThrow("div.product__item__pic").attr("data-setbg").orEmpty(),
title = div.selectFirstOrThrow("div.product__item__text").text().orEmpty(),
altTitle = null,
rating = RATING_UNKNOWN,
tags = emptySet(),
author = null,
state = null,
source = source,
isNsfw = isNsfwSource,
)
}
}
override suspend fun getAvailableTags(): Set<MangaTag> {
val doc = webClient.httpGet("https://$domain$listUrl").parseHtml()
return doc.select("div.product__page__filter div:contains(Genre:) option ").mapNotNullToSet { option ->
val key = option.attr("value") ?: return@mapNotNullToSet null
val name = option.text()
MangaTag(
key = key,
title = name,
source = source,
)
}
}
protected open val selectDesc = "div.anime__details__text p"
protected open val selectState = "div.anime__details__widget li:contains(Ongoing)"
protected open val selectTag = "div.anime__details__widget li:contains(Categorie) a"
override suspend fun getDetails(manga: Manga): Manga = coroutineScope {
val fullUrl = manga.url.toAbsoluteUrl(domain)
val doc = webClient.httpGet(fullUrl).parseHtml()
val chaptersDeferred = async { getChapters(doc) }
val desc = doc.selectFirstOrThrow(selectDesc).html()
val state = if (doc.select(selectState).isNullOrEmpty()) {
MangaState.FINISHED
} else {
MangaState.ONGOING
}
manga.copy(
tags = doc.body().select(selectTag).mapNotNullToSet { a ->
MangaTag(
key = a.attr("href").substringAfterLast('='),
title = a.text().toTitleCase().replace(",", ""),
source = source,
)
},
description = desc,
state = state,
chapters = chaptersDeferred.await(),
)
}
protected open val selectChapter = "div.anime__details__episodes a"
protected open suspend fun getChapters(doc: Document): List<MangaChapter> {
return doc.body().select(selectChapter).mapChapters(reversed = true) { i, a ->
val href = a.attr("href")
MangaChapter(
id = generateUid(href),
name = a.text(),
number = i + 1,
url = href,
uploadDate = 0,
source = source,
scanlator = null,
branch = null,
)
}
}
protected open val selectPage = "div.read-img img"
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(domain)
val doc = webClient.httpGet(fullUrl).parseHtml()
if (doc.select("script:containsData(page_image)").isNullOrEmpty()) {
return doc.select(selectPage).map { img ->
val url = img.attr("onerror").replace("this.onerror=null;this.src=`", "").replace("`;", "")
MangaPage(
id = generateUid(url),
url = url,
preview = null,
source = source,
)
}
} else {
val script = doc.selectFirstOrThrow("script:containsData(page_image)")
val images = JSONArray(script.data().substringAfterLast("var pages = ").substringBefore(';'))
val pages = ArrayList<MangaPage>(images.length())
for (i in 0 until images.length()) {
val pageTake = images.getJSONObject(i)
pages.add(
MangaPage(
id = generateUid(pageTake.getString("page_image")),
url = pageTake.getString("page_image"),
preview = null,
source = source,
),
)
}
return pages
}
}
}

View File

@@ -1,149 +0,0 @@
package org.koitharu.kotatsu.parsers.site.animebootstrap.fr
import kotlinx.coroutines.async
import kotlinx.coroutines.coroutineScope
import org.jsoup.nodes.Document
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.site.animebootstrap.AnimeBootstrapParser
import org.koitharu.kotatsu.parsers.util.*
import java.text.SimpleDateFormat
import java.util.EnumSet
import java.util.Locale
@MangaSourceParser("PAPSCAN", "PapScan", "fr")
internal class PapScan(context: MangaLoaderContext) :
AnimeBootstrapParser(context, MangaSource.PAPSCAN, "papscan.com") {
override val sourceLocale: Locale = Locale.ENGLISH
override val isMultipleTagsSupported = false
override val listUrl = "/liste-manga"
override val selectState = "div.anime__details__widget li:contains(En cours)"
override val selectTag = "div.anime__details__widget li:contains(Genre) a"
override val selectChapter = "ul.chapters li"
override val availableSortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.POPULARITY,
SortOrder.ALPHABETICAL,
)
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val tag = tags.oneOrThrowIfMany()
val url = buildString {
append("https://")
append(domain)
append("/filterList")
append("?page=")
append(page.toString())
if (!query.isNullOrEmpty()) {
append("&alpha=")
append(query.urlEncoded())
}
if (!tags.isNullOrEmpty()) {
append("&cat=")
append(tag?.key.orEmpty())
}
append("&sortBy=")
when (sortOrder) {
SortOrder.POPULARITY -> append("views")
SortOrder.ALPHABETICAL -> append("name")
else -> append("updated")
}
}
val doc = webClient.httpGet(url).parseHtml()
return doc.select("div.product__item").map { div ->
val href = div.selectFirstOrThrow("h5 a").attrAsRelativeUrl("href")
Manga(
id = generateUid(href),
url = href,
publicUrl = href.toAbsoluteUrl(div.host ?: domain),
coverUrl = div.selectFirstOrThrow("div.product__item__pic").attr("data-setbg").orEmpty(),
title = div.selectFirstOrThrow("div.product__item__text h5").text().orEmpty(),
altTitle = null,
rating = RATING_UNKNOWN,
tags = emptySet(),
author = null,
state = null,
source = source,
isNsfw = isNsfwSource,
)
}
}
override suspend fun getAvailableTags(): Set<MangaTag> {
val doc = webClient.httpGet("https://$domain$listUrl").parseHtml()
return doc.select("a.category ").mapNotNullToSet { a ->
val key = a.attr("href").substringAfterLast('=')
val name = a.text()
MangaTag(
key = key,
title = name,
source = source,
)
}
}
override suspend fun getDetails(manga: Manga): Manga = coroutineScope {
val fullUrl = manga.url.toAbsoluteUrl(domain)
val doc = webClient.httpGet(fullUrl).parseHtml()
val chaptersDeferred = async { getChapters(doc) }
val desc = doc.selectFirstOrThrow(selectDesc).html()
val state = if (doc.select(selectState).isNullOrEmpty()) {
MangaState.FINISHED
} else {
MangaState.ONGOING
}
manga.copy(
tags = doc.body().select(selectTag).mapNotNullToSet { a ->
MangaTag(
key = a.attr("href").removeSuffix('/').substringAfterLast('/'),
title = a.text().toTitleCase(),
source = source,
)
},
description = desc,
state = state,
chapters = chaptersDeferred.await(),
)
}
override suspend fun getChapters(doc: Document): List<MangaChapter> {
val dateFormat = SimpleDateFormat(datePattern, sourceLocale)
return doc.body().select(selectChapter).mapChapters(reversed = true) { i, li ->
val href = li.selectFirstOrThrow("a").attr("href")
val dateText = li.selectFirst("span.date-chapter-title-rtl")?.text()
MangaChapter(
id = generateUid(href),
name = li.selectFirstOrThrow("span em").text(),
number = i + 1,
url = href,
uploadDate = dateFormat.tryParse(dateText),
source = source,
scanlator = null,
branch = null,
)
}
}
}

View File

@@ -1,10 +0,0 @@
package org.koitharu.kotatsu.parsers.site.animebootstrap.id
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.site.animebootstrap.AnimeBootstrapParser
@MangaSourceParser("KOMIKZOID", "Komikzo Id", "id")
internal class KomikzoId(context: MangaLoaderContext) :
AnimeBootstrapParser(context, MangaSource.KOMIKZOID, "komikzoid.xyz")

View File

@@ -1,12 +0,0 @@
package org.koitharu.kotatsu.parsers.site.animebootstrap.id
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.site.animebootstrap.AnimeBootstrapParser
@MangaSourceParser("NEUMANGA", "NeuManga", "id")
internal class NeuManga(context: MangaLoaderContext) :
AnimeBootstrapParser(context, MangaSource.NEUMANGA, "neumanga.xyz")

View File

@@ -1,11 +0,0 @@
package org.koitharu.kotatsu.parsers.site.animebootstrap.id
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.site.animebootstrap.AnimeBootstrapParser
@MangaSourceParser("SEKTEKOMIK", "SekteKomik", "id")
internal class SekteKomik(context: MangaLoaderContext) :
AnimeBootstrapParser(context, MangaSource.SEKTEKOMIK, "sektekomik.xyz")

View File

@@ -1,167 +0,0 @@
package org.koitharu.kotatsu.parsers.site.ar
import kotlinx.coroutines.async
import kotlinx.coroutines.coroutineScope
import org.json.JSONArray
import org.json.JSONObject
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import org.koitharu.kotatsu.parsers.util.json.mapJSON
import org.koitharu.kotatsu.parsers.util.json.mapJSONIndexed
import java.text.SimpleDateFormat
import java.util.*
@MangaSourceParser("FLIXSCANS", "FlixScans", "ar")
internal class FlixScans(context: MangaLoaderContext) : PagedMangaParser(context, MangaSource.FLIXSCANS, 18) {
override val availableSortOrders: Set<SortOrder> = EnumSet.of(SortOrder.UPDATED)
override val configKeyDomain = ConfigKey.Domain("flixscans.com")
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val json = if (!query.isNullOrEmpty()) {
if (page > 1) {
return emptyList()
}
val url = "https://api.$domain/api/v1/search/serie"
val body = JSONObject()
body.put("title", query.urlEncoded())
webClient.httpPost(url, body).parseJson().getJSONArray("data")
} else if (!tags.isNullOrEmpty()) {
if (page > 1) {
return emptyList()
}
val tagQuery = tags.joinToString(separator = ",") { it.key }
val url = "https://api.$domain/api/v1/search/advance?=&genres=$tagQuery&serie_type=webtoon"
webClient.httpGet(url).parseJson().getJSONArray("data")
} else {
val url = "https://api.$domain/api/v1/webtoon/homepage/latest/home?page=$page"
webClient.httpGet(url).parseJson().getJSONArray("data")
}
return json.mapJSON { j ->
val href = "https://$domain/series/${j.getString("prefix")}-${j.getString("id")}-${j.getString("slug")}"
val cover = "https://api.$domain/storage/" + j.getString("thumbnail")
Manga(
id = generateUid(href),
title = j.getString("title"),
altTitle = null,
url = href,
publicUrl = href.toAbsoluteUrl(domain),
rating = RATING_UNKNOWN,
isNsfw = false,
coverUrl = cover,
tags = emptySet(),
state = when (j.getString("status")) {
"ongoing" -> MangaState.ONGOING
"completed" -> MangaState.FINISHED
else -> null
},
author = null,
source = source,
)
}
}
override suspend fun getAvailableTags(): Set<MangaTag> {
val doc = webClient.httpGet("https://$domain/search/advance").parseHtml()
val json = JSONArray(doc.requireElementById("__NUXT_DATA__").data())
val tagsList = json.getJSONArray(3).toString().replace("[", "").replace("]", "").split(",")
return tagsList.mapNotNullToSet { idTag ->
val id = idTag.toInt()
val idKey = json.getJSONObject(id).getInt("id")
val key = json.get(idKey).toString()
val idName = json.getJSONObject(id).getInt("name")
val name = json.get(idName).toString()
MangaTag(
key = key,
title = name,
source = source,
)
}
}
override suspend fun getDetails(manga: Manga): Manga = coroutineScope {
val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml()
val chaptersDeferred = async { loadChapters(manga.url) }
val json = JSONArray(doc.requireElementById("__NUXT_DATA__").data())
val descId = json.getJSONObject(6).getInt("story")
val desc = json.getString(descId)
val tagsId = json.getJSONObject(6).getInt("genres")
val tagsList = json.getJSONArray(tagsId).toString().replace("[", "").replace("]", "").split(",")
val ratingId = json.getJSONObject(6).getInt("rating")
val rating = json.getString(ratingId)
val nsfwId = json.getJSONObject(6).getInt("nsfw")
val nsfw = json.getBoolean(nsfwId)
manga.copy(
description = desc,
tags = tagsList.mapToSet { idTag ->
val id = idTag.toInt()
val idKey = json.getJSONObject(id).getInt("id")
val key = json.get(idKey).toString()
val idName = json.getJSONObject(id).getInt("name")
val name = json.get(idName).toString()
MangaTag(
key = key,
title = name,
source = source,
)
},
rating = rating?.toFloatOrNull()?.div(5f) ?: RATING_UNKNOWN,
isNsfw = nsfw,
chapters = chaptersDeferred.await(),
)
}
private val dateFormat = SimpleDateFormat("yyyy-MM-dd", sourceLocale)
private suspend fun loadChapters(baseUrl: String): List<MangaChapter> {
val key = baseUrl.substringAfter("-").substringBefore("-")
val seriesKey = baseUrl.substringAfterLast("/").substringBefore("-")
val json = JSONArray(webClient.httpGet("https://api.$domain/api/v1/webtoon/chapters/$key-desc").parseRaw())
return json.mapJSONIndexed { i, j ->
val url = "https://$domain/read/webtoon/$seriesKey-${j.getString("id")}-${j.getString("slug")}"
val date = j.getString("createdAt").substringBeforeLast("T")
MangaChapter(
id = generateUid(url),
url = url,
name = j.getString("slug").replace('-', ' '),
number = i + 1,
branch = null,
uploadDate = dateFormat.tryParse(date),
scanlator = null,
source = source,
)
}
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(domain)
val doc = webClient.httpGet(fullUrl).parseHtml()
val json = JSONArray(doc.requireElementById("__NUXT_DATA__").data())
val chapterData = json.getJSONObject(6).getInt("chapterData")
val pageLocate = json.getJSONObject(chapterData).getInt("webtoon")
val jsonPages = json.getJSONArray(pageLocate)
val pages = ArrayList<MangaPage>(jsonPages.length())
for (i in 0 until jsonPages.length()) {
val id = jsonPages.getInt(i)
val url = "https://api.$domain/storage/" + json.getString(id)
pages.add(
MangaPage(
id = generateUid(url),
url = url,
preview = null,
source = source,
),
)
}
return pages
}
}

View File

@@ -1,121 +0,0 @@
package org.koitharu.kotatsu.parsers.site.ar
import okhttp3.Headers
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.network.UserAgents
import org.koitharu.kotatsu.parsers.util.*
import java.util.*
@MangaSourceParser("MANGASTORM", "MangaStorm", "ar")
internal class MangaStorm(context: MangaLoaderContext) : PagedMangaParser(context, MangaSource.MANGASTORM, 30) {
override val availableSortOrders: Set<SortOrder> = EnumSet.of(SortOrder.POPULARITY)
override val configKeyDomain = ConfigKey.Domain("mangastorm.org")
override val isMultipleTagsSupported = false
override val headers: Headers = Headers.Builder()
.add("User-Agent", UserAgents.CHROME_DESKTOP)
.build()
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val tag = tags.oneOrThrowIfMany()
val url =
if (!tags.isNullOrEmpty()) {
buildString {
append("https://")
append(domain)
append("/categories/")
append(tag?.key.orEmpty())
append("?page=")
append(page)
}
} else {
buildString {
append("https://")
append(domain)
append("/mangas?page=")
append(page)
if (!query.isNullOrEmpty()) {
append("&query=")
append(query.urlEncoded())
}
}
}
val doc = webClient.httpGet(url).parseHtml()
return doc.select("div.row div.col").map { div ->
val href = div.selectFirstOrThrow("a").attrAsRelativeUrl("href")
Manga(
id = generateUid(href),
title = div.select(".manga-ct-title").text(),
altTitle = null,
url = href,
publicUrl = href.toAbsoluteUrl(domain),
rating = RATING_UNKNOWN,
isNsfw = false,
coverUrl = div.selectFirstOrThrow("img").src().orEmpty(),
tags = emptySet(),
state = null,
author = null,
source = source,
)
}
}
override suspend fun getAvailableTags(): Set<MangaTag> = emptySet()
override suspend fun getDetails(manga: Manga): Manga {
val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml()
val root = doc.selectFirstOrThrow(".card-body .col-lg-9")
return manga.copy(
altTitle = null,
state = null,
tags = root.select(".flex-wrap a").mapNotNullToSet { a ->
MangaTag(
key = a.attr("href").substringAfterLast('/'),
title = a.text(),
source = source,
)
},
author = null,
description = root.selectFirstOrThrow(".card-text").text(),
chapters = doc.select(".card-body a.btn-fixed-width").mapChapters(reversed = true) { i, a ->
val url = a.attrAsRelativeUrl("href")
MangaChapter(
id = generateUid(url),
name = a.text(),
number = i + 1,
url = url,
scanlator = null,
uploadDate = 0,
branch = null,
source = source,
)
},
)
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(domain)
val doc = webClient.httpGet(fullUrl).parseHtml().requireElementById("content")
return doc.select("div.text-center .img-fluid").map { img ->
val url = img.src()?.toRelativeUrl(domain) ?: img.parseFailed("Image src not found")
MangaPage(
id = generateUid(url),
url = url,
preview = null,
source = source,
)
}
}
}

View File

@@ -1,182 +0,0 @@
package org.koitharu.kotatsu.parsers.site.ar
import kotlinx.coroutines.async
import kotlinx.coroutines.awaitAll
import kotlinx.coroutines.coroutineScope
import org.jsoup.nodes.Element
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import java.text.SimpleDateFormat
import java.util.*
@MangaSourceParser("TEAMXNOVEL", "TeamXNovel", "ar")
internal class TeamXNovel(context: MangaLoaderContext) : PagedMangaParser(context, MangaSource.TEAMXNOVEL, 10) {
override val availableSortOrders: Set<SortOrder> = EnumSet.of(SortOrder.UPDATED, SortOrder.POPULARITY)
override val configKeyDomain = ConfigKey.Domain("team11x11.com")
override val isMultipleTagsSupported = false
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val tag = tags.oneOrThrowIfMany()
val url = buildString {
append("https://")
append(domain)
if (!tags.isNullOrEmpty()) {
append("/series?genre=")
append(tag?.key.orEmpty())
if (page > 1) {
append("&page=")
append(page)
}
} else if (!query.isNullOrEmpty()) {
append("/series?search=")
append(query.urlEncoded())
if (page > 1) {
append("&page=")
append(page)
}
} else {
when (sortOrder) {
SortOrder.POPULARITY -> append("/series")
SortOrder.UPDATED -> append("/")
else -> append("/")
}
if (page > 1) {
append("?page=")
append(page)
}
}
}
val doc = webClient.httpGet(url).parseHtml()
return doc.select("div.listupd .bs .bsx").ifEmpty {
doc.select("div.post-body .box")
}.map { div ->
val href = div.selectFirstOrThrow("a").attrAsRelativeUrl("href")
Manga(
id = generateUid(href),
title = div.select(".tt, h3").text(),
altTitle = null,
url = href,
publicUrl = href.toAbsoluteUrl(domain),
rating = RATING_UNKNOWN,
isNsfw = false,
coverUrl = div.selectFirstOrThrow("img").src().orEmpty(),
tags = emptySet(),
state = when (div.selectFirst(".status")?.text()) {
"مستمرة" -> MangaState.ONGOING
"متوقف", "مكتمل" -> MangaState.FINISHED
else -> null
},
author = null,
source = source,
)
}
}
override suspend fun getAvailableTags(): Set<MangaTag> {
val doc = webClient.httpGet("https://$domain/series").parseHtml()
return doc.requireElementById("select_genre").select("option").mapNotNullToSet {
MangaTag(
key = it.attr("value"),
title = it.text(),
source = source,
)
}
}
override suspend fun getDetails(manga: Manga): Manga {
val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml()
val mangaUrl = manga.url.toAbsoluteUrl(domain)
val maxPageChapterSelect = doc.select(".pagination .page-item a")
var maxPageChapter = 1
if (!maxPageChapterSelect.isNullOrEmpty()) {
maxPageChapterSelect.map {
val i = it.attr("href").substringAfterLast("=").toInt()
if (i > maxPageChapter) {
maxPageChapter = i
}
}
}
return manga.copy(
altTitle = null,
state = when (doc.selectFirstOrThrow(".full-list-info:contains(الحالة:) a").text()) {
"مستمرة" -> MangaState.ONGOING
"متوقف", "مكتمل" -> MangaState.FINISHED
else -> null
},
tags = doc.select(".review-author-info a").mapNotNullToSet { a ->
MangaTag(
key = a.attr("href").substringAfterLast("="),
title = a.text(),
source = source,
)
},
author = null,
description = doc.selectFirstOrThrow(".review-content").text(),
chapters = run {
if (maxPageChapter == 1) {
parseChapters(doc)
} else {
coroutineScope {
val result = ArrayList(parseChapters(doc))
result.ensureCapacity(result.size * maxPageChapter)
(2..maxPageChapter).map { i ->
async {
loadChapters(mangaUrl, i)
}
}.awaitAll()
.flattenTo(result)
result
}
}
}.reversed(),
)
}
private suspend fun loadChapters(baseUrl: String, page: Int): List<MangaChapter> {
return parseChapters(webClient.httpGet("$baseUrl?page=$page").parseHtml().body())
}
private val dateFormat = SimpleDateFormat("yyyy-MM-dd HH:mm:ss", sourceLocale)
private fun parseChapters(root: Element): List<MangaChapter> {
return root.requireElementById("chapter-contact").select(".eplister ul li")
.map { li ->
val url = li.selectFirstOrThrow("a").attrAsRelativeUrl("href")
MangaChapter(
id = generateUid(url),
name = li.selectFirstOrThrow(".epl-title").text(),
number = url.substringAfterLast('/').toIntOrNull() ?: 0,
url = url,
scanlator = null,
uploadDate = dateFormat.tryParse(li.selectFirstOrThrow(".epl-date").text()),
branch = null,
source = source,
)
}
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(domain)
val doc = webClient.httpGet(fullUrl).parseHtml()
return doc.select(".image_list img").map { img ->
val url = img.src()?.toRelativeUrl(domain) ?: img.parseFailed("Image src not found")
MangaPage(
id = generateUid(url),
url = url,
preview = null,
source = source,
)
}
}
}

View File

@@ -1,138 +0,0 @@
package org.koitharu.kotatsu.parsers.site.en
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import java.text.SimpleDateFormat
import java.util.*
@MangaSourceParser("BEETOON", "BeeToon.net", "en")
internal class BeeToon(context: MangaLoaderContext) :
PagedMangaParser(context, MangaSource.BEETOON, pageSize = 30) {
override val availableSortOrders: Set<SortOrder> = EnumSet.of(SortOrder.UPDATED, SortOrder.POPULARITY)
override val configKeyDomain = ConfigKey.Domain("ww7.beetoon.net")
override val isMultipleTagsSupported = false
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val tag = tags.oneOrThrowIfMany()
val url = buildString {
append("https://")
append(domain)
when {
!query.isNullOrEmpty() -> {
if (page > 1) {
return emptyList()
}
append("/?s=")
append(query.urlEncoded())
}
!tags.isNullOrEmpty() -> {
append("/genre/")
append(tag?.key.orEmpty())
append("/page-")
append(page)
append("/")
}
else -> {
when (sortOrder) {
SortOrder.UPDATED -> append("/latest-update/")
SortOrder.POPULARITY -> append("/popular-manga/")
else -> append("/latest-update/")
}
append("page-")
append(page)
append("/")
}
}
}
val doc = webClient.httpGet(url).parseHtml()
return doc.select(".comics-grid .entry").map { div ->
val href = div.selectFirstOrThrow("a").attrAsRelativeUrl("href")
Manga(
id = generateUid(href),
url = href,
publicUrl = href.toAbsoluteUrl(domain),
coverUrl = div.selectFirst("img")?.src().orEmpty(),
title = div.selectFirst(".name")?.text().orEmpty(),
altTitle = null,
rating = div.selectFirst(".counter")?.text()?.toFloatOrNull()?.div(5f) ?: RATING_UNKNOWN,
tags = emptySet(),
author = null,
state = when (div.selectLastOrThrow(".status span").text()) {
"Ongoing" -> MangaState.ONGOING
"Completed" -> MangaState.FINISHED
else -> null
},
source = source,
isNsfw = isNsfwSource,
)
}
}
override suspend fun getAvailableTags(): Set<MangaTag> {
val doc = webClient.httpGet("https://$domain/").parseHtml()
return doc.requireElementById("menu-item-3").select("ul.sub-menu li a").mapNotNullToSet {
MangaTag(
key = it.attr("href").removeSuffix('/').substringAfterLast('/'),
title = it.text(),
source = source,
)
}
}
override suspend fun getDetails(manga: Manga): Manga {
val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml()
return manga.copy(
description = doc.getElementById("desc")?.text().orEmpty(),
rating = doc.selectFirst(".counter")?.text()?.toFloatOrNull()?.div(5f) ?: RATING_UNKNOWN,
tags = doc.body().select(".info .genre a").mapNotNullToSet {
MangaTag(
key = it.attr("href").removeSuffix('/').substringAfterLast('/'),
title = it.text(),
source = source,
)
},
author = doc.selectFirst(".info .author a")?.text(),
chapters = doc.select(".items-chapters a").mapChapters(reversed = true) { i, a ->
val url = a.attrAsRelativeUrl("href").toAbsoluteUrl(domain)
MangaChapter(
id = generateUid(url),
name = a.selectFirstOrThrow(".chap").text(),
number = i + 1,
url = url,
scanlator = null,
uploadDate = SimpleDateFormat("MM/dd/yyyy HH:mm:ss", Locale.ENGLISH)
.tryParse(a.selectFirst(".chapter-date")?.attr("title") ?: "0"),
branch = null,
source = source,
)
},
)
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml()
return doc.select(".chapter-content-inner center img").map { img ->
val urlPage = img.src()?.toRelativeUrl(domain) ?: img.parseFailed("Image src not found")
MangaPage(
id = generateUid(urlPage),
url = urlPage,
preview = null,
source = source,
)
}
}
}

View File

@@ -1,149 +0,0 @@
package org.koitharu.kotatsu.parsers.site.en
import okhttp3.Headers
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.network.UserAgents
import org.koitharu.kotatsu.parsers.util.*
import java.text.SimpleDateFormat
import java.util.*
@MangaSourceParser("COMICEXTRA", "ComicExtra", "en")
internal class ComicExtra(context: MangaLoaderContext) : PagedMangaParser(context, MangaSource.COMICEXTRA, 25) {
override val availableSortOrders: Set<SortOrder> =
EnumSet.of(SortOrder.POPULARITY, SortOrder.UPDATED, SortOrder.NEWEST)
override val configKeyDomain = ConfigKey.Domain("comicextra.me")
override val isMultipleTagsSupported = false
override val headers: Headers = Headers.Builder()
.add("User-Agent", UserAgents.CHROME_DESKTOP)
.build()
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val tag = tags.oneOrThrowIfMany()
val url = buildString {
append("https://$domain/")
if (!tags.isNullOrEmpty()) {
append(tag?.key.orEmpty())
if (page > 1) {
append("/")
append(page)
}
} else if (!query.isNullOrEmpty()) {
append("comic-search?key=")
append(query.urlEncoded())
if (page > 1) {
append("&page=")
append(page)
}
} else {
when (sortOrder) {
SortOrder.POPULARITY -> append("popular-comic/")
SortOrder.UPDATED -> append("new-comic/")
SortOrder.NEWEST -> append("recent-comic/")
else -> append("new-comic/")
}
if (page > 1) {
append(page)
}
}
}
val doc = webClient.httpGet(url).parseHtml()
return doc.select("div.movie-list-index div.cartoon-box").map { div ->
val href = div.selectFirstOrThrow("a").attrAsRelativeUrl("href")
Manga(
id = generateUid(href),
title = div.selectFirstOrThrow("h3").text(),
altTitle = null,
url = href,
publicUrl = href.toAbsoluteUrl(domain),
rating = RATING_UNKNOWN,
isNsfw = false,
coverUrl = div.selectFirstOrThrow("img").attrAsAbsoluteUrl("src"),
tags = emptySet(),
state = when (div.selectFirstOrThrow(".detail:contains(Stasus: )").text()) {
"Stasus: Ongoing" -> MangaState.ONGOING
"Stasus: Completed" -> MangaState.FINISHED
else -> null
},
author = null,
source = source,
)
}
}
override suspend fun getAvailableTags(): Set<MangaTag> {
val doc = webClient.httpGet("https://$domain/popular-comic").parseHtml()
return doc.select("li.tag-item a").mapNotNullToSet { a ->
MangaTag(
key = a.attr("href").substringAfterLast('/'),
title = a.text(),
source = source,
)
}
}
override suspend fun getDetails(manga: Manga): Manga {
val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml()
val dateFormat = SimpleDateFormat("MM/dd/yy", sourceLocale)
return manga.copy(
altTitle = doc.selectFirstOrThrow("dt.movie-dt:contains(Alternate name:) + dd").text(),
state = when (doc.selectFirstOrThrow("dt.movie-dt:contains(Status:) + dd a").text()) {
"Ongoing" -> MangaState.ONGOING
"Completed" -> MangaState.FINISHED
else -> null
},
tags = doc.select("dt.movie-dt:contains(Genres:) + dd a").mapNotNullToSet { a ->
MangaTag(
key = a.attr("href").substringAfterLast("/"),
title = a.text(),
source = source,
)
},
author = doc.select("dt.movie-dt:contains(Author:) + dd").text(),
description = doc.getElementById("film-content")?.text(),
chapters = doc.requireElementById("list").select("tr")
.mapChapters(reversed = true) { i, tr ->
val a = tr.selectFirstOrThrow("a")
val url = a.attrAsRelativeUrl("href") + "/full"
val name = a.text()
val dateText = tr.select("td").last()?.text()
MangaChapter(
id = generateUid(url),
name = name,
number = i + 1,
url = url,
scanlator = null,
uploadDate = dateFormat.tryParse(dateText),
branch = null,
source = source,
)
},
)
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(domain)
val doc = webClient.httpGet(fullUrl).parseHtml()
return doc.select(".chapter-container img").map { img ->
val url = img.src()?.toRelativeUrl(domain) ?: img.parseFailed("Image src not found")
MangaPage(
id = generateUid(url),
url = url,
preview = null,
source = source,
)
}
}
}

View File

@@ -1,170 +0,0 @@
package org.koitharu.kotatsu.parsers.site.en
import okhttp3.Headers
import org.json.JSONArray
import org.jsoup.nodes.Document
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.network.UserAgents
import org.koitharu.kotatsu.parsers.util.*
import java.text.SimpleDateFormat
import java.util.*
@MangaSourceParser("DYNASTYSCANS", "DynastyScans", "en")
internal class DynastyScans(context: MangaLoaderContext) : PagedMangaParser(context, MangaSource.DYNASTYSCANS, 117) {
override val availableSortOrders: Set<SortOrder> = EnumSet.of(SortOrder.ALPHABETICAL)
override val configKeyDomain = ConfigKey.Domain("dynasty-scans.com")
override val headers: Headers = Headers.Builder()
.add("User-Agent", UserAgents.CHROME_DESKTOP)
.build()
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val url = buildString {
append("https://")
append(domain)
if (!query.isNullOrEmpty()) {
append("/search?q=")
append(query.urlEncoded())
append("&")
append("classes[]".urlEncoded())
append("=Serie&page=")
append(page.toString())
} else if (!tags.isNullOrEmpty()) {
append("/tags/")
for (tag in tags) {
append(tag.key)
}
append("?view=groupings&page=")
append(page.toString())
} else {
append("/series?view=cover&page=")
append(page.toString())
}
}
val doc = webClient.httpGet(url).parseHtml()
// There are no images on the search page
if (!query.isNullOrEmpty()) {
return doc.select("dl.chapter-list dd")
.map { div ->
val href = div.selectFirstOrThrow("a").attrAsRelativeUrl("href")
Manga(
id = generateUid(href),
title = div.selectFirstOrThrow("a").text(),
altTitle = null,
url = href,
publicUrl = href.toAbsoluteUrl(domain),
rating = RATING_UNKNOWN,
isNsfw = false,
coverUrl = "",
tags = div.select("span.tags a").mapNotNullToSet { a ->
MangaTag(
key = a.attr("href").removeSuffix('/').substringAfterLast('/'),
title = a.text(),
source = source,
)
},
state = null,
author = null,
source = source,
)
}
} else {
return doc.select("li.span2")
.map { div ->
val href = div.selectFirstOrThrow("a").attrAsRelativeUrl("href")
Manga(
id = generateUid(href),
title = div.selectFirstOrThrow("div.caption").text(),
altTitle = null,
url = href,
publicUrl = href.toAbsoluteUrl(domain),
rating = RATING_UNKNOWN,
isNsfw = false,
coverUrl = div.selectFirstOrThrow("img").attrAsAbsoluteUrl("src"),
tags = setOf(),
state = null,
author = null,
source = source,
)
}
}
}
override suspend fun getAvailableTags(): Set<MangaTag> = emptySet()
override suspend fun getDetails(manga: Manga): Manga {
val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml()
val chapters = getChapters(doc)
val root = doc.requireElementById("main")
return manga.copy(
altTitle = null,
state = when (root.select("h2.tag-title small").last()?.text()) {
"— Ongoing" -> MangaState.ONGOING
"— Completed" -> MangaState.FINISHED
else -> null
},
coverUrl = root.selectFirst("img.thumbnail")?.src()
.orEmpty(), // It is needed if the manga was found via the search.
tags = root.select("div.tag-tags a").mapNotNullToSet { a ->
val href = a.attr("href").removeSuffix('/').substringAfterLast('/')
MangaTag(
key = href,
title = a.text(),
source = source,
)
},
author = null,
description = null,
chapters = chapters,
)
}
private fun getChapters(doc: Document): List<MangaChapter> {
val dateFormat = SimpleDateFormat("MMM dd yy", sourceLocale)
return doc.body().select("dl.chapter-list dd").mapChapters { i, li ->
val a = li.selectFirstOrThrow("a")
val href = a.attrAsRelativeUrl("href")
val dateText = li.select("small").last()?.text()?.replace("released ", "")?.replace("'", "")
MangaChapter(
id = generateUid(href),
name = a.text(),
number = i + 1,
url = href,
uploadDate = dateFormat.tryParse(dateText),
source = source,
scanlator = null,
branch = null,
)
}
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val chapterUrl = chapter.url.toAbsoluteUrl(domain)
val docs = webClient.httpGet(chapterUrl).parseHtml()
val script = docs.selectFirstOrThrow("script:containsData(var pages =)")
val json = JSONArray(script.data().substringAfter('=').substringBeforeLast(';'))
val pages = ArrayList<MangaPage>(json.length())
for (i in 0 until json.length()) {
val url = "https://" + domain + json.getString(i).substringAfter(":\"").substringBefore("\",")
pages.add(
MangaPage(
id = generateUid(url),
url = url,
preview = null,
source = source,
),
)
}
return pages
}
}

View File

@@ -1,148 +0,0 @@
package org.koitharu.kotatsu.parsers.site.en
import kotlinx.coroutines.async
import kotlinx.coroutines.coroutineScope
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import org.koitharu.kotatsu.parsers.util.json.mapJSONToSet
import java.util.*
@MangaSourceParser("FAKKU", "Fakku", "en", ContentType.HENTAI)
internal class Fakku(context: MangaLoaderContext) :
PagedMangaParser(context, MangaSource.FAKKU, pageSize = 25) {
override val availableSortOrders: Set<SortOrder> =
EnumSet.of(SortOrder.ALPHABETICAL, SortOrder.NEWEST, SortOrder.UPDATED)
override val configKeyDomain = ConfigKey.Domain("fakku.cc")
override val isMultipleTagsSupported = false
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val tag = tags.oneOrThrowIfMany()
val url = buildString {
append("https://")
append(domain)
when {
!query.isNullOrEmpty() -> {
append("/search?q=")
append(query.urlEncoded())
append("&")
}
!tags.isNullOrEmpty() -> {
append("/tags/")
append(tag?.key.orEmpty())
append("?")
}
else -> {
append("?")
}
}
append("page=")
append(page)
append("&sort=")
when (sortOrder) {
SortOrder.ALPHABETICAL -> append("title")
SortOrder.NEWEST -> append("created_at")
SortOrder.UPDATED -> append("published_at")
else -> append("published_at")
}
}
val doc = webClient.httpGet(url).parseHtml()
return doc.select("div.entries .entry a").map { a ->
val href = a.attrAsRelativeUrl("href")
Manga(
id = generateUid(href),
url = href,
publicUrl = href.toAbsoluteUrl(domain),
coverUrl = a.selectFirst("img")?.src().orEmpty(),
title = a.selectFirst(".title")?.text().orEmpty(),
altTitle = null,
rating = RATING_UNKNOWN,
tags = emptySet(),
author = null,
state = null,
source = source,
isNsfw = isNsfwSource,
)
}
}
override suspend fun getAvailableTags(): Set<MangaTag> {
val root = webClient.httpGet("https://$domain/tags").parseHtml()
return root.select("div.entries .entry a").mapToSet {
MangaTag(
key = it.attr("href").substringAfterLast("/"),
title = it.selectFirstOrThrow(".name").text(),
source = source,
)
}
}
override suspend fun getDetails(manga: Manga): Manga = coroutineScope {
val fullUrl = manga.url.toAbsoluteUrl(domain)
val doc = webClient.httpGet(fullUrl).parseHtml()
val genreDeferred = async {
webClient.httpGet(manga.url.toAbsoluteUrl(domain) + ".json").parseJson()
}
val genre = genreDeferred.await()
manga.copy(
author = doc.selectFirst("tr.artists a")?.text(),
tags = if (genre.toString().contains("tags")) {
genre.getJSONArray("tags").mapJSONToSet {
MangaTag(
key = it.getString("slug"),
title = it.getString("name"),
source = source,
)
}
} else {
emptySet()
},
chapters = listOf(
MangaChapter(
id = manga.id,
name = manga.title,
number = 1,
url = manga.url + "/1",
scanlator = null,
uploadDate = 0,
branch = null,
source = source,
),
),
)
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml()
val totalPages = doc.selectFirstOrThrow(".total").text().toInt()
val rawUrl = chapter.url.substringBeforeLast("/")
return (1..totalPages).map {
val url = "$rawUrl/$it"
MangaPage(
id = generateUid(url),
url = url,
preview = null,
source = source,
)
}
}
override suspend fun getPageUrl(page: MangaPage): String {
val doc = webClient.httpGet(page.url.toAbsoluteUrl(domain)).parseHtml()
val root = doc.body()
return root.selectFirstOrThrow(".page img").attrAsAbsoluteUrl("src")
}
}

View File

@@ -1,187 +0,0 @@
package org.koitharu.kotatsu.parsers.site.en
import androidx.collection.ArraySet
import kotlinx.coroutines.async
import kotlinx.coroutines.awaitAll
import kotlinx.coroutines.coroutineScope
import okhttp3.HttpUrl.Companion.toHttpUrlOrNull
import org.json.JSONArray
import org.jsoup.nodes.Element
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import org.koitharu.kotatsu.parsers.util.json.mapJSON
import java.text.SimpleDateFormat
import java.util.*
@MangaSourceParser("KSKMOE", "Ksk.moe", "en", ContentType.HENTAI)
internal class KskMoe(context: MangaLoaderContext) : PagedMangaParser(context, MangaSource.KSKMOE, 35) {
override val availableSortOrders: Set<SortOrder> =
EnumSet.of(SortOrder.UPDATED, SortOrder.POPULARITY, SortOrder.NEWEST, SortOrder.ALPHABETICAL)
override val configKeyDomain = ConfigKey.Domain("ksk.moe")
override val isMultipleTagsSupported = false
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val tag = tags.oneOrThrowIfMany()
val url = buildString {
append("https://")
append(domain)
if (!tags.isNullOrEmpty()) {
append("/tags/")
append(tag?.key.orEmpty())
} else {
append("/browse")
}
if (page > 1) {
append("/page/")
append(page)
}
when (sortOrder) {
SortOrder.POPULARITY -> append("?sort=32")
SortOrder.UPDATED -> append("")
SortOrder.NEWEST -> append("?sort=16")
SortOrder.ALPHABETICAL -> append("?sort=1")
else -> append("")
}
if (!query.isNullOrEmpty()) {
append("?s=")
append(query.urlEncoded())
}
}
val doc = webClient.httpGet(url).parseHtml()
if (!doc.html().contains("pagination") && page > 1) {
return emptyList()
}
return doc.requireElementById("galleries").select("article").map { div ->
val href = div.selectFirstOrThrow("a").attrAsRelativeUrl("href")
Manga(
id = generateUid(href),
title = div.selectLastOrThrow("h3 span").text(),
altTitle = null,
url = href,
publicUrl = href.toAbsoluteUrl(domain),
rating = RATING_UNKNOWN,
isNsfw = true,
coverUrl = div.selectFirstOrThrow("img").src()?.toAbsoluteUrl(domain).orEmpty(),
tags = div.select("footer span").mapNotNullToSet { span ->
MangaTag(
key = span.text().urlEncoded(),
title = span.text(),
source = source,
)
},
state = null,
author = null,
source = source,
)
}
}
override suspend fun getAvailableTags(): Set<MangaTag> {
return coroutineScope {
(1..2).map { page ->
async { getTags(page) }
}
}.awaitAll().flattenTo(ArraySet(360))
}
private suspend fun getTags(page: Int): Set<MangaTag> {
val url = if (page == 1) {
"https://$domain/tags"
} else {
"https://$domain/tags/page/$page"
}
val root = webClient.httpGet(url).parseHtml().body().getElementById("tags")
return root?.parseTags().orEmpty()
}
private fun Element.parseTags() = select("section.tags div a").mapToSet { a ->
MangaTag(
key = a.attr("href").substringAfterLast("/tags/"),
title = a.selectFirstOrThrow("span").text(),
source = source,
)
}
private val date = SimpleDateFormat("dd.MM.yyyy hh:mm 'UTC'", Locale.US)
override suspend fun getDetails(manga: Manga): Manga {
val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml()
return manga.copy(
tags = doc.requireElementById("metadata").select("main div:contains(Tag) a").mapNotNullToSet { a ->
MangaTag(
key = a.attr("href").substringAfterLast("/tags/"),
title = a.selectFirstOrThrow("span").text(),
source = source,
)
},
author = doc.requireElementById("metadata").selectFirstOrThrow("main div:contains(Artist) a span").text(),
chapters = listOf(
MangaChapter(
id = generateUid(manga.id),
name = manga.title,
number = 1,
url = manga.url,
scanlator = null,
uploadDate = date.tryParse(doc.selectFirstOrThrow("time.updated").text()),
branch = null,
source = source,
),
),
)
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url
.replace("/view/", "/read/")
.let { "$it/1" }
.toAbsoluteUrl(domain)
val document = webClient.httpGet(fullUrl).parseHtml()
val id = fullUrl
.substringAfter("/read/")
.substringBeforeLast("/")
val cdnUrl = document.selectFirst("meta[itemprop=image]")
?.attr("content")
?.toHttpUrlOrNull()
?.host
.let { "https://" + (it ?: domain) }
val script = document.select("script:containsData(window.metadata)").html()
val rawJson = script
.substringAfter("original:")
.substringBefore("resampled:")
.substringBeforeLast(",")
return JSONArray(rawJson).mapJSON {
val fileName = it.getString("n")
val url = "$cdnUrl/original/$id/$fileName"
val preview = "$cdnUrl/t/$id/320/$fileName"
MangaPage(
id = generateUid(url),
url = url,
preview = preview,
source = source,
)
}
}
}

View File

@@ -1,148 +0,0 @@
package org.koitharu.kotatsu.parsers.site.en
import okhttp3.Headers
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.network.UserAgents
import org.koitharu.kotatsu.parsers.util.*
import java.text.SimpleDateFormat
import java.util.*
@MangaSourceParser("MANGAGEKO", "MangaGeko", "en")
internal class MangaGeko(context: MangaLoaderContext) : PagedMangaParser(context, MangaSource.MANGAGEKO, 30) {
override val availableSortOrders: Set<SortOrder> =
EnumSet.of(SortOrder.POPULARITY, SortOrder.UPDATED, SortOrder.NEWEST)
override val configKeyDomain = ConfigKey.Domain("www.mangageko.com")
override val isMultipleTagsSupported = false
override val headers: Headers = Headers.Builder()
.add("User-Agent", UserAgents.CHROME_DESKTOP)
.build()
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val tag = tags.oneOrThrowIfMany()
val url = if (!query.isNullOrEmpty()) {
if (page > 1) {
return emptyList()
}
buildString {
append("https://$domain/search/?search=")
append(query.urlEncoded())
}
} else {
buildString {
append("https://$domain/browse-comics/?results=")
append(page)
append("&filter=")
when (sortOrder) {
SortOrder.POPULARITY -> append("views")
SortOrder.UPDATED -> append("Updated")
SortOrder.NEWEST -> append("New")
else -> append("Updated")
}
if (!tags.isNullOrEmpty()) {
append("&genre=")
append(tag?.key.orEmpty())
}
}
}
val doc = webClient.httpGet(url).parseHtml()
return doc.select("li.novel-item").map { div ->
val href = div.selectFirstOrThrow("a").attrAsRelativeUrl("href")
Manga(
id = generateUid(href),
title = div.selectFirstOrThrow("h4").text(),
altTitle = null,
url = href,
publicUrl = href.toAbsoluteUrl(domain),
rating = RATING_UNKNOWN,
isNsfw = false,
coverUrl = div.selectFirstOrThrow("img").src().orEmpty(),
tags = emptySet(),
state = null,
author = div.selectFirstOrThrow("h6").text().removePrefix("Author(S): "),
source = source,
)
}
}
override suspend fun getAvailableTags(): Set<MangaTag> {
val doc = webClient.httpGet("https://$domain/browse-comics/").parseHtml()
return doc.select("label.checkbox-inline").mapNotNullToSet { label ->
MangaTag(
key = label.selectFirstOrThrow("input").attr("value"),
title = label.text(),
source = source,
)
}
}
override suspend fun getDetails(manga: Manga): Manga {
val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml()
val dateFormat = SimpleDateFormat("MMM dd, yyyy", sourceLocale)
return manga.copy(
altTitle = doc.selectFirstOrThrow(".alternative-title").text(),
state = when (doc.selectFirstOrThrow(".header-stats span:contains(Status) strong").text()) {
"Ongoing" -> MangaState.ONGOING
"Completed" -> MangaState.FINISHED
else -> null
},
tags = doc.select(".categories ul li a").mapNotNullToSet { a ->
MangaTag(
key = a.attr("href").substringAfterLast("="),
title = a.text(),
source = source,
)
},
author = doc.selectFirstOrThrow(".author").text(),
description = doc.selectFirstOrThrow(".description").html(),
chapters = doc.requireElementById("chapters").select("ul.chapter-list li")
.mapChapters(reversed = true) { i, li ->
val a = li.selectFirstOrThrow("a")
val url = a.attrAsRelativeUrl("href")
val name = li.selectFirstOrThrow(".chapter-title").text()
val dateText = li.select(".chapter-update").attr("datetime").substringBeforeLast(',')
.replace(".", "").replace("Sept", "Sep")
MangaChapter(
id = generateUid(url),
name = name,
number = i + 1,
url = url,
scanlator = null,
uploadDate = dateFormat.tryParse(dateText),
branch = null,
source = source,
)
},
)
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(domain)
val doc = webClient.httpGet(fullUrl).parseHtml()
return doc.requireElementById("chapter-reader").select("img").map { img ->
val url = img.src()?.toRelativeUrl(domain) ?: img.parseFailed("Image src not found")
MangaPage(
id = generateUid(url),
url = url,
preview = null,
source = source,
)
}
}
}

View File

@@ -1,193 +0,0 @@
package org.koitharu.kotatsu.parsers.site.en
import kotlinx.coroutines.coroutineScope
import okhttp3.Headers
import org.jsoup.nodes.Document
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.network.UserAgents
import org.koitharu.kotatsu.parsers.util.*
import org.koitharu.kotatsu.parsers.util.json.mapJSON
import java.text.SimpleDateFormat
import java.util.*
@MangaSourceParser("MANGAOWL", "MangaOwl.to", "en")
internal class Mangaowl(context: MangaLoaderContext) :
PagedMangaParser(context, MangaSource.MANGAOWL, pageSize = 24) {
override val availableSortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.POPULARITY,
SortOrder.NEWEST,
SortOrder.UPDATED,
SortOrder.RATING,
)
override val configKeyDomain = ConfigKey.Domain("mangaowl.to")
override val headers: Headers = Headers.Builder()
.add("User-Agent", UserAgents.CHROME_DESKTOP)
.build()
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val sort = when (sortOrder) {
SortOrder.POPULARITY -> "view_count"
SortOrder.UPDATED -> "-modified_at"
SortOrder.NEWEST -> "created_at"
SortOrder.RATING -> "rating"
else -> "modified_at"
}
val url = buildString {
append("https://")
append(domain)
when {
!query.isNullOrEmpty() -> {
append("/8-search")
append("?q=")
append(query.urlEncoded())
append("&page=")
append(page.toString())
}
!tags.isNullOrEmpty() -> {
append("/8-genres/")
for (tag in tags) {
append(tag.key)
}
append("?page=")
append(page.toString())
}
else -> {
append("/8-comics")
append("?page=")
append(page.toString())
append("&ordering=")
append(sort)
}
}
}
val doc = webClient.httpGet(url).parseHtml()
return doc.select("div.manga-item.column").map { div ->
val href = div.selectFirstOrThrow("a").attrAsRelativeUrl("href")
Manga(
id = generateUid(href),
url = href,
publicUrl = href.toAbsoluteUrl(div.host ?: domain),
coverUrl = div.selectFirst("img")?.src().orEmpty(),
title = div.selectFirst("a.one-line")?.text().orEmpty(),
altTitle = null,
rating = div.select("span").last()?.text()?.toFloatOrNull()?.div(5f) ?: RATING_UNKNOWN,
tags = emptySet(),
author = null,
state = null,
source = source,
isNsfw = false,
)
}
}
override suspend fun getAvailableTags(): Set<MangaTag> {
val doc = webClient.httpGet("https://$domain/8-genres").parseHtml()
return doc.select("div.genres-container span.genre-item a").mapNotNullToSet { a ->
val key = a.attr("href").substringAfterLast("/")
MangaTag(
key = key,
title = a.text(),
source = source,
)
}
}
override suspend fun getDetails(manga: Manga): Manga = coroutineScope {
val fullUrl = manga.url.toAbsoluteUrl(domain)
val doc = webClient.httpGet(fullUrl).parseHtml()
manga.copy(
tags = doc.body().select("div.comic-attrs div.column.my-2:contains(Genres) a").mapNotNullToSet { a ->
MangaTag(
key = a.attr("href").removeSuffix("/").substringAfterLast('/'),
title = a.text().toTitleCase().replace(",", ""),
source = source,
)
},
description = doc.select("span.story-desc").html(),
state = when (doc.select("div.section-status:contains(Status) span").last()?.text()) {
"Ongoing" -> MangaState.ONGOING
"Completed" -> MangaState.FINISHED
else -> null
},
chapters = getChapters(manga.url, doc),
)
}
private fun getChapters(mangaUrl: String, doc: Document): List<MangaChapter> {
val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'", sourceLocale)
val script = doc.selectFirstOrThrow("script:containsData(chapters:)")
val json =
script.data().substringAfter("chapters:[").substringBeforeLast(')').substringBefore("],latest_chapter:")
.split("},")
val slug = mangaUrl.substringAfterLast("/")
val chapter = ArrayList<MangaChapter>()
var lastIndexed = 0
json.mapIndexed { i, t ->
if (t.contains("Chapter")) {
val id = t.substringAfter("id:").substringBefore(",created_at")
val url = "/reading/$slug/$id"
val date = t.substringAfter("created_at:\"").substringBefore("\"")
val name = t.substringAfter("name:\"").substringBefore("\"")
chapter.add(
MangaChapter(
id = generateUid(url),
name = name,
number = i + 1,
url = url,
uploadDate = dateFormat.tryParse(date),
source = source,
scanlator = null,
branch = null,
),
)
lastIndexed = i
}
}
// last chapter
val id = script.data().substringAfter("Sign in\",").substringBefore(",\"").split(",").last()
val url = "/reading/$slug/$id"
val date = script.data().substringAfter("$id,\"").substringBefore("\",")
val name = script.data().substringAfter("$date\",\"").substringBefore("\",")
chapter.add(
MangaChapter(
id = generateUid(url),
name = name,
number = lastIndexed + 1,
url = url,
uploadDate = dateFormat.tryParse(date),
source = source,
scanlator = null,
branch = null,
),
)
return chapter
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val id = chapter.url.substringAfterLast("/")
val json = webClient.httpGet("https://api.mangaowl.to/v1/chapters/$id/images?page_size=100").parseJson()
return json.getJSONArray("results").mapJSON { jo ->
MangaPage(
id = generateUid(jo.getString("image")),
preview = null,
source = chapter.source,
url = jo.getString("image"),
)
}
}
}

View File

@@ -1,194 +0,0 @@
package org.koitharu.kotatsu.parsers.site.en
import androidx.collection.ArrayMap
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import java.util.*
@MangaSourceParser("MANHWA18", "Manhwa18", "en", type = ContentType.HENTAI)
class Manhwa18Parser(context: MangaLoaderContext) :
PagedMangaParser(context, MangaSource.MANHWA18, pageSize = 18, searchPageSize = 18) {
override val configKeyDomain: ConfigKey.Domain = ConfigKey.Domain("manhwa18.net")
override val availableSortOrders: Set<SortOrder>
get() = EnumSet.of(SortOrder.UPDATED, SortOrder.POPULARITY, SortOrder.ALPHABETICAL, SortOrder.NEWEST)
private val tagsMap = SuspendLazy(::parseTags)
override suspend fun getFavicons(): Favicons {
return Favicons(
listOf(
Favicon("https://$domain/uploads/logos/logo-mini.png", 92, null),
),
domain,
)
}
override suspend fun getDetails(manga: Manga): Manga {
val docs = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml()
val cardInfoElement = docs.selectFirst("div.series-information")
val author = cardInfoElement?.selectFirst(".info-name:contains(Author)")?.parent()
?.select("a")
?.joinToString(", ") { it.text() }
val availableTags = tagsMap.get()
val tags = cardInfoElement?.selectFirst(".info-name:contains(Genre)")?.parent()
?.select("a")
?.mapNotNullToSet { availableTags[it.text().lowercase(Locale.ENGLISH)] }
val state = cardInfoElement?.selectFirst(".info-name:contains(Status)")?.parent()
?.selectFirst("a")
?.let {
when (it.text().lowercase()) {
"on going" -> MangaState.ONGOING
"completed" -> MangaState.FINISHED
else -> null
}
}
return manga.copy(
altTitle = cardInfoElement?.selectFirst("b:contains(Other names)")?.parent()?.ownText()?.removePrefix(": "),
author = author,
description = docs.selectFirst(".series-summary .summary-content")?.html(),
tags = tags.orEmpty(),
state = state,
chapters = docs.select(".card-body > .list-chapters > a").mapChapters(reversed = true) { index, element ->
val chapterUrl = element.attrAsAbsoluteUrlOrNull("href")?.toRelativeUrl(domain)
?: return@mapChapters null
val uploadDate = parseUploadDate(element.selectFirst(".chapter-time")?.text())
MangaChapter(
id = generateUid(chapterUrl),
name = element.selectFirst(".chapter-name")?.text().orEmpty(),
number = index + 1,
url = chapterUrl,
scanlator = null,
uploadDate = uploadDate,
branch = null,
source = MangaSource.MANHWA18,
)
},
)
}
// 7 minutes ago
// 5 hours ago
// 2 days ago
// 2 weeks ago
// 4 years ago
private fun parseUploadDate(timeStr: String?): Long {
timeStr ?: return 0
val timeWords = timeStr.split(' ')
if (timeWords.size != 3) return 0
val timeWord = timeWords[1]
val timeAmount = timeWords[0].toIntOrNull() ?: return 0
val timeUnit = when (timeWord) {
"minute", "minutes" -> Calendar.MINUTE
"hour", "hours" -> Calendar.HOUR
"day", "days" -> Calendar.DAY_OF_YEAR
"week", "weeks" -> Calendar.WEEK_OF_YEAR
"month", "months" -> Calendar.MONTH
"year", "years" -> Calendar.YEAR
else -> return 0
}
val cal = Calendar.getInstance()
cal.add(timeUnit, -timeAmount)
return cal.time.time
}
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val sortQuery = when (sortOrder) {
SortOrder.ALPHABETICAL -> "az"
SortOrder.POPULARITY -> "top"
SortOrder.UPDATED -> "update"
SortOrder.NEWEST -> "new"
else -> ""
}
val tagQuery = tags?.joinToString(",") { it.key }.orEmpty()
val url = buildString {
append("https://")
append(domain)
append("/tim-kiem?page=")
append(page)
if (!query.isNullOrEmpty()) {
append("&q=")
append(query.urlEncoded())
}
append("&accept_genres=$tagQuery")
append("&sort=")
append(sortQuery)
}
val docs = webClient.httpGet(url).parseHtml()
return docs.select(".card-body .thumb-item-flow")
.map {
val titleElement = it.selectFirstOrThrow(".thumb_attr.series-title > a")
val absUrl = titleElement.attrAsAbsoluteUrl("href")
Manga(
id = generateUid(absUrl.toRelativeUrl(domain)),
title = titleElement.text(),
altTitle = null,
url = absUrl.toRelativeUrl(domain),
publicUrl = absUrl,
rating = RATING_UNKNOWN,
isNsfw = true,
coverUrl = it.selectFirst("div.img-in-ratio")?.attrAsAbsoluteUrl("data-bg").orEmpty(),
tags = emptySet(),
state = null,
author = null,
largeCoverUrl = null,
description = null,
source = MangaSource.MANHWA18,
)
}
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val chapterUrl = chapter.url.toAbsoluteUrl(domain)
val doc = webClient.httpGet(chapterUrl).parseHtml()
return doc.requireElementById("chapter-content").select("img").mapNotNull {
val url = it.attrAsRelativeUrlOrNull("data-src")
?: it.attrAsRelativeUrlOrNull("src")
?: return@mapNotNull null
MangaPage(
id = generateUid(url),
url = url,
preview = null,
source = MangaSource.MANHWA18,
)
}
}
override suspend fun getAvailableTags(): Set<MangaTag> {
return tagsMap.get().values.toSet()
}
private suspend fun parseTags(): Map<String, MangaTag> {
val doc = webClient.httpGet("https://$domain/tim-kiem?q=").parseHtml()
val list = doc.getElementsByAttribute("data-genre-id")
if (list.isEmpty()) {
return emptyMap()
}
val result = ArrayMap<String, MangaTag>(list.size)
for (item in list) {
val id = item.attr("data-genre-id")
val name = item.text()
result[name.lowercase(Locale.ENGLISH)] = MangaTag(
title = name.toTitleCase(Locale.ENGLISH),
key = id,
source = source,
)
}
return result
}
}

View File

@@ -1,151 +0,0 @@
package org.koitharu.kotatsu.parsers.site.en
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import java.text.DateFormat
import java.text.SimpleDateFormat
import java.util.*
@MangaSourceParser("MANHWASMEN", "ManhwasMen", "en", type = ContentType.HENTAI)
class ManhwasMen(context: MangaLoaderContext) :
PagedMangaParser(context, MangaSource.MANHWASMEN, pageSize = 30, searchPageSize = 30) {
override val configKeyDomain: ConfigKey.Domain = ConfigKey.Domain("manhwas.men")
override val isMultipleTagsSupported = false
override val availableSortOrders: Set<SortOrder>
get() = EnumSet.of(SortOrder.POPULARITY)
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val tag = tags.oneOrThrowIfMany()
val url = buildString {
append("https://")
append(domain)
append("/manga-list")
append("?page=")
append(page)
when {
!query.isNullOrEmpty() -> {
append("&search=")
append(query.urlEncoded())
}
!tags.isNullOrEmpty() -> {
append("&genero=")
append(tag?.key.orEmpty())
}
}
}
val doc = webClient.httpGet(url).parseHtml()
return doc.select("ul.animes li").map { li ->
val href = li.selectFirstOrThrow("a").attrAsRelativeUrl("href")
Manga(
id = generateUid(href),
url = href,
publicUrl = href.toAbsoluteUrl(domain),
coverUrl = li.selectFirst("img")?.src().orEmpty(),
title = li.selectFirst(".title")?.text().orEmpty(),
altTitle = null,
rating = RATING_UNKNOWN,
tags = emptySet(),
author = null,
state = null,
source = source,
isNsfw = isNsfwSource,
)
}
}
override suspend fun getAvailableTags(): Set<MangaTag> {
val tags = webClient.httpGet("https://$domain/manga-list").parseHtml()
.selectLastOrThrow(".filter-bx .form-group select.custom-select").select("option").drop(1)
return tags.mapNotNullToSet { option ->
MangaTag(
key = option.attr("value").substringAfterLast("="),
title = option.text(),
source = source,
)
}
}
override suspend fun getDetails(manga: Manga): Manga {
val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml()
return manga.copy(
tags = doc.body().select(".genres a").mapNotNullToSet { a ->
MangaTag(
key = a.attr("href").substringAfterLast('='),
title = a.text(),
source = source,
)
},
description = doc.select(".sinopsis").html(),
state = when (doc.selectLast(".anime-type-peli")?.text()?.lowercase()) {
"ongoing" -> MangaState.ONGOING
"completed" -> MangaState.FINISHED
else -> null
},
chapters = doc.select(".episodes-list li").mapChapters(reversed = true) { i, li ->
val url = li.selectFirstOrThrow("a").attrAsRelativeUrl("href")
MangaChapter(
id = generateUid(url),
name = li.selectFirstOrThrow(".flex-grow-1 span").text(),
number = i + 1,
url = url,
scanlator = null,
uploadDate = parseChapterDate(
SimpleDateFormat("dd/MM/yyyy", sourceLocale),
li.selectLastOrThrow(".flex-grow-1 span").text(),
),
branch = null,
source = source,
)
},
)
}
private fun parseChapterDate(dateFormat: DateFormat, date: String?): Long {
val d = date?.lowercase() ?: return 0
return when {
d.endsWith(" ago") -> parseRelativeDate(date)
else -> dateFormat.tryParse(date)
}
}
private fun parseRelativeDate(date: String): Long {
val number = Regex("""(\d+)""").find(date)?.value?.toIntOrNull() ?: return 0
val cal = Calendar.getInstance()
return when {
WordSet("second").anyWordIn(date) -> cal.apply { add(Calendar.SECOND, -number) }.timeInMillis
WordSet("minute", "minutes").anyWordIn(date) -> cal.apply { add(Calendar.MINUTE, -number) }.timeInMillis
WordSet("hour", "hours").anyWordIn(date) -> cal.apply { add(Calendar.HOUR, -number) }.timeInMillis
WordSet("day", "days").anyWordIn(date) -> cal.apply { add(Calendar.DAY_OF_MONTH, -number) }.timeInMillis
WordSet("week", "weeks").anyWordIn(date) -> cal.apply { add(Calendar.WEEK_OF_YEAR, -number) }.timeInMillis
WordSet("month", "months").anyWordIn(date) -> cal.apply { add(Calendar.MONTH, -number) }.timeInMillis
WordSet("year").anyWordIn(date) -> cal.apply { add(Calendar.YEAR, -number) }.timeInMillis
else -> 0
}
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml().requireElementById("chapter_imgs")
return doc.select("img").map { img ->
val url = img.src()?.toRelativeUrl(domain) ?: img.parseFailed("Image src not found")
MangaPage(
id = generateUid(url),
url = url,
preview = null,
source = source,
)
}
}
}

View File

@@ -1,96 +0,0 @@
package org.koitharu.kotatsu.parsers.site.en
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import java.text.SimpleDateFormat
import java.util.*
@MangaSourceParser("PO2SCANS", "Po2Scans", "en")
internal class Po2Scans(context: MangaLoaderContext) : MangaParser(context, MangaSource.PO2SCANS) {
override val availableSortOrders: Set<SortOrder> = EnumSet.of(SortOrder.ALPHABETICAL)
override val configKeyDomain = ConfigKey.Domain("po2scans.com")
override suspend fun getList(offset: Int, query: String?, tags: Set<MangaTag>?, sortOrder: SortOrder): List<Manga> {
if (offset > 0) {
return emptyList()
}
val url = buildString {
append("https://$domain/series")
if (!query.isNullOrEmpty()) {
append("?search=")
append(query.urlEncoded())
}
}
val doc = webClient.httpGet(url).parseHtml()
return doc.select(".series-list").map { div ->
val href = div.selectFirstOrThrow("a").attrAsRelativeUrl("href")
Manga(
id = generateUid(href),
title = div.selectFirstOrThrow("h2").text(),
altTitle = null,
url = href,
publicUrl = href.toAbsoluteUrl(domain),
rating = RATING_UNKNOWN,
isNsfw = false,
coverUrl = div.selectFirstOrThrow("img").src()?.toAbsoluteUrl(domain).orEmpty(),
tags = emptySet(),
state = null,
author = null,
source = source,
)
}
}
override suspend fun getAvailableTags(): Set<MangaTag> = emptySet()
override suspend fun getDetails(manga: Manga): Manga {
val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml()
val dateFormat = SimpleDateFormat("dd MMM, yy", Locale.ENGLISH)
return manga.copy(
altTitle = null,
state = when (doc.select(".status span").last()?.text()) {
"Ongoing" -> MangaState.ONGOING
"Done" -> MangaState.FINISHED
else -> null
},
tags = emptySet(),
author = doc.select(".author span").last()?.text(),
description = doc.selectFirstOrThrow(".summary").text(),
chapters = doc.select(".chap-section .chap")
.mapChapters(reversed = true) { i, div ->
val a = div.selectFirstOrThrow("a")
val url = "/" + a.attrAsRelativeUrl("href").toAbsoluteUrl(domain)
MangaChapter(
id = generateUid(url),
name = a.text(),
number = i + 1,
url = url,
scanlator = null,
uploadDate = dateFormat.tryParse(div.select(".detail span").last()?.text()),
branch = null,
source = source,
)
},
)
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(domain)
val doc = webClient.httpGet(fullUrl).parseHtml()
return doc.select(".swiper-slide img").map { img ->
val url = img.src()?.replace("./assets", "/assets")?.toRelativeUrl(domain)
?: img.parseFailed("Image src not found")
MangaPage(
id = generateUid(url),
url = url,
preview = null,
source = source,
)
}
}
}

View File

@@ -1,170 +0,0 @@
package org.koitharu.kotatsu.parsers.site.en
import androidx.collection.ArraySet
import kotlinx.coroutines.async
import kotlinx.coroutines.awaitAll
import kotlinx.coroutines.coroutineScope
import org.jsoup.nodes.Element
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import java.util.*
@MangaSourceParser("PURURIN", "Pururin", "en", ContentType.HENTAI)
internal class Pururin(context: MangaLoaderContext) :
PagedMangaParser(context, MangaSource.PURURIN, pageSize = 20) {
override val availableSortOrders: Set<SortOrder> =
EnumSet.of(SortOrder.UPDATED, SortOrder.POPULARITY, SortOrder.RATING, SortOrder.ALPHABETICAL)
override val configKeyDomain = ConfigKey.Domain("pururin.to")
override val isMultipleTagsSupported = false
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val tag = tags.oneOrThrowIfMany()
val url = buildString {
append("https://")
append(domain)
if (!query.isNullOrEmpty()) {
append("/search?q=")
append(query.urlEncoded())
append("&page=")
append(page)
} else {
append("/browse")
if (!tags.isNullOrEmpty()) {
append("/tags/content/")
append(tag?.key.orEmpty())
append("/")
}
append("?page=")
append(page)
append("&sort=")
when (sortOrder) {
SortOrder.UPDATED -> append("")
SortOrder.POPULARITY -> append("most-viewed")
SortOrder.RATING -> append("highest-rated")
SortOrder.ALPHABETICAL -> append("title")
else -> append("")
}
}
}
val doc = webClient.httpGet(url).parseHtml()
return doc.select(".row-gallery a.card-gallery").map { a ->
val href = a.attrAsRelativeUrl("href")
Manga(
id = generateUid(href),
url = href,
publicUrl = href.toAbsoluteUrl(domain),
coverUrl = a.selectFirst("img.card-img-top")?.src().orEmpty(),
title = a.selectFirst(".title")?.text().orEmpty(),
altTitle = null,
rating = RATING_UNKNOWN,
tags = emptySet(),
author = null,
state = null,
source = source,
isNsfw = isNsfwSource,
)
}
}
override suspend fun getAvailableTags(): Set<MangaTag> {
return coroutineScope {
(1..4).map { page ->
async { getTags(page) }
}
}.awaitAll().flattenTo(ArraySet(360))
}
private suspend fun getTags(page: Int): Set<MangaTag> {
val url = "https://$domain/tags/content?order=uses&page=$page"
val root = webClient.httpGet(url).parseHtml()
return root.parseTags()
}
private fun Element.parseTags() = select("table tr td a").mapToSet {
val href = it.attr("href").substringAfterLast("content/").substringBeforeLast('/')
MangaTag(
key = href,
title = it.text(),
source = source,
)
}
override suspend fun getDetails(manga: Manga): Manga = coroutineScope {
val fullUrl = manga.url.toAbsoluteUrl(domain)
val doc = webClient.httpGet(fullUrl).parseHtml()
manga.copy(
description = doc.selectFirst("p.mb-2")?.text().orEmpty(),
rating = doc.selectFirst("td span.rating")?.attr("content")?.toFloatOrNull()?.div(5f) ?: RATING_UNKNOWN,
tags = doc.body().select("tr:contains(Contents) ul.list-inline a").mapNotNullToSet {
val href = it.attr("href").substringAfterLast("content/").substringBeforeLast('/')
MangaTag(
key = href,
title = it.text(),
source = source,
)
},
author = doc.selectFirst("a[itemprop=author]")?.text(),
chapters = listOf(
MangaChapter(
id = manga.id,
name = manga.title,
number = 1,
url = manga.url,
scanlator = null,
uploadDate = 0,
branch = null,
source = source,
),
),
)
}
override suspend fun getRelatedManga(seed: Manga): List<Manga> {
val doc = webClient.httpGet(seed.url.toAbsoluteUrl(domain)).parseHtml()
val root = doc.body().selectFirstOrThrow(".row-gallery-small")
return root.select("a.card-gallery").mapNotNull { a ->
val href = a.attrAsRelativeUrl("href")
Manga(
id = generateUid(href),
url = href,
publicUrl = href.toAbsoluteUrl(domain),
coverUrl = a.selectFirst("img.card-img-top")?.src().orEmpty(),
title = a.selectFirst(".title")?.text().orEmpty(),
altTitle = null,
rating = RATING_UNKNOWN,
tags = emptySet(),
author = null,
state = null,
source = source,
isNsfw = false,
)
}
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(domain)
val doc = webClient.httpGet(fullUrl).parseHtml()
return doc.select(".gallery-preview img").map { url ->
val img = url.src()?.toRelativeUrl(domain) ?: url.parseFailed("Image src not found")
val urlImage = img.replace("t.", ".")
MangaPage(
id = generateUid(urlImage),
url = urlImage,
preview = null,
source = source,
)
}
}
}

View File

@@ -1,132 +0,0 @@
package org.koitharu.kotatsu.parsers.site.es
import kotlinx.coroutines.coroutineScope
import okhttp3.Headers
import org.jsoup.nodes.Document
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.network.UserAgents
import org.koitharu.kotatsu.parsers.util.*
import java.util.*
@MangaSourceParser("TEMPLESCANESP", "TempleScanEsp", "es", ContentType.HENTAI)
internal class TempleScanEsp(context: MangaLoaderContext) :
PagedMangaParser(context, MangaSource.TEMPLESCANESP, pageSize = 15) {
override val availableSortOrders: Set<SortOrder> = EnumSet.of(SortOrder.NEWEST, SortOrder.UPDATED)
override val configKeyDomain = ConfigKey.Domain("templescanesp.net")
override val headers: Headers = Headers.Builder()
.add("User-Agent", UserAgents.CHROME_DESKTOP)
.build()
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val url = buildString {
append("https://")
append(domain)
if (sortOrder == SortOrder.NEWEST) {
append("/comics")
append("?page=")
append(page.toString())
} else {
if (page > 1) {
return emptyList()
}
}
}
val doc = webClient.httpGet(url).parseHtml()
return doc.select("div.grid figure").ifEmpty {
doc.requireElementById("projectsDiv").select("figure")
}.map { div ->
val href = div.selectFirstOrThrow("a").attrAsRelativeUrl("href")
Manga(
id = generateUid(href),
url = href,
publicUrl = href.toAbsoluteUrl(div.host ?: domain),
coverUrl = div.selectFirst("img")?.src().orEmpty(),
title = div.selectFirst("figcaption")?.text().orEmpty(),
altTitle = null,
rating = RATING_UNKNOWN,
tags = emptySet(),
author = null,
state = null,
source = source,
isNsfw = isNsfwSource,
)
}
}
override suspend fun getAvailableTags(): Set<MangaTag> = emptySet()
override suspend fun getDetails(manga: Manga): Manga = coroutineScope {
val fullUrl = manga.url.toAbsoluteUrl(domain)
val doc = webClient.httpGet(fullUrl).parseHtml()
val chaptersDeferred = getChapters(doc)
manga.copy(
description = doc.requireElementById("section-sinopsis").html(),
chapters = chaptersDeferred,
)
}
private fun getChapters(doc: Document): List<MangaChapter> {
return doc.body().select("div.grid-capitulos div.contenedor-capitulo-miniatura")
.mapChapters(reversed = true) { i, div ->
val href = div.selectFirstOrThrow("a").attrAsRelativeUrl("href")
val date = parseUploadDate(div.selectFirstOrThrow("time").text())
MangaChapter(
id = generateUid(href),
name = div.requireElementById("name").text(),
number = i + 1,
url = href,
uploadDate = date,
source = source,
scanlator = null,
branch = null,
)
}
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(domain)
val doc = webClient.httpGet(fullUrl).parseHtml()
return doc.select("main.contenedor-imagen img").map { url ->
val img = url.src()?.toRelativeUrl(domain) ?: url.parseFailed("Image src not found")
MangaPage(
id = generateUid(img),
url = img,
preview = null,
source = source,
)
}
}
private fun parseUploadDate(timeStr: String?): Long {
timeStr ?: return 0
val timeWords = timeStr.split(' ')
if (timeWords.size != 3) return 0
val timeWord = timeWords[1]
val timeAmount = timeWords[0].toIntOrNull() ?: return 0
val timeUnit = when (timeWord) {
"minute", "minutes" -> Calendar.MINUTE
"hour", "hours" -> Calendar.HOUR
"day", "days" -> Calendar.DAY_OF_YEAR
"week", "weeks" -> Calendar.WEEK_OF_YEAR
"month", "months" -> Calendar.MONTH
"year", "years" -> Calendar.YEAR
else -> return 0
}
val cal = Calendar.getInstance()
cal.add(timeUnit, -timeAmount)
return cal.time.time
}
}

View File

@@ -1,227 +0,0 @@
package org.koitharu.kotatsu.parsers.site.es
import okhttp3.Headers
import okhttp3.HttpUrl.Companion.toHttpUrl
import org.jsoup.nodes.Document
import org.jsoup.nodes.Element
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import java.text.SimpleDateFormat
import java.util.*
@MangaSourceParser("TUMANGAONLINE", "TuMangaOnline", "es")
class TuMangaOnlineParser(context: MangaLoaderContext) : PagedMangaParser(
context,
source = MangaSource.TUMANGAONLINE,
pageSize = 24,
) {
override val configKeyDomain = ConfigKey.Domain("visortmo.com")
private val chapterDateFormat = SimpleDateFormat("yyyy-MM-dd", sourceLocale)
override val availableSortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.ALPHABETICAL,
SortOrder.UPDATED,
SortOrder.NEWEST,
SortOrder.POPULARITY,
SortOrder.RATING,
)
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val order =
when (sortOrder) {
SortOrder.POPULARITY -> "likes_count"
SortOrder.UPDATED -> "release_date"
SortOrder.NEWEST -> "creation"
SortOrder.ALPHABETICAL -> "alphabetically"
SortOrder.RATING -> "score"
}
val url = buildString {
append("/library")
if (query.isNullOrEmpty()) {
append("?order_item=")
append(order)
append("&order_dir=desc")
append("&filter_by=title")
if (tags != null) {
for (tag in tags) {
append("&genders[]=${tag.key}")
}
}
} else {
append("?title=$query")
}
append("&_pg=1")
append("&page=$page")
}.toAbsoluteUrl(domain)
val doc = webClient.httpGet(url, headers).parseHtml()
val items = doc.body().select("div.element")
return items.mapNotNull { item ->
val href =
item.selectFirst("a")?.attrAsRelativeUrlOrNull("href")?.substringAfter(' ') ?: return@mapNotNull null
Manga(
id = generateUid(href),
title = item.selectFirst("h4.text-truncate")?.text() ?: return@mapNotNull null,
coverUrl = item.select("style").toString().substringAfter("('").substringBeforeLast("')"),
altTitle = null,
author = null,
rating = item.selectFirst("span.score")?.text()?.toFloatOrNull()?.div(10F) ?: RATING_UNKNOWN,
url = href,
isNsfw = item.select("i").hasClass("fas fa-heartbeat fa-2x"),
tags = emptySet(),
state = null,
publicUrl = href.toAbsoluteUrl(doc.host ?: domain),
source = source,
)
}
}
override suspend fun getDetails(manga: Manga): Manga {
val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml()
val contents = doc.body().selectFirstOrThrow("section.element-header-content")
return manga.copy(
description = contents.selectFirst("p.element-description")?.html(),
largeCoverUrl = contents.selectFirst(".book-thumbnail")?.attrAsAbsoluteUrlOrNull("src"),
state = parseStatus(contents.select("span.book-status").text().orEmpty()),
author = contents.selectFirst("h5.card-title")?.attr("title")?.substringAfter(", "),
chapters = if (doc.select("div.chapters").isEmpty()) {
doc.select(oneShotChapterListSelector).mapChapters(reversed = true) { _, item ->
oneShotChapterFromElement(item)
}
} else {
val chapters = ChaptersListBuilder(10)
doc.select(regularChapterListSelector).reversed().forEachIndexed { i, item ->
val chaptername = item.select("div.col-10.text-truncate").text().replace("&nbsp;", " ").trim()
val scanelement = item.select("ul.chapter-list > li")
scanelement.forEach { chapters.add(regularChapterFromElement(it, chaptername, i)) }
}
chapters.toList()
},
)
}
private val oneShotChapterListSelector = "div.chapter-list-element > ul.list-group li.list-group-item"
private fun oneShotChapterFromElement(element: Element): MangaChapter {
val href = element.selectFirstOrThrow("div.row > .text-right > a").attrAsRelativeUrl("href")
return MangaChapter(
id = generateUid(href),
name = "One Shot",
number = 1,
url = href,
scanlator = element.select("div.col-md-6.text-truncate").text(),
branch = null,
uploadDate = chapterDateFormat.tryParse(element.select("span.badge.badge-primary.p-2").first()?.text()),
source = source,
)
}
private val regularChapterListSelector = "div.chapters > ul.list-group li.p-0.list-group-item"
private fun regularChapterFromElement(element: Element, chName: String, number: Int): MangaChapter {
val href = element.selectFirstOrThrow("div.row > .text-right > a").attrAsRelativeUrl("href")
return MangaChapter(
id = generateUid(href),
name = chName,
number = number + 1,
url = href,
scanlator = element.select("div.col-md-6.text-truncate").text(),
branch = null,
uploadDate = chapterDateFormat.tryParse(element.select("span.badge.badge-primary.p-2").first()?.text()),
source = source,
)
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val redirectDoc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain), headers).parseHtml()
var doc = redirectToReadingPage(redirectDoc)
val currentUrl = doc.location()
val newUrl = if (!currentUrl.contains("cascade")) {
currentUrl.substringBefore("paginated") + "cascade"
} else {
currentUrl
}
if (currentUrl != newUrl) {
doc = webClient.httpGet(newUrl, headers).parseHtml()
}
return doc.select("div.viewer-container img:not(noscript img)").map {
val href = if (it.hasAttr("data-src")) {
it.attr("abs:data-src")
} else {
it.attr("abs:src")
}
MangaPage(
id = generateUid(href),
url = href,
preview = null,
source = source,
)
}
}
private suspend fun redirectToReadingPage(document: Document): Document {
val script1 = document.selectFirst("script:containsData(uniqid)")
val script2 = document.selectFirst("script:containsData(window.location.replace)")
val redirectHeaders = Headers.Builder().set("Referer", document.baseUri()).build()
if (script1 != null) {
val data = script1.data()
val regexParams = """\{uniqid:'(.+)',cascade:(.+)\}""".toRegex()
val regexAction = """form\.action\s?=\s?'(.+)'""".toRegex()
val params = regexParams.find(data)!!
val action = regexAction.find(data)!!.groupValues[1].toHttpUrl()
val formBody = mapOf(
"uniqid" to params.groupValues[1],
"cascade" to params.groupValues[2],
)
return redirectToReadingPage(webClient.httpPost(action, formBody, redirectHeaders).parseHtml())
}
if (script2 != null) {
val data = script2.data()
val regexRedirect = """window\.location\.replace\('(.+)'\)""".toRegex()
val url = regexRedirect.find(data)!!.groupValues[1]
return redirectToReadingPage(webClient.httpGet(url, redirectHeaders).parseHtml())
}
return document
}
override suspend fun getAvailableTags(): Set<MangaTag> {
val doc = webClient.httpGet("https://$domain/library", headers).parseHtml()
val elements = doc.body().select("div#books-genders > div > div")
return elements.mapNotNullToSet { element ->
MangaTag(
title = element.select("label").text(),
key = element.select("input").attr("value"),
source = source,
)
}
}
private fun parseStatus(status: String) = when {
status.contains("Publicándose") -> MangaState.ONGOING
status.contains("Finalizado") -> MangaState.FINISHED
else -> null
}
}

View File

@@ -1,294 +0,0 @@
package org.koitharu.kotatsu.parsers.site.fmreader
import kotlinx.coroutines.async
import kotlinx.coroutines.coroutineScope
import org.jsoup.nodes.Document
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import java.text.DateFormat
import java.text.SimpleDateFormat
import java.util.*
internal abstract class FmreaderParser(
context: MangaLoaderContext,
source: MangaSource,
domain: String,
pageSize: Int = 20,
) : PagedMangaParser(context, source, pageSize) {
override val configKeyDomain = ConfigKey.Domain(domain)
override val availableSortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.UPDATED,
SortOrder.POPULARITY,
SortOrder.ALPHABETICAL,
)
override val isMultipleTagsSupported = false
protected open val listUrl = "/manga-list.html"
protected open val datePattern = "MMMM d, yyyy"
protected open val tagPrefix = "manga-list-genre-"
init {
paginator.firstPage = 1
searchPaginator.firstPage = 1
}
@JvmField
protected val ongoing: Set<String> = setOf(
"On going",
"Incomplete",
"En curso",
)
@JvmField
protected val finished: Set<String> = setOf(
"Completed",
"Completado",
)
@JvmField
protected val abandoned: Set<String> = hashSetOf(
"Canceled",
"Cancelled",
"Drop",
)
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val tag = tags.oneOrThrowIfMany()
val url = buildString {
append("https://")
append(domain)
append(listUrl)
append("?page=")
append(page.toString())
when {
!query.isNullOrEmpty() -> {
append("&name=")
append(query.urlEncoded())
}
!tags.isNullOrEmpty() -> {
append("&genre=")
append(tag?.key.orEmpty())
}
}
append("&sort=")
when (sortOrder) {
SortOrder.POPULARITY -> append("views")
SortOrder.UPDATED -> append("last_update")
SortOrder.ALPHABETICAL -> append("name")
else -> append("last_update")
}
}
val doc = webClient.httpGet(url).parseHtml()
return doc.select("div.thumb-item-flow").map { div ->
val href = div.selectFirstOrThrow("div.series-title a").attrAsRelativeUrl("href")
Manga(
id = generateUid(href),
url = href,
publicUrl = href.toAbsoluteUrl(div.host ?: domain),
coverUrl = div.selectFirstOrThrow("div.img-in-ratio").attr("data-bg")
?: div.selectFirstOrThrow("div.img-in-ratio").attr("style").substringAfter("('")
.substringBeforeLast("')"),
title = div.selectFirstOrThrow("div.series-title").text().orEmpty(),
altTitle = null,
rating = RATING_UNKNOWN,
tags = emptySet(),
author = null,
state = null,
source = source,
isNsfw = isNsfwSource,
)
}
}
protected open val selectBodyTag = "ul.filter-type li a"
override suspend fun getAvailableTags(): Set<MangaTag> {
val doc = webClient.httpGet("https://$domain/$listUrl").parseHtml()
return doc.select(selectBodyTag).mapNotNullToSet { a ->
val href = a.attr("href").substringAfter(tagPrefix).substringBeforeLast(".html")
MangaTag(
key = href,
title = a.text(),
source = source,
)
}
}
protected open val selectDesc = "div.summary-content"
protected open val selectState = "ul.manga-info li:contains(Status) a"
protected open val selectAlt = "ul.manga-info li:contains(Other names)"
protected open val selectAut = "ul.manga-info li:contains(Author(s)) a"
protected open val selectTag = "ul.manga-info li:contains(Genre(s)) a"
override suspend fun getDetails(manga: Manga): Manga = coroutineScope {
val fullUrl = manga.url.toAbsoluteUrl(domain)
val doc = webClient.httpGet(fullUrl).parseHtml()
val chaptersDeferred = async { getChapters(doc) }
val desc = doc.selectFirst(selectDesc)?.html()
val stateDiv = doc.selectFirst(selectState)
val state = stateDiv?.let {
when (it.text()) {
in ongoing -> MangaState.ONGOING
in finished -> MangaState.FINISHED
in abandoned -> MangaState.ABANDONED
else -> null
}
}
val alt = doc.body().selectFirst(selectAlt)?.text()?.replace("Other names", "")
val auth = doc.body().selectFirst(selectAut)?.text()
manga.copy(
tags = doc.body().select(selectTag).mapNotNullToSet { a ->
MangaTag(
key = a.attr("href").substringAfter(tagPrefix).substringBeforeLast(".html"),
title = a.text().toTitleCase(),
source = source,
)
},
description = desc,
altTitle = alt,
author = auth,
state = state,
chapters = chaptersDeferred.await(),
)
}
protected open val selectDate = "div.chapter-time"
protected open val selectChapter = "ul.list-chapters a"
protected open suspend fun getChapters(doc: Document): List<MangaChapter> {
val dateFormat = SimpleDateFormat(datePattern, sourceLocale)
return doc.body().select(selectChapter).mapChapters(reversed = true) { i, a ->
val href = a.attrAsRelativeUrl("href")
val dateText = a.selectFirst(selectDate)?.text()
MangaChapter(
id = generateUid(href),
name = a.selectFirstOrThrow("div.chapter-name").text(),
number = i + 1,
url = href,
uploadDate = parseChapterDate(
dateFormat,
dateText,
),
source = source,
scanlator = null,
branch = null,
)
}
}
protected open val selectPage = "div.chapter-content img"
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(domain)
val doc = webClient.httpGet(fullUrl).parseHtml()
return doc.select(selectPage).map { img ->
val url = img.src()?.toRelativeUrl(domain) ?: img.parseFailed("Image src not found")
MangaPage(
id = generateUid(url),
url = url,
preview = null,
source = source,
)
}
}
protected fun parseChapterDate(dateFormat: DateFormat, date: String?): Long {
// Clean date (e.g. 5th December 2019 to 5 December 2019) before parsing it
val d = date?.lowercase() ?: return 0
return when {
d.endsWith(" ago") ||
d.endsWith(" atrás") ||
// short Hours
d.endsWith(" h") ||
// short Day
d.endsWith(" d") -> parseRelativeDate(date)
// Handle 'yesterday' and 'today', using midnight
d.startsWith("year") -> Calendar.getInstance().apply {
add(Calendar.DAY_OF_MONTH, -1) // yesterday
set(Calendar.HOUR_OF_DAY, 0)
set(Calendar.MINUTE, 0)
set(Calendar.SECOND, 0)
set(Calendar.MILLISECOND, 0)
}.timeInMillis
d.startsWith("today") -> Calendar.getInstance().apply {
set(Calendar.HOUR_OF_DAY, 0)
set(Calendar.MINUTE, 0)
set(Calendar.SECOND, 0)
set(Calendar.MILLISECOND, 0)
}.timeInMillis
date.contains(Regex("""\d(st|nd|rd|th)""")) -> date.split(" ").map {
if (it.contains(Regex("""\d\D\D"""))) {
it.replace(Regex("""\D"""), "")
} else {
it
}
}.let { dateFormat.tryParse(it.joinToString(" ")) }
else -> dateFormat.tryParse(date)
}
}
// Parses dates in this form:
// 21 hours ago
private fun parseRelativeDate(date: String): Long {
val number = Regex("""(\d+)""").find(date)?.value?.toIntOrNull() ?: return 0
val cal = Calendar.getInstance()
return when {
WordSet("second").anyWordIn(date) -> cal.apply { add(Calendar.SECOND, -number) }.timeInMillis
WordSet("min", "minute", "minutes", "minuto", "minutos").anyWordIn(date) -> cal.apply {
add(
Calendar.MINUTE,
-number,
)
}.timeInMillis
WordSet("hour", "hours", "hora", "horas", "h").anyWordIn(date) -> cal.apply {
add(
Calendar.HOUR,
-number,
)
}.timeInMillis
WordSet("day", "days", "día", "dia").anyWordIn(date) -> cal.apply {
add(
Calendar.DAY_OF_MONTH,
-number,
)
}.timeInMillis
WordSet("week", "weeks", "semana", "semanas").anyWordIn(date) -> cal.apply {
add(
Calendar.WEEK_OF_YEAR,
-number,
)
}.timeInMillis
WordSet("month", "months", "mes", "meses").anyWordIn(date) -> cal.apply {
add(
Calendar.MONTH,
-number,
)
}.timeInMillis
WordSet("year", "año", "años").anyWordIn(date) -> cal.apply { add(Calendar.YEAR, -number) }.timeInMillis
else -> 0
}
}
}

View File

@@ -1,152 +0,0 @@
package org.koitharu.kotatsu.parsers.site.fmreader.en
import kotlinx.coroutines.async
import kotlinx.coroutines.coroutineScope
import org.jsoup.nodes.Document
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.site.fmreader.FmreaderParser
import org.koitharu.kotatsu.parsers.util.*
import java.text.SimpleDateFormat
@MangaSourceParser("MANHWA18COM", "Manhwa18.com", "en", ContentType.HENTAI)
internal class Manhwa18Com(context: MangaLoaderContext) :
FmreaderParser(context, MangaSource.MANHWA18COM, "manhwa18.com") {
override val listUrl = "/tim-kiem"
override val selectState = "div.info-item:contains(Status) span.info-value "
override val selectAlt = "div.info-item:contains(Other name) span.info-value "
override val selectTag = "div.info-item:contains(Genre) span.info-value a"
override val datePattern = "dd/MM/yyyy"
override val selectPage = "div#chapter-content img"
override val selectBodyTag = "div.genres-menu a"
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val tag = tags.oneOrThrowIfMany()
val url = buildString {
append("https://")
append(domain)
if (!tags.isNullOrEmpty()) {
append("/genre/")
append(tag?.key.orEmpty())
append("?page=")
append(page.toString())
append("&sort=")
when (sortOrder) {
SortOrder.POPULARITY -> append("views")
SortOrder.UPDATED -> append("last_update")
SortOrder.ALPHABETICAL -> append("name")
else -> append("last_update")
}
} else {
append(listUrl)
append("?page=")
append(page.toString())
when {
!query.isNullOrEmpty() -> {
append("&q=")
append(query.urlEncoded())
}
}
append("&sort=")
when (sortOrder) {
SortOrder.POPULARITY -> append("views")
SortOrder.UPDATED -> append("last_update")
SortOrder.ALPHABETICAL -> append("name")
else -> append("last_update")
}
}
}
val doc = webClient.httpGet(url).parseHtml()
return doc.select("div.thumb-item-flow").map { div ->
val href = div.selectFirstOrThrow("div.series-title a").attrAsRelativeUrl("href")
Manga(
id = generateUid(href),
url = href,
publicUrl = href.toAbsoluteUrl(div.host ?: domain),
coverUrl = div.selectFirstOrThrow("div.img-in-ratio").attr("data-bg")
?: div.selectFirstOrThrow("div.img-in-ratio").attr("style").substringAfter("('")
.substringBeforeLast("')"),
title = div.selectFirstOrThrow("div.series-title").text().orEmpty(),
altTitle = null,
rating = RATING_UNKNOWN,
tags = emptySet(),
author = null,
state = null,
source = source,
isNsfw = isNsfwSource,
)
}
}
override suspend fun getAvailableTags(): Set<MangaTag> {
val doc = webClient.httpGet("https://$domain/$listUrl").parseHtml()
return doc.select(selectBodyTag).mapNotNullToSet { a ->
val href = a.attr("href").substringAfterLast("/")
MangaTag(
key = href,
title = a.text(),
source = source,
)
}
}
override suspend fun getDetails(manga: Manga): Manga = coroutineScope {
val fullUrl = manga.url.toAbsoluteUrl(domain)
val doc = webClient.httpGet(fullUrl).parseHtml()
val chaptersDeferred = async { getChapters(doc) }
val desc = doc.selectFirstOrThrow(selectDesc).html()
val stateDiv = doc.selectFirst(selectState)
val state = stateDiv?.let {
when (it.text()) {
in ongoing -> MangaState.ONGOING
in finished -> MangaState.FINISHED
else -> null
}
}
val alt = doc.body().selectFirst(selectAlt)?.text()?.replace("Other name", "")
val auth = doc.body().selectFirst(selectAut)?.text()
manga.copy(
tags = doc.body().select(selectTag).mapNotNullToSet { a ->
MangaTag(
key = a.attr("href").substringAfter("manga-list-genre-").substringBeforeLast(".html"),
title = a.text().toTitleCase(),
source = source,
)
},
description = desc,
altTitle = alt,
author = auth,
state = state,
chapters = chaptersDeferred.await(),
)
}
override suspend fun getChapters(doc: Document): List<MangaChapter> {
val dateFormat = SimpleDateFormat(datePattern, sourceLocale)
return doc.body().select(selectChapter).mapChapters(reversed = true) { i, a ->
val href = a.attrAsRelativeUrl("href")
val dateText = a.selectFirst(selectDate)?.text()?.substringAfter("- ")
MangaChapter(
id = generateUid(href),
name = a.selectFirstOrThrow("div.chapter-name").text(),
number = i + 1,
url = href,
uploadDate = parseChapterDate(
dateFormat,
dateText,
),
source = source,
scanlator = null,
branch = null,
)
}
}
}

View File

@@ -1,89 +0,0 @@
package org.koitharu.kotatsu.parsers.site.fmreader.es
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.site.fmreader.FmreaderParser
import org.koitharu.kotatsu.parsers.util.*
@MangaSourceParser("OLIMPOSCANS", "OlimpoScans", "es")
internal class OlimpoScans(context: MangaLoaderContext) :
FmreaderParser(context, MangaSource.OLIMPOSCANS, "olimposcans.com") {
override val selectState = "ul.manga-info li:contains(Estado) a"
override val selectAlt = "ul.manga-info li:contains(Otros nombres)"
override val selectTag = "ul.manga-info li:contains(Género) a"
override val tagPrefix = "lista-de-comics-genero-"
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val tag = tags.oneOrThrowIfMany()
val url = buildString {
append("https://")
append(domain)
append(listUrl)
append("?page=")
append(page.toString())
when {
!query.isNullOrEmpty() -> {
append("&name=")
append(query.urlEncoded())
}
!tags.isNullOrEmpty() -> {
append("&genre=")
append(tag?.key.orEmpty())
}
}
append("&sort=")
when (sortOrder) {
SortOrder.POPULARITY -> append("views")
SortOrder.UPDATED -> append("last_update")
SortOrder.ALPHABETICAL -> append("name")
else -> append("last_update")
}
}
val doc = webClient.httpGet(url).parseHtml()
val lastPage =
doc.selectLast(".pagination a")?.attr("href")?.substringAfterLast("page=")?.substringBeforeLast("&artist")
?.toInt() ?: 1
if (lastPage < page) {
return emptyList()
}
return doc.select("div.thumb-item-flow").map { div ->
val href = "/" + div.selectFirstOrThrow("div.series-title a").attrAsRelativeUrl("href")
Manga(
id = generateUid(href),
url = href,
publicUrl = href.toAbsoluteUrl(div.host ?: domain),
coverUrl = div.selectFirstOrThrow("div.img-in-ratio").attr("data-bg").toAbsoluteUrl(domain),
title = div.selectFirstOrThrow("div.series-title").text().orEmpty(),
altTitle = null,
rating = RATING_UNKNOWN,
tags = emptySet(),
author = null,
state = null,
source = source,
isNsfw = isNsfwSource,
)
}
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = ("/" + chapter.url).toAbsoluteUrl(domain)
val doc = webClient.httpGet(fullUrl).parseHtml()
return doc.select(selectPage).map { img ->
val url = ("/proxy.php?link=" + img.src()).toRelativeUrl(domain)
MangaPage(
id = generateUid(url),
url = url,
preview = null,
source = source,
)
}
}
}

View File

@@ -1,125 +0,0 @@
package org.koitharu.kotatsu.parsers.site.fmreader.ja
import org.jsoup.nodes.Document
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.model.Manga
import org.koitharu.kotatsu.parsers.model.MangaChapter
import org.koitharu.kotatsu.parsers.model.MangaPage
import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.model.MangaTag
import org.koitharu.kotatsu.parsers.model.RATING_UNKNOWN
import org.koitharu.kotatsu.parsers.model.SortOrder
import org.koitharu.kotatsu.parsers.site.fmreader.FmreaderParser
import org.koitharu.kotatsu.parsers.util.*
import java.text.SimpleDateFormat
@MangaSourceParser("KLZ9", "Klz9", "ja")
internal class Klz9(context: MangaLoaderContext) :
FmreaderParser(context, MangaSource.KLZ9, "klz9.com") {
override val selectDesc = "div.row:contains(Description)"
override val selectState = "ul.manga-info li:contains(Status) a"
override val selectAlt = "ul.manga-info li:contains(Other name (s))"
override val selectTag = "ul.manga-info li:contains(Genre(s)) a"
override val selectChapter = "tr"
override val selectDate = "td i"
override val selectPage = "img"
override val selectBodyTag = "div.panel-body a"
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val tag = tags.oneOrThrowIfMany()
val url = buildString {
append("https://")
append(domain)
append("/$listUrl")
append("?page=")
append(page.toString())
when {
!query.isNullOrEmpty() -> {
append("&name=")
append(query.urlEncoded())
}
!tags.isNullOrEmpty() -> {
append("&genre=")
append(tag?.key.orEmpty())
}
}
append("&sort=")
when (sortOrder) {
SortOrder.POPULARITY -> append("views")
SortOrder.UPDATED -> append("last_update")
SortOrder.ALPHABETICAL -> append("name")
else -> append("last_update")
}
}
val doc = webClient.httpGet(url).parseHtml()
return doc.select("div.thumb-item-flow").map { div ->
val href = "/" + div.selectFirstOrThrow("a").attrAsRelativeUrl("href")
Manga(
id = generateUid(href),
url = href,
publicUrl = href.toAbsoluteUrl(div.host ?: domain),
coverUrl = div.selectFirstOrThrow("div.img-in-ratio").attr("style").substringAfter("('")
.substringBeforeLast("')"),
title = div.selectFirstOrThrow("div.series-title").text().orEmpty(),
altTitle = null,
rating = RATING_UNKNOWN,
tags = emptySet(),
author = null,
state = null,
source = source,
isNsfw = isNsfwSource,
)
}
}
override suspend fun getChapters(doc: Document): List<MangaChapter> {
val slug = doc.selectFirstOrThrow("div.h0rating").attr("slug")
val docLoad =
webClient.httpGet("https://$domain/app/manga/controllers/cont.listChapter.php?slug=$slug").parseHtml()
val dateFormat = SimpleDateFormat(datePattern, sourceLocale)
return docLoad.body().select(selectChapter).mapChapters(reversed = true) { i, a ->
val href = "/" + a.selectFirstOrThrow("a.chapter").attrAsRelativeUrl("href")
val dateText = a.selectFirst(selectDate)?.text()
MangaChapter(
id = generateUid(href),
name = a.selectFirstOrThrow("a").text(),
number = i + 1,
url = href,
uploadDate = parseChapterDate(
dateFormat,
dateText,
),
source = source,
scanlator = null,
branch = null,
)
}
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(domain)
val doc = webClient.httpGet(fullUrl).parseHtml()
val cid = doc.selectFirstOrThrow("#chapter").attr("value")
val docLoad = webClient.httpGet("https://$domain/app/manga/controllers/cont.listImg.php?cid=$cid").parseHtml()
return docLoad.select(selectPage).map { img ->
val url = img.src()?.toRelativeUrl(domain) ?: img.parseFailed("Image src not found")
MangaPage(
id = generateUid(url),
url = url,
preview = null,
source = source,
)
}
}
}

View File

@@ -1,57 +0,0 @@
package org.koitharu.kotatsu.parsers.site.fmreader.ja
import org.jsoup.nodes.Document
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.model.MangaChapter
import org.koitharu.kotatsu.parsers.model.MangaPage
import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.site.fmreader.FmreaderParser
import org.koitharu.kotatsu.parsers.util.*
import java.text.SimpleDateFormat
@MangaSourceParser("WELOVEMANGA", "WeLoveManga", "ja")
internal class WeLoveManga(context: MangaLoaderContext) :
FmreaderParser(context, MangaSource.WELOVEMANGA, "welovemanga.one") {
override suspend fun getChapters(doc: Document): List<MangaChapter> {
val mid = doc.selectFirstOrThrow("div.cmt input").attr("value")
val docLoad =
webClient.httpGet("https://$domain/app/manga/controllers/cont.Listchapter.php?mid=$mid").parseHtml()
val dateFormat = SimpleDateFormat(datePattern, sourceLocale)
return docLoad.body().select(selectChapter).mapChapters(reversed = true) { i, a ->
val href = a.selectFirstOrThrow("a").attrAsRelativeUrl("href")
val dateText = a.selectFirst(selectDate)?.text()
MangaChapter(
id = generateUid(href),
name = a.selectFirstOrThrow("a").text(),
number = i + 1,
url = href,
uploadDate = parseChapterDate(
dateFormat,
dateText,
),
source = source,
scanlator = null,
branch = null,
)
}
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(domain)
val doc = webClient.httpGet(fullUrl).parseHtml()
val cid = doc.selectFirstOrThrow("#chapter").attr("value")
val docLoad = webClient.httpGet("https://$domain/app/manga/controllers/cont.listImg.php?cid=$cid").parseHtml()
return docLoad.select("img").map { img ->
val url = img.src()?.toRelativeUrl(domain) ?: img.parseFailed("Image src not found")
MangaPage(
id = generateUid(url),
url = url,
preview = null,
source = source,
)
}
}
}

View File

@@ -1,10 +0,0 @@
package org.koitharu.kotatsu.parsers.site.fmreader.ja
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.site.fmreader.FmreaderParser
@MangaSourceParser("WELOMA", "Weloma", "ja")
internal class Weloma(context: MangaLoaderContext) :
FmreaderParser(context, MangaSource.WELOMA, "weloma.art")

View File

@@ -1,163 +0,0 @@
package org.koitharu.kotatsu.parsers.site.foolslide
import kotlinx.coroutines.coroutineScope
import org.json.JSONArray
import org.jsoup.nodes.Document
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import java.text.SimpleDateFormat
import java.util.*
internal abstract class FoolSlideParser(
context: MangaLoaderContext,
source: MangaSource,
domain: String,
pageSize: Int = 25,
) : PagedMangaParser(context, source, pageSize) {
override val configKeyDomain = ConfigKey.Domain(domain)
override val availableSortOrders: Set<SortOrder> = EnumSet.of(SortOrder.ALPHABETICAL)
protected open val listUrl = "directory/"
protected open val searchUrl = "search/"
protected open val pagination = true // false if the manga list has no pages
protected open val datePattern = "yyyy.MM.dd"
init {
paginator.firstPage = 1
searchPaginator.firstPage = 1
}
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val doc = if (!query.isNullOrEmpty()) {
val url = buildString {
append("https://$domain/$searchUrl")
if (page > 1) {
return emptyList()
}
}
val q = query.urlEncoded()
webClient.httpPost(url, "search=$q").parseHtml()
} else {
val url = buildString {
append("https://$domain/$listUrl")
// For some sites that don't have enough manga and page 2 links to page 1
if (!pagination) {
if (page > 1) {
return emptyList()
}
} else {
append(page.toString())
}
}
webClient.httpGet(url).parseHtml()
}
return doc.select("div.list div.group").map { div ->
val href = div.selectFirstOrThrow("a").attrAsRelativeUrl("href")
Manga(
id = generateUid(href),
url = href,
publicUrl = href.toAbsoluteUrl(div.host ?: domain),
coverUrl = div.selectFirst("img")?.src().orEmpty(),// in search no img
title = div.selectFirstOrThrow(".title a").text().orEmpty(),
altTitle = null,
rating = RATING_UNKNOWN,
tags = emptySet(),
author = null,
state = null,
source = source,
isNsfw = isNsfwSource,
)
}
}
override suspend fun getAvailableTags(): Set<MangaTag> = emptySet()
protected open val selectInfo = "div.info"
override suspend fun getDetails(manga: Manga): Manga = coroutineScope {
val fullUrl = manga.url.toAbsoluteUrl(domain)
val testAdultPage = webClient.httpGet(fullUrl).parseHtml()
val doc = if (testAdultPage.selectFirst("div.info form") != null) {
webClient.httpPost(fullUrl, "adult=true").parseHtml()
} else {
testAdultPage
}
val chapters = getChapters(doc)
val desc = if (doc.selectFirstOrThrow(selectInfo).html().contains("</b>")) {
doc.selectFirstOrThrow(selectInfo).text().substringAfterLast(": ")
} else {
doc.selectFirstOrThrow(selectInfo).text()
}
val author = if (doc.selectFirstOrThrow(selectInfo).html().contains("</b>")) {
doc.selectFirstOrThrow(selectInfo).text().substringAfter(": ").substringBefore("Art")
} else {
null
}
manga.copy(
coverUrl = doc.selectFirst(".thumbnail img")?.src() ?: manga.coverUrl,
description = desc,
altTitle = null,
author = author,
state = null,
chapters = chapters,
)
}
protected open val selectDate = ".meta_r"
protected open val selectChapter = "div.list div.element"
protected open suspend fun getChapters(doc: Document): List<MangaChapter> {
val dateFormat = SimpleDateFormat(datePattern, sourceLocale)
return doc.body().select(selectChapter).mapChapters(reversed = true) { i, div ->
val a = div.selectFirstOrThrow(".title a")
val href = a.attrAsRelativeUrl("href")
val dateText = div.selectFirstOrThrow(selectDate).text().substringAfter(", ")
MangaChapter(
id = generateUid(href),
name = a.text(),
number = i + 1,
url = href,
uploadDate = if (div.selectFirstOrThrow(selectDate).text().contains(", ")) {
dateFormat.tryParse(dateText)
} else {
0
},
source = source,
scanlator = null,
branch = null,
)
}
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(domain)
val doc = webClient.httpGet(fullUrl).parseHtml()
val script = doc.selectFirstOrThrow("script:containsData(var pages = )")
val images = JSONArray(script.data().substringAfterLast("var pages = ").substringBefore(';'))
val pages = ArrayList<MangaPage>(images.length())
for (i in 0 until images.length()) {
val pageTake = images.getJSONObject(i)
pages.add(
MangaPage(
id = generateUid(pageTake.getString("url")),
url = pageTake.getString("url"),
preview = null,
source = source,
),
)
}
return pages
}
}

View File

@@ -1,135 +0,0 @@
package org.koitharu.kotatsu.parsers.site.foolslide.en
import kotlinx.coroutines.coroutineScope
import org.jsoup.nodes.Document
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.site.foolslide.FoolSlideParser
import org.koitharu.kotatsu.parsers.util.*
@MangaSourceParser("ASSORTEDSCANS", "AssortedScans", "en")
internal class AssortedScans(context: MangaLoaderContext) :
FoolSlideParser(context, MangaSource.ASSORTEDSCANS, "assortedscans.com", 56) {
override val listUrl = "reader/"
override val pagination = false
override val selectInfo = "div.#series-info"
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val doc = if (!query.isNullOrEmpty()) {
if (page > 1) {
return emptyList()
}
val url = buildString {
append("https://")
append(domain)
append('/')
append(searchUrl)
append("?q=")
append(query.urlEncoded())
}
webClient.httpGet(url).parseHtml()
} else {
val url = buildString {
append("https://$domain/$listUrl")
// For some sites that don't have enough manga and page 2 links to page 1
if (!pagination) {
if (page > 1) {
return emptyList()
}
} else {
append(page.toString())
}
}
webClient.httpGet(url).parseHtml()
}
return doc.select("section.series, tr.result").map { div ->
val href = div.selectFirstOrThrow("a").attrAsRelativeUrl("href")
Manga(
id = generateUid(href),
url = href,
publicUrl = href.toAbsoluteUrl(div.host ?: domain),
coverUrl = div.selectFirst("img")?.src().orEmpty(),// in search no img
title = div.selectFirstOrThrow("a").text().orEmpty(),
altTitle = null,
rating = RATING_UNKNOWN,
tags = emptySet(),
author = null,
state = null,
source = source,
isNsfw = isNsfwSource,
)
}
}
override suspend fun getDetails(manga: Manga): Manga = coroutineScope {
val fullUrl = manga.url.toAbsoluteUrl(domain)
val testAdultPage = webClient.httpGet(fullUrl).parseHtml()
val doc = if (testAdultPage.selectFirst("div.info form") != null) {
webClient.httpPost(fullUrl, "adult=true").parseHtml()
} else {
testAdultPage
}
val chapters = getChapters(doc)
val desc = doc.getElementById("series-desc")?.selectFirst("div")?.html()
val alt = doc.getElementById("series-aliases")?.selectFirst("div.alias")?.text()
val author = doc.getElementById("series-authors")?.selectFirst("div.author")?.text()
val state = doc.getElementById("series-status")?.selectFirst("span")?.text()
manga.copy(
coverUrl = doc.selectFirst(".cover")?.src() ?: manga.coverUrl,
description = desc,
altTitle = alt,
author = author,
state = when (state) {
"Ongoing" -> MangaState.ONGOING
"Completed" -> MangaState.FINISHED
"Canceled" -> MangaState.ABANDONED
else -> null
},
chapters = chapters,
)
}
override suspend fun getChapters(doc: Document): List<MangaChapter> {
return doc.body().select("div.chapter").mapChapters(reversed = true) { i, div ->
val a = div.selectFirstOrThrow("a")
val href = a.attrAsRelativeUrl("href")
MangaChapter(
id = generateUid(href),
name = a.text(),
number = i + 1,
url = href,
uploadDate = 0,
source = source,
scanlator = null,
branch = null,
)
}
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml()
return doc.body().select(".page-list .dropdown-list li a").map { a ->
val url = a.attr("href").toRelativeUrl(domain)
MangaPage(
id = generateUid(url),
url = url,
preview = null,
source = source,
)
}
}
override suspend fun getPageUrl(page: MangaPage): String {
val doc = webClient.httpGet(page.url.toAbsoluteUrl(domain)).parseHtml()
val root = doc.body()
return root.requireElementById("page-image").attr("src") ?: doc.parseFailed("Page image not found")
}
}

View File

@@ -1,10 +0,0 @@
package org.koitharu.kotatsu.parsers.site.foolslide.en
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.site.foolslide.FoolSlideParser
@MangaSourceParser("DEATHTOLLSCANS", "DeathTollScans", "en")
internal class Deathtollscans(context: MangaLoaderContext) :
FoolSlideParser(context, MangaSource.DEATHTOLLSCANS, "reader.deathtollscans.net", 26)

View File

@@ -1,12 +0,0 @@
package org.koitharu.kotatsu.parsers.site.foolslide.en
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.site.foolslide.FoolSlideParser
@MangaSourceParser("MANGATELLERS", "Mangatellers", "en")
internal class Mangatellers(context: MangaLoaderContext) :
FoolSlideParser(context, MangaSource.MANGATELLERS, "reader.mangatellers.gr") {
override val pagination = false
}

View File

@@ -1,10 +0,0 @@
package org.koitharu.kotatsu.parsers.site.foolslide.en
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.site.foolslide.FoolSlideParser
@MangaSourceParser("READER_EVILFLOWERS", "Evil Flowers", "en")
internal class ReaderEvilflowers(context: MangaLoaderContext) :
FoolSlideParser(context, MangaSource.READER_EVILFLOWERS, "reader.evilflowers.com")

View File

@@ -1,44 +0,0 @@
package org.koitharu.kotatsu.parsers.site.foolslide.en
import kotlinx.coroutines.coroutineScope
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.site.foolslide.FoolSlideParser
import org.koitharu.kotatsu.parsers.util.*
@MangaSourceParser("SEINAGI", "Seinagi", "en")
internal class Seinagi(context: MangaLoaderContext) :
FoolSlideParser(context, MangaSource.SEINAGI, "reader.seinagi.org.es") {
override val pagination = false
override suspend fun getDetails(manga: Manga): Manga = coroutineScope {
val fullUrl = manga.url.toAbsoluteUrl(domain)
val testAdultPage = webClient.httpGet(fullUrl).parseHtml()
val doc = if (testAdultPage.selectFirst("div.info form") != null) {
webClient.httpPost(fullUrl, "adult=true").parseHtml()
} else {
testAdultPage
}
val chapters = getChapters(doc)
val desc = if (doc.selectFirstOrThrow(selectInfo).html().contains("Description")) {
doc.selectFirstOrThrow(selectInfo).text().substringAfter("Description: ").substringBefore("Readings")
} else {
doc.selectFirstOrThrow(selectInfo).text()
}
val author = if (doc.selectFirstOrThrow(selectInfo).html().contains("Author")) {
doc.selectFirstOrThrow(selectInfo).text().substringAfter("Author: ").substringBefore("Art")
} else {
null
}
manga.copy(
coverUrl = doc.selectFirst(".thumbnail img")?.src() ?: manga.coverUrl,
description = desc,
altTitle = null,
author = author,
state = null,
chapters = chapters,
)
}
}

View File

@@ -1,13 +0,0 @@
package org.koitharu.kotatsu.parsers.site.foolslide.es
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.site.foolslide.FoolSlideParser
@MangaSourceParser("MENUDO_FANSUB", "Menudo Fansub", "es")
internal class MenudoFansub(context: MangaLoaderContext) :
FoolSlideParser(context, MangaSource.MENUDO_FANSUB, "www.menudo-fansub.com", 25) {
override val searchUrl = "slide/search/"
override val listUrl = "slide/directory/"
}

View File

@@ -1,42 +0,0 @@
package org.koitharu.kotatsu.parsers.site.foolslide.es
import kotlinx.coroutines.coroutineScope
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.site.foolslide.FoolSlideParser
import org.koitharu.kotatsu.parsers.util.*
@MangaSourceParser("PZYKOSIS666HFANSUB", "Pzykosis666h Fansub", "es", ContentType.HENTAI)
internal class Pzykosis666hFansub(context: MangaLoaderContext) :
FoolSlideParser(context, MangaSource.PZYKOSIS666HFANSUB, "lector.pzykosis666hfansub.com") {
override suspend fun getDetails(manga: Manga): Manga = coroutineScope {
val fullUrl = manga.url.toAbsoluteUrl(domain)
val testAdultPage = webClient.httpGet(fullUrl).parseHtml()
val doc = if (testAdultPage.selectFirst("div.info form") != null) {
webClient.httpPost(fullUrl, "adult=true").parseHtml()
} else {
testAdultPage
}
val chapters = getChapters(doc)
val desc = if (doc.selectFirstOrThrow(selectInfo).html().contains("Descripción")) {
doc.selectFirstOrThrow(selectInfo).text().substringAfter("Descripción: ").substringBefore("Lecturas")
} else {
doc.selectFirstOrThrow(selectInfo).text()
}
val author = if (doc.selectFirstOrThrow(selectInfo).html().contains("Author")) {
doc.selectFirstOrThrow(selectInfo).text().substringAfter("Author: ").substringBefore("Art")
} else {
null
}
manga.copy(
coverUrl = doc.selectFirst(".thumbnail img")?.src() ?: manga.coverUrl,
description = desc,
altTitle = null,
author = author,
state = null,
chapters = chapters,
)
}
}

View File

@@ -1,48 +0,0 @@
package org.koitharu.kotatsu.parsers.site.foolslide.es
import kotlinx.coroutines.coroutineScope
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.model.ContentType
import org.koitharu.kotatsu.parsers.model.Manga
import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.site.foolslide.FoolSlideParser
import org.koitharu.kotatsu.parsers.util.domain
import org.koitharu.kotatsu.parsers.util.parseHtml
import org.koitharu.kotatsu.parsers.util.selectFirstOrThrow
import org.koitharu.kotatsu.parsers.util.src
import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl
@MangaSourceParser("SEINAGIADULTO", "Seinagi Adulto", "es", ContentType.HENTAI)
internal class SeinagiAdulto(context: MangaLoaderContext) :
FoolSlideParser(context, MangaSource.SEINAGIADULTO, "adulto.seinagi.org.es") {
override suspend fun getDetails(manga: Manga): Manga = coroutineScope {
val fullUrl = manga.url.toAbsoluteUrl(domain)
val testAdultPage = webClient.httpGet(fullUrl).parseHtml()
val doc = if (testAdultPage.selectFirst("div.info form") != null) {
webClient.httpPost(fullUrl, "adult=true").parseHtml()
} else {
testAdultPage
}
val chapters = getChapters(doc)
val desc = if (doc.selectFirstOrThrow(selectInfo).html().contains("Descripción")) {
doc.selectFirstOrThrow(selectInfo).text().substringAfter("Descripción: ").substringBefore("Lecturas")
} else {
doc.selectFirstOrThrow(selectInfo).text()
}
val author = if (doc.selectFirstOrThrow(selectInfo).html().contains("Author")) {
doc.selectFirstOrThrow(selectInfo).text().substringAfter("Author: ").substringBefore("Art")
} else {
null
}
manga.copy(
coverUrl = doc.selectFirst(".thumbnail img")?.src().orEmpty(),// for manga result on search
description = desc,
altTitle = null,
author = author,
state = null,
chapters = chapters,
)
}
}

View File

@@ -1,14 +0,0 @@
package org.koitharu.kotatsu.parsers.site.foolslide.fr
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.site.foolslide.FoolSlideParser
@MangaSourceParser("HNISCANTRAD", "Hni Scantrad", "fr")
internal class HniScantrad(context: MangaLoaderContext) :
FoolSlideParser(context, MangaSource.HNISCANTRAD, "hni-scantrad.com") {
override val pagination = false
override val searchUrl = "lel/search/"
override val listUrl = "lel/directory/"
}

View File

@@ -1,14 +0,0 @@
package org.koitharu.kotatsu.parsers.site.foolslide.it
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.site.foolslide.FoolSlideParser
@MangaSourceParser("POWERMANGA", "PowerManga", "it")
internal class PowerManga(context: MangaLoaderContext) :
FoolSlideParser(context, MangaSource.POWERMANGA, "reader.powermanga.org") {
override val pagination = false
}

View File

@@ -1,15 +0,0 @@
package org.koitharu.kotatsu.parsers.site.foolslide.it
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.site.foolslide.FoolSlideParser
@MangaSourceParser("RAMAREADER", "RamaReader", "it")
internal class Ramareader(context: MangaLoaderContext) :
FoolSlideParser(context, MangaSource.RAMAREADER, "www.ramareader.it") {
override val searchUrl = "read/search/"
override val listUrl = "read/directory/"
}

View File

@@ -1,15 +0,0 @@
package org.koitharu.kotatsu.parsers.site.foolslide.it
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.site.foolslide.FoolSlideParser
@MangaSourceParser("READNIFTEAM", "ReadNifTeam", "it")
internal class ReadNifteam(context: MangaLoaderContext) :
FoolSlideParser(context, MangaSource.READNIFTEAM, "read-nifteam.info") {
override val searchUrl = "slide/search/"
override val listUrl = "slide/directory/"
}

View File

@@ -1,229 +0,0 @@
package org.koitharu.kotatsu.parsers.site.fr
import kotlinx.coroutines.async
import kotlinx.coroutines.awaitAll
import kotlinx.coroutines.coroutineScope
import okhttp3.Headers
import org.jsoup.nodes.Element
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.exception.ParseException
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import org.koitharu.kotatsu.parsers.util.json.getIntOrDefault
import java.util.*
@MangaSourceParser("BENTOMANGA", "BentoManga", "fr")
internal class BentomangaParser(context: MangaLoaderContext) : PagedMangaParser(context, MangaSource.BENTOMANGA, 10) {
override val availableSortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.UPDATED,
SortOrder.POPULARITY,
SortOrder.RATING,
SortOrder.NEWEST,
SortOrder.ALPHABETICAL,
)
override val configKeyDomain = ConfigKey.Domain("bentomanga.com", "www.bentomanga.com")
init {
paginator.firstPage = 0
searchPaginator.firstPage = 0
}
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val url = urlBuilder()
.addPathSegment("manga_list")
.addQueryParameter("limit", page.toString())
.addQueryParameter(
"order_by",
when (sortOrder) {
SortOrder.UPDATED -> "update"
SortOrder.POPULARITY -> "views"
SortOrder.RATING -> "top"
SortOrder.NEWEST -> "create"
SortOrder.ALPHABETICAL -> "name"
},
)
if (!tags.isNullOrEmpty()) {
url.addQueryParameter("withCategories", tags.joinToString(",") { it.key })
}
if (!query.isNullOrEmpty()) {
url.addQueryParameter("search", query)
}
val root = webClient.httpGet(url.build()).parseHtml().requireElementById("mangas_content")
return root.select(".manga[data-manga]").map { div ->
val header = div.selectFirstOrThrow(".manga_header")
val href = header.selectFirstOrThrow("a").attrAsRelativeUrl("href")
Manga(
id = generateUid(href),
title = div.selectFirstOrThrow("h1").text(),
altTitle = null,
url = href,
publicUrl = href.toAbsoluteUrl(domain),
rating = div.getElementsByAttributeValue("data-icon", "avg_rate")
.firstOrNull()
?.textOrNull()
?.toFloatOrNull()
?.div(10f)
?: RATING_UNKNOWN,
isNsfw = div.selectFirst(".badge-adult_content") != null,
coverUrl = div.selectFirstOrThrow("img").src().assertNotNull("src").orEmpty(),
tags = div.selectFirst(".component-manga-categories")
.assertNotNull("tags")
?.select("a")
?.mapToSet { a ->
MangaTag(
title = a.text().toTitleCase(sourceLocale),
key = a.attr("href").substringAfterLast('='),
source = source,
)
}.orEmpty(),
state = null,
author = null,
description = div.selectFirst(".manga_synopsis")?.html().assertNotNull("description"),
source = source,
)
}
}
override suspend fun getDetails(manga: Manga): Manga {
val mangaUrl = manga.url.toAbsoluteUrl(domain)
val root = webClient.httpGet(mangaUrl).parseHtml()
.requireElementById("container_manga_show")
return manga.copy(
altTitle = root.selectFirst(".component-manga-title_alt")?.text(),
description = root.selectFirst(".datas_synopsis")?.html().assertNotNull("description")
?: manga.description,
state = when (root.selectFirst(".datas_more-status-data")?.textOrNull().assertNotNull("status")) {
"En cours" -> MangaState.ONGOING
"Terminé" -> MangaState.FINISHED
"Abandonné" -> MangaState.ABANDONED
"En pause" -> MangaState.PAUSED
else -> null
},
author = root.selectFirst(".datas_more-authors-people")?.textOrNull().assertNotNull("author"),
chapters = run {
val input = root.selectFirst("input[name=\"limit\"]") ?: return@run parseChapters(root)
val max = input.attr("max").toInt()
if (max <= 1) {
parseChapters(root)
} else {
coroutineScope {
val result = ArrayList(parseChapters(root))
result.ensureCapacity(result.size * max)
(1..max).map { i ->
async {
loadChapters(mangaUrl, i)
}
}.awaitAll()
.flattenTo(result)
result
}
}
}.reversed(),
)
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val chapterUrl = chapter.url.toAbsoluteUrl(domain)
val doc = webClient.httpGet(chapterUrl).parseHtml()
val chapterId = doc.head().getElementsByAttribute("data-chapter-id").first()!!.attr("data-chapter-id")
val json = webClient.httpGet(
"https://$domain/api/?id=$chapterId&type=chapter",
Headers.headersOf(
"Referer", chapterUrl,
"x-requested-with", "XMLHttpRequest",
),
).parseJson()
if (json.getIntOrDefault("type", 1) == 2) {
throw ParseException("Light Novels are not supported", chapterUrl)
}
val baseUrl = json.getString("baseImagesUrl")
val pages = json.getJSONArray("page_array")
return (0 until pages.length()).map { i ->
val url = concatUrl(baseUrl, pages.getString(i))
MangaPage(
id = generateUid(url),
url = url.toAbsoluteUrl(domain),
preview = null,
source = source,
)
}
}
override suspend fun getAvailableTags(): Set<MangaTag> {
val root = webClient.httpGet(urlBuilder().addPathSegment("manga_list").build())
.parseHtml()
.requireElementById("search_options-form")
return root.getElementsByAttributeValue("name", "categories[]")
.mapToSet { input ->
val div = input.parents().first()!!
MangaTag(
title = div.text().toTitleCase(sourceLocale),
key = input.attr("value"),
source = source,
)
}
}
private suspend fun loadChapters(baseUrl: String, page: Int): List<MangaChapter> {
return parseChapters(webClient.httpGet("$baseUrl?limit=$page").parseHtml().body())
}
private fun parseChapters(root: Element): List<MangaChapter> {
return root.requireElementById("chapters_content")
.select(".component-chapter").map { div ->
val a = div.selectFirstOrThrow("a:not([style*='display:none'])")
val href = a.attrAsRelativeUrl("href")
val title = div.selectFirstOrThrow(".chapter_volume").text()
val name = div.selectFirst(".chapter_title")?.textOrNull()
MangaChapter(
id = generateUid(href),
name = if (name != null && name != title) "$title: $name" else title,
number = href.substringAfterLast('/').toIntOrNull() ?: 0,
url = href,
scanlator = div.selectFirst(".team_link-name")?.textOrNull(),
uploadDate = div.selectFirst(".component-chapter-date")
?.ownTextOrNull()
.parseDate(),
branch = null,
source = source,
)
}
}
private fun String?.parseDate(): Long {
if (this == null) {
assert(false) { "Date is null" }
return 0L
}
val parts = split(' ')
assert(parts.size == 2) { "Wrong date $this" }
val count = parts.getOrNull(0)?.toIntOrNull() ?: return 0L
val unit = parts.getOrNull(1) ?: return 0L
val calendarUnit = when (unit) {
"s" -> Calendar.SECOND
"min" -> Calendar.MINUTE
"h" -> Calendar.HOUR
"j" -> Calendar.DAY_OF_YEAR
"sem." -> Calendar.WEEK_OF_YEAR
"mois" -> Calendar.MONTH
"ans", "an" -> Calendar.YEAR
else -> {
assert(false) { "Unknown time unit $unit" }
return 0L
}
}
val calendar = Calendar.getInstance()
calendar.add(calendarUnit, -count)
return calendar.timeInMillis
}
}

View File

@@ -1,148 +0,0 @@
package org.koitharu.kotatsu.parsers.site.fr
import kotlinx.coroutines.coroutineScope
import org.json.JSONArray
import org.json.JSONObject
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import org.koitharu.kotatsu.parsers.util.json.*
import java.text.SimpleDateFormat
import java.util.*
@MangaSourceParser("FMTEAM", "FmTeam", "fr")
internal class FmTeam(context: MangaLoaderContext) :
PagedMangaParser(context, MangaSource.FMTEAM, 0) {
override val availableSortOrders: Set<SortOrder> = EnumSet.of(SortOrder.ALPHABETICAL)
override val configKeyDomain = ConfigKey.Domain("fmteam.fr")
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
if (page > 1) {
return emptyList()
}
val jsonManga = if (!query.isNullOrEmpty()) {
//3 letters minimum
webClient.httpGet("https://$domain/api/search/${query.urlEncoded()}").parseJson().getJSONArray("comics")
} else {
webClient.httpGet("https://$domain/api/comics").parseJson().getJSONArray("comics")
}
val manga = ArrayList<Manga>(jsonManga.length())
for (i in 0 until jsonManga.length()) {
val j = jsonManga.getJSONObject(i)
val href = "/api" + j.getString("url")
when {
!tags.isNullOrEmpty() -> {
val a = j.getJSONArray("genres").toString()
var found = true
tags.forEach {
if (!a.contains(it.key, ignoreCase = true)) {
found = false
}
}
if (found) {
manga.add(
addManga(href, j),
)
}
}
else -> {
manga.add(
addManga(href, j),
)
}
}
}
return manga
}
private fun addManga(href: String, j: JSONObject): Manga {
return Manga(
id = generateUid(href),
url = href,
publicUrl = href.toAbsoluteUrl(domain),
coverUrl = j.getString("thumbnail"),
title = j.getString("title"),
description = j.getString("description"),
altTitle = j.getJSONArray("alt_titles").toString()
.replace("[\"", "")
.replace("\"]", "")
.replace("\",\"", " , "),
rating = j.getString("rating").toFloatOrNull()?.div(10f)
?: RATING_UNKNOWN,
tags = emptySet(),
author = j.getString("author"),
state = when (j.getString("status").lowercase()) {
"en cours" -> MangaState.ONGOING
"terminé" -> MangaState.FINISHED
else -> null
},
source = source,
isNsfw = when (j.getString("adult").toInt()) {
0 -> false
1 -> true
else -> true
},
)
}
override suspend fun getAvailableTags(): Set<MangaTag> = emptySet()
override suspend fun getDetails(manga: Manga): Manga = coroutineScope {
val fullUrl = manga.url.toAbsoluteUrl(domain)
val json = webClient.httpGet(fullUrl).parseJson().getJSONObject("comic")
val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.US)
val chapters = JSONArray(json.getJSONArray("chapters").toJSONList().reversed())
manga.copy(
tags = json.getJSONArray("genres").toJSONList().mapNotNullToSet {
MangaTag(
key = it.getString("slug"),
title = it.getString("name"),
source = source,
)
},
chapters = chapters.mapJSONIndexed { i, j ->
val url = "/api" + j.getString("url").toRelativeUrl(domain)
val name = j.getString("full_title")
val date = j.getStringOrNull("updated_at")
MangaChapter(
id = generateUid(url),
name = name,
number = i + 1,
url = url,
scanlator = null,
uploadDate = dateFormat.tryParse(date),
branch = null,
source = source,
)
},
)
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(domain)
val jsonPages = webClient.httpGet(fullUrl).parseJson().getJSONObject("chapter").getJSONArray("pages").toString()
val pages = jsonPages.replace("[", "").replace("]", "")
.replace("\\", "").split("\",\"").drop(1)
return pages.map { url ->
MangaPage(
id = generateUid(url),
url = url,
preview = null,
source = source,
)
}
}
}

View File

@@ -1,158 +0,0 @@
package org.koitharu.kotatsu.parsers.site.fr
import kotlinx.coroutines.coroutineScope
import okhttp3.Headers
import org.jsoup.nodes.Document
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.network.UserAgents
import org.koitharu.kotatsu.parsers.util.*
import java.text.DateFormat
import java.text.SimpleDateFormat
import java.util.*
@MangaSourceParser("FURYOSOCIETY", "FuryoSociety", "fr")
internal class FuryoSociety(context: MangaLoaderContext) :
PagedMangaParser(context, MangaSource.FURYOSOCIETY, 0) {
override val availableSortOrders: Set<SortOrder> = EnumSet.of(SortOrder.ALPHABETICAL, SortOrder.UPDATED)
override val configKeyDomain = ConfigKey.Domain("furyosociety.com")
override val headers: Headers = Headers.Builder()
.add("User-Agent", UserAgents.CHROME_DESKTOP)
.build()
override suspend fun getFavicons(): Favicons {
return Favicons(
listOf(
Favicon("https://$domain/fs_favicon/favicon-32x32.png", 32, null),
),
domain,
)
}
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val url = buildString {
append("https://")
append(domain)
if (page == 1) {
if (sortOrder == SortOrder.ALPHABETICAL) {
append("/mangas")
}
} else {
return emptyList()
}
}
val doc = webClient.httpGet(url).parseHtml()
return doc.select("div.fs-card-container div.grid-item-container").ifEmpty {
doc.select("div.container-tight.latest table tr")
}.map { div ->
val href = div.selectFirstOrThrow("a").attrAsRelativeUrl("href")
Manga(
id = generateUid(href),
url = href,
publicUrl = href.toAbsoluteUrl(div.host ?: domain),
coverUrl = div.selectFirst("img")?.src().orEmpty(),
title = (div.selectFirst("div.media-body") ?: div.selectFirst("a"))?.text().orEmpty(),
altTitle = null,
rating = RATING_UNKNOWN,
tags = emptySet(),
author = null,
state = null,
source = source,
isNsfw = false,
)
}
}
override suspend fun getAvailableTags(): Set<MangaTag> = emptySet()
override suspend fun getDetails(manga: Manga): Manga = coroutineScope {
val fullUrl = manga.url.toAbsoluteUrl(domain)
val doc = webClient.httpGet(fullUrl).parseHtml()
val chaptersDeferred = getChapters(doc)
manga.copy(
description = doc.selectFirstOrThrow("div.fs-comic-description").html(),
chapters = chaptersDeferred,
isNsfw = doc.selectFirst(".adult-text") != null,
)
}
private fun getChapters(doc: Document): List<MangaChapter> {
return doc.body().select("div.list.fs-chapter-list div.element").mapChapters(reversed = true) { i, div ->
val a = div.selectFirstOrThrow("div.title a")
val href = a.attrAsRelativeUrl("href")
val dateFormat = SimpleDateFormat("dd/MM/yyyy", sourceLocale)
val dateText = div.selectFirstOrThrow("div.meta_r").text().replace("Hier", "1 jour")
MangaChapter(
id = generateUid(href),
name = div.selectFirstOrThrow("div.title").text() + " : " + div.selectFirstOrThrow("div.name").text(),
number = i + 1,
url = href,
uploadDate = parseChapterDate(
dateFormat,
dateText,
),
source = source,
scanlator = null,
branch = null,
)
}
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(domain)
val doc = webClient.httpGet(fullUrl).parseHtml()
return doc.select("div.main-img img").map { url ->
val img = url.src()?.toRelativeUrl(domain) ?: url.parseFailed("Image src not found")
MangaPage(
id = generateUid(img),
url = img,
preview = null,
source = source,
)
}
}
private fun parseChapterDate(dateFormat: DateFormat, date: String?): Long {
val d = date?.lowercase() ?: return 0
return when {
d.startsWith("il y a") || // Handle translated 'ago' in French.
d.endsWith(" an") || d.endsWith(" ans") ||
d.endsWith(" mois") ||
d.endsWith(" jour") || d.endsWith(" jours") ||
d.endsWith(" heure") || d.endsWith(" heures") ||
d.endsWith(" seconde") || d.endsWith(" secondes") ||
d.endsWith(" minute") || d.endsWith(" minutes") -> parseRelativeDate(date)
else -> dateFormat.tryParse(date)
}
}
private fun parseRelativeDate(date: String): Long {
val number = Regex("""(\d+)""").find(date)?.value?.toIntOrNull() ?: return 0
val cal = Calendar.getInstance()
return when {
WordSet("seconde", "secondes").anyWordIn(date) -> cal.apply { add(Calendar.SECOND, -number) }.timeInMillis
WordSet("minute", "minutes").anyWordIn(date) -> cal.apply { add(Calendar.MINUTE, -number) }.timeInMillis
WordSet("heure", "heures").anyWordIn(date) -> cal.apply { add(Calendar.HOUR, -number) }.timeInMillis
WordSet("jour", "jours").anyWordIn(date) -> cal.apply { add(Calendar.DAY_OF_MONTH, -number) }.timeInMillis
WordSet("mois").anyWordIn(date) -> cal.apply { add(Calendar.MONTH, -number) }.timeInMillis
WordSet("an", "ans").anyWordIn(date) -> cal.apply { add(Calendar.YEAR, -number) }.timeInMillis
else -> 0
}
}
}

Some files were not shown because too many files have changed in this diff Show More