47 Commits

Author SHA1 Message Date
e1db2994d7 [TuMangaOnline] Initial adding 2022-07-30 20:14:09 +03:00
85bfe42ddf [Tatakae Scans] Fix favicon 2022-07-30 17:11:08 +03:00
dac06ddd0b [Tatakae Scans] New source 2022-07-30 15:10:44 +03:00
b632571ebc Update readme 2022-07-28 10:04:07 +03:00
7588617316 New source: PrismaScans 2022-07-27 15:56:06 +03:00
dd6de64c44 New manga source: UnionManga 2022-07-27 15:56:06 +03:00
fadb06aabb [Remanga] Fix details parsing 2022-07-20 19:41:49 +03:00
c4abb758f3 Make jsoup dependency api 2022-07-20 18:19:07 +03:00
6495ddf277 Add NotFoundException for 404 response 2022-07-20 17:09:37 +03:00
30071709af Add base url to ParseException 2022-07-19 11:36:26 +03:00
37fda4bd9e Oops! 2022-07-14 17:00:15 +03:00
a1dc901439 Update issue templates 2022-07-14 16:58:54 +03:00
6af8cec134 Update dependencies 2022-07-14 14:35:03 +03:00
67f6af63f8 [Madara] New source Mangas Origines (fr) 2022-07-14 11:59:00 +03:00
be595af075 Paginator class to compute page number by offset 2022-07-14 11:17:55 +03:00
c5c84c8630 Merge branch 'feature/favicons' 2022-07-13 15:13:56 +03:00
abc7eca40b Rename parseFavicons to getFavicons 2022-07-13 15:13:46 +03:00
2d1907569b [MangaLib] Fix authorization required detection 2022-07-12 10:43:29 +03:00
cb7a23950a [BatoTo] Fix pages parsing 2022-07-12 10:38:25 +03:00
941cc61577 Ignore malformed domains while ops with cookies 2022-07-12 10:21:21 +03:00
330495556a Refactor extra source headers 2022-07-12 09:31:18 +03:00
358910bafe [NineManga] Fix favicons redirect 2022-07-12 08:58:55 +03:00
c6b9180013 Parse favicons from websites 2022-07-11 13:31:50 +03:00
1f5d02ddb0 Merge pull request #36 from OshekharO/patch-3
Update Madara5Parser.kt
2022-07-08 14:50:52 +03:00
Saksham Shekher
f04d626e21 Update Madara5Parser.kt 2022-07-08 16:13:49 +05:30
Saksham Shekher
973d8fae27 Update Madara5Parser.kt 2022-07-08 16:08:19 +05:30
26d951bc20 [Madara5] Fix cover urls 2022-07-08 11:58:47 +03:00
dfaa0f5685 Merge pull request #35 from OshekharO/patch-3
Update Madara5Parser.kt
2022-07-08 11:51:25 +03:00
dee3e3d554 Favicons fixes 2022-07-08 11:51:07 +03:00
bccb1a491f [YaoiLib] New parser 2022-07-08 11:38:56 +03:00
5be4fb1114 Fix chapters duplicates 2022-07-08 11:26:05 +03:00
Saksham Shekher
a7c2235290 Update Madara5Parser.kt 2022-07-07 16:08:31 +05:30
8c26f3c790 [Grouple] Fix author parsing 2022-07-07 09:58:21 +03:00
aa341d6cab [Grouple] Fix auth url 2022-07-07 09:24:57 +03:00
b8b493fd15 Assume that 0 is no rating too 2022-07-06 16:50:29 +03:00
da3b0ae0cf [Grouple] Authorization support 2022-07-05 13:25:26 +03:00
79cdd18682 [Multichan] Authorization support 2022-07-05 13:10:13 +03:00
9f75b90325 [Madara] Add Madara5 parser 2022-07-02 14:22:25 +03:00
e97992b923 Update dependencies 2022-07-02 11:17:00 +03:00
035062d4bb Generate html report for parsers test 2022-06-30 15:18:46 +03:00
57667cb2b9 No fail test workflow 2022-06-30 15:18:45 +03:00
55ddc14244 [Issue template] Update 2022-06-24 23:16:52 +03:00
Saksham Shekher
a3f19abfb6 [Madara] Add manga sources (#24)
* Update MadaraParser.kt

* Update MadaraParser.kt

* Update MadaraParser.kt

* Update MadaraParser.kt

Have used " th " code for declaring Thai language.

* Update MadaraParser.kt

* Update MadaraParser.kt

* Update MadaraParser.kt

* Update MadaraParser.kt

* Update MadaraParser.kt

* Update MadaraParser.kt

* Update MadaraParser.kt

* Update MadaraParser.kt

From jp to ja

* Update MadaraParser.kt

* Update MadaraParser.kt

* VINLOAD

* Update MadaraParser.kt

* Manhwa

* MANGAKIK

* Update MadaraParser.kt
2022-06-24 22:16:38 +03:00
c92f89f307 [Madara] Fix empty tag names 2022-06-24 13:24:44 +03:00
a9684533b2 Merge pull request #21 from OshekharO/patch-1
[Madara] Add MangaRosie
2022-06-24 11:51:25 +03:00
e6659fabb0 Merge pull request #22 from KotatsuApp/feature/ci
Add ci job to run parsers test
2022-06-24 11:43:12 +03:00
Saksham Shekher
c660f356c7 Update MadaraParser.kt 2022-06-24 10:56:42 +05:30
67 changed files with 2251 additions and 402 deletions

View File

@@ -1,5 +1,5 @@
blank_issues_enabled: false
contact_links:
- name: ⚠️Application issue
url: https://github.com/nv95/Kotatsu/issues/new/choose
- name: ⚠️ Application issue
url: https://github.com/KotatsuApp/Kotatsu/issues/new/choose
about: Issues and requests about the app itself should be opened in the Kotatsu repository instead

View File

@@ -25,27 +25,7 @@ body:
2. Second step
3. Issue here
validations:
required: true
- type: textarea
id: expected-behavior
attributes:
label: Expected behavior
placeholder: |
Example:
"This should happen..."
validations:
required: true
- type: textarea
id: actual-behavior
attributes:
label: Actual behavior
placeholder: |
Example:
"This happened instead..."
validations:
required: true
required: false
- type: input
id: kotatsu-version
@@ -67,7 +47,7 @@ body:
placeholder: |
Example: "Android 12"
validations:
required: true
required: false
- type: textarea
id: other-details
@@ -80,15 +60,6 @@ body:
id: acknowledgements
attributes:
label: Acknowledgements
description: Your issue will be closed if you haven't done these steps.
options:
- label: I have searched the existing issues and this is a new ticket, **NOT** a duplicate or related to another open issue.
required: true
- label: I have written a short but informative title.
required: true
- label: I have updated the app to version **[3.3](https://github.com/nv95/Kotatsu/releases/latest)**.
required: true
- label: If this is an issue with the app itself, I should be opening an issue in the [app repository](https://github.com/nv95/Kotatsu/issues/new/choose).
required: true
- label: I will fill out all of the requested information in this form.
required: true

View File

@@ -25,15 +25,6 @@ body:
id: acknowledgements
attributes:
label: Acknowledgements
description: Your issue will be closed if you haven't done these steps.
options:
- label: I have searched the existing issues and this is a new ticket, **NOT** a duplicate or related to another open issue.
required: true
- label: I have written a short but informative title.
required: true
- label: If this is an issue with the app itself, I should be opening an issue in the [app repository](https://github.com/nv95/Kotatsu/issues/new/choose).
required: true
- label: I have updated the app to version **[3.3](https://github.com/nv95/Kotatsu/releases/latest)**.
required: true
- label: I will fill out all of the requested information in this form.
required: true

View File

@@ -41,15 +41,10 @@ body:
id: acknowledgements
attributes:
label: Acknowledgements
description: Your issue will be closed if you haven't done these steps.
options:
- label: I have searched the existing issues and this is a new ticket, **NOT** a duplicate or related to another open issue.
required: true
- label: I have written a title with source name.
required: true
- label: I have checked that the source does not already exist on the app.
required: true
- label: I have checked that the source does not already exist by searching the [GitHub repository](https://github.com/nv95/kotatsu-parsers) and verified it does not appear in the code base.
required: true
- label: I will fill out all of the requested information in this form.
- label: I have checked that the source does not already exist by searching the [GitHub repository](https://github.com/KotatsuApp/kotatsu-parsers) and verified it does not appear in the code base.
required: true

View File

@@ -1,19 +1,17 @@
name: Parsers test
on:
workflow_dispatch:
pull_request:
branches: [ "master" ]
paths:
- 'src/main/kotlin/org/koitharu/kotatsu/parsers/site'
- 'src/main/kotlin/org/koitharu/kotatsu/parsers/site/*'
permissions:
contents: read
jobs:
build:
build-and-test:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/setup-java@v3
@@ -21,8 +19,9 @@ jobs:
java-version: '11'
distribution: 'temurin'
cache: 'gradle'
- run: ./gradlew :test --tests "org.koitharu.kotatsu.parsers.MangaParserTest"
- run: ./gradlew :test --tests "org.koitharu.kotatsu.parsers.MangaParserTest" || true
- run: ./gradlew generateTestsReport
- uses: actions/upload-artifact@v3
with:
name: Report
path: build/reports/tests/test
path: build/test-results-html/TEST-org.koitharu.kotatsu.parsers.MangaParserTest.htm

6
.idea/kotlinc.xml generated Normal file
View File

@@ -0,0 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="KotlinJpsPluginSettings">
<option name="version" value="1.6.21" />
</component>
</project>

View File

@@ -2,48 +2,50 @@
Library that provides manga sources.
[![](https://jitpack.io/v/nv95/kotatsu-parsers.svg)](https://jitpack.io/#nv95/kotatsu-parsers) ![Kotlin](https://img.shields.io/github/languages/top/nv95/kotatsu-parsers) ![License](https://img.shields.io/github/license/nv95/Kotatsu) [![Discord](https://img.shields.io/discord/898363402467045416?color=5865f2&label=discord)](https://discord.gg/NNJ5RgVBC5)
[![](https://jitpack.io/v/KotatsuApp/kotatsu-parsers.svg)](https://jitpack.io/#KotatsuApp/kotatsu-parsers) ![Kotlin](https://img.shields.io/github/languages/top/KotatsuApp/kotatsu-parsers) ![License](https://img.shields.io/github/license/KotatsuApp/Kotatsu) [![Discord](https://img.shields.io/discord/898363402467045416?color=5865f2&label=discord)](https://discord.gg/NNJ5RgVBC5)
### Usage
1. Add it in your root build.gradle at the end of repositories:
```groovy
allprojects {
repositories {
...
maven { url 'https://jitpack.io' }
}
}
```
```groovy
allprojects {
repositories {
...
maven { url 'https://jitpack.io' }
}
}
```
2. Add the dependency
2. Add the dependency
For Java/Kotlin project:
For Java/Kotlin project:
```groovy
dependencies {
implementation("com.github.nv95:kotatsu-parsers:$parsers_version")
implementation("com.github.KotatsuApp:kotatsu-parsers:$parsers_version")
}
```
For Android project:
For Android project:
```groovy
dependencies {
implementation("com.github.nv95:kotatsu-parsers:$parsers_version") {
implementation("com.github.KotatsuApp:kotatsu-parsers:$parsers_version") {
exclude group: 'org.json', module: 'json'
}
}
```
See for versions at [JitPack](https://jitpack.io/#nv95/kotatsu-parsers)
See for versions at [JitPack](https://jitpack.io/#KotatsuApp/kotatsu-parsers)
3. Usage in code
```kotlin
val parser = MangaSource.MANGADEX.newParser(mangaLoaderContext)
```
`mangaLoaderContext` is an implementation of the `MangaLoaderContext` class.
See [Android](https://github.com/nv95/Kotatsu/blob/devel/app/src/main/java/org/koitharu/kotatsu/core/parser/MangaLoaderContextImpl.kt) and [Non-Android](https://github.com/nv95/kotatsu-dl/blob/master/src/main/kotlin/org/koitharu/kotatsu_dl/env/MangaLoaderContextImpl.kt) implementation examples.
```kotlin
val parser = MangaSource.MANGADEX.newParser(mangaLoaderContext)
```
Note that the `MangaSource.LOCAL` and `MangaSource.DUMMY` parsers cannot be instantiated.
`mangaLoaderContext` is an implementation of the `MangaLoaderContext` class.
See [Android](https://github.com/KotatsuApp/Kotatsu/blob/devel/app/src/main/java/org/koitharu/kotatsu/core/parser/MangaLoaderContextImpl.kt)
and [Non-Android](https://github.com/KotatsuApp/kotatsu-dl/blob/master/src/main/kotlin/org/koitharu/kotatsu_dl/env/MangaLoaderContextImpl.kt)
implementation examples.
Note that the `MangaSource.LOCAL` and `MangaSource.DUMMY` parsers cannot be instantiated.

View File

@@ -1,3 +1,5 @@
import tasks.ReportGenerateTask
plugins {
id 'java-library'
id 'org.jetbrains.kotlin.jvm'
@@ -51,10 +53,10 @@ afterEvaluate {
}
dependencies {
implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-core:1.6.2'
implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-core:1.6.4'
implementation 'com.squareup.okhttp3:okhttp:4.10.0'
implementation 'com.squareup.okio:okio:3.1.0'
implementation 'org.jsoup:jsoup:1.15.1'
implementation 'com.squareup.okio:okio:3.2.0'
api 'org.jsoup:jsoup:1.15.2'
implementation 'org.json:json:20220320'
implementation 'androidx.collection:collection-ktx:1.2.0'
@@ -63,6 +65,8 @@ dependencies {
testImplementation 'org.junit.jupiter:junit-jupiter-api:5.8.2'
testImplementation 'org.junit.jupiter:junit-jupiter-engine:5.8.2'
testImplementation 'org.junit.jupiter:junit-jupiter-params:5.8.2'
testImplementation 'org.jetbrains.kotlinx:kotlinx-coroutines-test:1.6.2'
testImplementation 'org.jetbrains.kotlinx:kotlinx-coroutines-test:1.6.4'
testImplementation 'io.webfolder:quickjs:1.1.0'
}
}
task generateTestsReport(type: ReportGenerateTask)

13
buildSrc/build.gradle Normal file
View File

@@ -0,0 +1,13 @@
plugins {
id('org.jetbrains.kotlin.jvm') version '1.6.21'
}
repositories {
mavenCentral()
}
dependencies {
implementation gradleApi()
implementation 'org.simpleframework:simple-xml:2.7.1'
implementation 'com.soywiz.korlibs.korte:korte-jvm:3.0.0-Beta5'
implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-core:1.6.3'
}

Binary file not shown.

View File

@@ -0,0 +1,5 @@
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-7.4-bin.zip
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists

234
buildSrc/gradlew vendored Executable file
View File

@@ -0,0 +1,234 @@
#!/bin/sh
#
# Copyright © 2015-2021 the original authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
##############################################################################
#
# Gradle start up script for POSIX generated by Gradle.
#
# Important for running:
#
# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is
# noncompliant, but you have some other compliant shell such as ksh or
# bash, then to run this script, type that shell name before the whole
# command line, like:
#
# ksh Gradle
#
# Busybox and similar reduced shells will NOT work, because this script
# requires all of these POSIX shell features:
# * functions;
# * expansions «$var», «${var}», «${var:-default}», «${var+SET}»,
# «${var#prefix}», «${var%suffix}», and «$( cmd )»;
# * compound commands having a testable exit status, especially «case»;
# * various built-in commands including «command», «set», and «ulimit».
#
# Important for patching:
#
# (2) This script targets any POSIX shell, so it avoids extensions provided
# by Bash, Ksh, etc; in particular arrays are avoided.
#
# The "traditional" practice of packing multiple parameters into a
# space-separated string is a well documented source of bugs and security
# problems, so this is (mostly) avoided, by progressively accumulating
# options in "$@", and eventually passing that to Java.
#
# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS,
# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly;
# see the in-line comments for details.
#
# There are tweaks for specific operating systems such as AIX, CygWin,
# Darwin, MinGW, and NonStop.
#
# (3) This script is generated from the Groovy template
# https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
# within the Gradle project.
#
# You can find Gradle at https://github.com/gradle/gradle/.
#
##############################################################################
# Attempt to set APP_HOME
# Resolve links: $0 may be a link
app_path=$0
# Need this for daisy-chained symlinks.
while
APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path
[ -h "$app_path" ]
do
ls=$( ls -ld "$app_path" )
link=${ls#*' -> '}
case $link in #(
/*) app_path=$link ;; #(
*) app_path=$APP_HOME$link ;;
esac
done
APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit
APP_NAME="Gradle"
APP_BASE_NAME=${0##*/}
# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
# Use the maximum available, or set MAX_FD != -1 to use that value.
MAX_FD=maximum
warn () {
echo "$*"
} >&2
die () {
echo
echo "$*"
echo
exit 1
} >&2
# OS specific support (must be 'true' or 'false').
cygwin=false
msys=false
darwin=false
nonstop=false
case "$( uname )" in #(
CYGWIN* ) cygwin=true ;; #(
Darwin* ) darwin=true ;; #(
MSYS* | MINGW* ) msys=true ;; #(
NONSTOP* ) nonstop=true ;;
esac
CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
# Determine the Java command to use to start the JVM.
if [ -n "$JAVA_HOME" ] ; then
if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
# IBM's JDK on AIX uses strange locations for the executables
JAVACMD=$JAVA_HOME/jre/sh/java
else
JAVACMD=$JAVA_HOME/bin/java
fi
if [ ! -x "$JAVACMD" ] ; then
die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
Please set the JAVA_HOME variable in your environment to match the
location of your Java installation."
fi
else
JAVACMD=java
which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
Please set the JAVA_HOME variable in your environment to match the
location of your Java installation."
fi
# Increase the maximum file descriptors if we can.
if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then
case $MAX_FD in #(
max*)
MAX_FD=$( ulimit -H -n ) ||
warn "Could not query maximum file descriptor limit"
esac
case $MAX_FD in #(
'' | soft) :;; #(
*)
ulimit -n "$MAX_FD" ||
warn "Could not set maximum file descriptor limit to $MAX_FD"
esac
fi
# Collect all arguments for the java command, stacking in reverse order:
# * args from the command line
# * the main class name
# * -classpath
# * -D...appname settings
# * --module-path (only if needed)
# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables.
# For Cygwin or MSYS, switch paths to Windows format before running java
if "$cygwin" || "$msys" ; then
APP_HOME=$( cygpath --path --mixed "$APP_HOME" )
CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" )
JAVACMD=$( cygpath --unix "$JAVACMD" )
# Now convert the arguments - kludge to limit ourselves to /bin/sh
for arg do
if
case $arg in #(
-*) false ;; # don't mess with options #(
/?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath
[ -e "$t" ] ;; #(
*) false ;;
esac
then
arg=$( cygpath --path --ignore --mixed "$arg" )
fi
# Roll the args list around exactly as many times as the number of
# args, so each arg winds up back in the position where it started, but
# possibly modified.
#
# NB: a `for` loop captures its iteration list before it begins, so
# changing the positional parameters here affects neither the number of
# iterations, nor the values presented in `arg`.
shift # remove old arg
set -- "$@" "$arg" # push replacement arg
done
fi
# Collect all arguments for the java command;
# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of
# shell script including quotes and variable substitutions, so put them in
# double quotes to make sure that they get re-expanded; and
# * put everything else in single quotes, so that it's not re-expanded.
set -- \
"-Dorg.gradle.appname=$APP_BASE_NAME" \
-classpath "$CLASSPATH" \
org.gradle.wrapper.GradleWrapperMain \
"$@"
# Use "xargs" to parse quoted args.
#
# With -n1 it outputs one arg per line, with the quotes and backslashes removed.
#
# In Bash we could simply go:
#
# readarray ARGS < <( xargs -n1 <<<"$var" ) &&
# set -- "${ARGS[@]}" "$@"
#
# but POSIX shell has neither arrays nor command substitution, so instead we
# post-process each arg (as a line of input to sed) to backslash-escape any
# character that might be a shell metacharacter, then use eval to reverse
# that process (while maintaining the separation between arguments), and wrap
# the whole thing up as a single "set" statement.
#
# This will of course break if any of these variables contains a newline or
# an unmatched quote.
#
eval "set -- $(
printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" |
xargs -n1 |
sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' |
tr '\n' ' '
)" '"$@"'
exec "$JAVACMD" "$@"

89
buildSrc/gradlew.bat vendored Normal file
View File

@@ -0,0 +1,89 @@
@rem
@rem Copyright 2015 the original author or authors.
@rem
@rem Licensed under the Apache License, Version 2.0 (the "License");
@rem you may not use this file except in compliance with the License.
@rem You may obtain a copy of the License at
@rem
@rem https://www.apache.org/licenses/LICENSE-2.0
@rem
@rem Unless required by applicable law or agreed to in writing, software
@rem distributed under the License is distributed on an "AS IS" BASIS,
@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@rem See the License for the specific language governing permissions and
@rem limitations under the License.
@rem
@if "%DEBUG%" == "" @echo off
@rem ##########################################################################
@rem
@rem Gradle startup script for Windows
@rem
@rem ##########################################################################
@rem Set local scope for the variables with windows NT shell
if "%OS%"=="Windows_NT" setlocal
set DIRNAME=%~dp0
if "%DIRNAME%" == "" set DIRNAME=.
set APP_BASE_NAME=%~n0
set APP_HOME=%DIRNAME%
@rem Resolve any "." and ".." in APP_HOME to make it shorter.
for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi
@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m"
@rem Find java.exe
if defined JAVA_HOME goto findJavaFromJavaHome
set JAVA_EXE=java.exe
%JAVA_EXE% -version >NUL 2>&1
if "%ERRORLEVEL%" == "0" goto execute
echo.
echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
echo.
echo Please set the JAVA_HOME variable in your environment to match the
echo location of your Java installation.
goto fail
:findJavaFromJavaHome
set JAVA_HOME=%JAVA_HOME:"=%
set JAVA_EXE=%JAVA_HOME%/bin/java.exe
if exist "%JAVA_EXE%" goto execute
echo.
echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
echo.
echo Please set the JAVA_HOME variable in your environment to match the
echo location of your Java installation.
goto fail
:execute
@rem Setup the command line
set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
@rem Execute Gradle
"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %*
:end
@rem End local scope for the variables with windows NT shell
if "%ERRORLEVEL%"=="0" goto mainEnd
:fail
rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
rem the _cmd.exe /c_ return code!
if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
exit /b 1
:mainEnd
if "%OS%"=="Windows_NT" endlocal
:omega

View File

@@ -0,0 +1,68 @@
package tasks
import com.soywiz.korte.Template
import kotlinx.coroutines.runBlocking
import org.gradle.api.DefaultTask
import org.gradle.api.tasks.TaskAction
import org.simpleframework.xml.Serializer
import org.simpleframework.xml.core.Persister
import tasks.model.TestCase
import tasks.model.TestSuite
import java.io.File
import kotlin.math.roundToInt
open class ReportGenerateTask : DefaultTask() {
@TaskAction
fun execute() {
val reportsRoot = File(project.rootDir, "build/test-results/test")
val outputRoot = File(project.rootDir, "build/test-results-html")
outputRoot.deleteRecursively()
outputRoot.mkdir()
for (file in checkNotNull(reportsRoot.listFiles()) {
"No files found in $reportsRoot"
}) {
if (file.isDirectory) {
continue
}
val report = makeReport(file)
val output = File(outputRoot, file.nameWithoutExtension + ".htm")
output.writeText(report)
println("Report generated: ${output.absolutePath}")
}
}
private fun makeReport(file: File): String {
val serializer: Serializer = Persister()
val testSuite = serializer.read(TestSuite::class.java, file)
val templateText = javaClass.classLoader.getResourceAsStream("report.html")?.use {
it.bufferedReader().readText()
}
val results = LinkedHashMap<String, LinkedHashMap<String, TestCase>>()
val tests = LinkedHashSet<String>()
for (case in testSuite.testCases) {
tests.add(case.testName)
val map = results.getOrPut(case.source) { LinkedHashMap() }
val oldValue = map.put(case.testName, case)
check(oldValue == null)
}
val failPercent = (testSuite.failures.toDouble() / testSuite.tests * 100.0).roundToInt()
val errorPercent = (testSuite.errors.toDouble() / testSuite.tests * 100.0).roundToInt()
return runBlocking {
val template = Template(requireNotNull(templateText))
template(
mapOf(
"testSuite" to testSuite,
"tests" to tests,
"results" to results,
"success_percent" to 100 - (failPercent + errorPercent),
"error_percent" to errorPercent,
"fail_percent" to failPercent,
"success" to testSuite.tests - (testSuite.failures + testSuite.errors),
),
)
}
}
}

View File

@@ -0,0 +1,24 @@
package tasks.model
import org.simpleframework.xml.Attribute
import org.simpleframework.xml.Root
import org.simpleframework.xml.Text
@Root(name = "failure", strict = false)
class Failure {
@JvmField
@field:Attribute(name = "message")
var message: String = ""
@JvmField
@field:Attribute(name = "type")
var type: String = ""
@JvmField
@field:Text
var text: String = ""
fun textHtml(): String {
return text.replace("\n", "<br>")
}
}

View File

@@ -0,0 +1,32 @@
package tasks.model
import org.simpleframework.xml.Attribute
import org.simpleframework.xml.Element
import org.simpleframework.xml.Root
@Root(name = "testcase", strict = false)
class TestCase {
@JvmField
@field:Attribute(name = "name")
var name: String = ""
@JvmField
@field:Attribute(name = "time")
var time: Float = 0f
@JvmField
@field:Element(name = "failure", required = false)
var failure: Failure? = null
val index by lazy {
name.split('|')[0].toInt()
}
val testName by lazy {
name.split('|')[1]
}
val source by lazy {
name.split('|')[2]
}
}

View File

@@ -0,0 +1,27 @@
package tasks.model
import org.simpleframework.xml.Attribute
import org.simpleframework.xml.ElementList
import org.simpleframework.xml.Root
@Root(name = "testsuite", strict = false)
class TestSuite {
@JvmField
@field:Attribute(name = "name")
var name: String = ""
@JvmField
@field:Attribute(name = "tests")
var tests: Int = 0
@JvmField
@field:Attribute(name = "failures")
var failures: Int = 0
@JvmField
@field:Attribute(name = "errors")
var errors: Int = 0
@field:ElementList(entry = "testcase", inline = true)
lateinit var testCases: List<TestCase>
}

View File

@@ -0,0 +1,91 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>{{ testSuite.name }}</title>
<link href="https://cdn.jsdelivr.net/npm/bootstrap@5.2.0-beta1/dist/css/bootstrap.min.css" rel="stylesheet"
integrity="sha384-0evHe/X+R7YkIZDRvuzKMRqM+OrBnVFBL6DOitfPri4tjfHxaWutUpFmBp4vmVor" crossorigin="anonymous">
<script src="https://cdn.jsdelivr.net/npm/bootstrap@5.2.0-beta1/dist/js/bootstrap.bundle.min.js"
integrity="sha384-pprn3073KE6tl6bjs2QrFaJGz5/SUsLqktiwsUTF55Jfv3qYSDhgCecCxMW52nD2"
crossorigin="anonymous"></script>
<script src="https://cdn.jsdelivr.net/npm/feather-icons/dist/feather.min.js"></script>
</head>
<body class="py-4">
<div class="container">
<h1 class="h2">{{ testSuite.name }}</h1>
<div class="progress mt-4">
<div class="progress-bar bg-success" role="progressbar" style="width: {{ success_percent }}%">
{{ success }} ({{ success_percent }}%)
</div>
<div class="progress-bar bg-danger" role="progressbar" style="width: {{ fail_percent }}%">
{{ testSuite.failures }} ({{ fail_percent }}%)
</div>
<div class="progress-bar bg-danger" role="progressbar" style="width: {{ error_percent }}%">
{{ testSuite.errors }} ({{ error_percent }}%)
</div>
</div>
<div class="table-responsive mt-4">
<table class="table table-hover">
<thead>
<tr>
<th scope="col">Source</th>
{% for test in tests %}
<th scope="col" style="min-width: 5em;">{{ test }}</th>
{% endfor %}
</tr>
</thead>
{% for name, cases in results %}
<tr>
<th scope="row">{{ name }}</th>
{% for test in tests %}
{% set case = cases[test] %}
{% if case.failure == null %}
<td class="table-success text-center">
<i data-feather="check"></i>
</td>
{% else %}
{% if case.failure.type == 'java.lang.AssertionError' %}
<td class="table-warning text-center" style="cursor: pointer;"
data-bs-toggle="modal" data-bs-target="#failure_{{ case.hashCode }}">
<i data-feather="alert-triangle"></i>
</td>
{% else %}
<td class="table-danger text-center" style="cursor: pointer;"
data-bs-toggle="modal" data-bs-target="#failure_{{ case.hashCode }}">
<i data-feather="x"></i>
</td>
{% endif %}
<!--suppress HtmlUnknownTag -->
<div class="modal fade" id="failure_{{ case.hashCode }}" tabindex="-1">
<div class="modal-dialog modal-lg modal-dialog-scrollable">
<div class="modal-content">
<div class="modal-header">
<h5 class="modal-title">{{ case.testName }} failed</h5>
<button type="button" class="btn-close" data-bs-dismiss="modal"
aria-label="Close"></button>
</div>
<div class="modal-body font-monospace lh-sm bg-light" style="font-size: 0.7em;">
{{ case.failure.textHtml()|raw }}
</div>
<div class="modal-footer">
<button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Close</button>
</div>
</div>
</div>
</div>
{% endif %}
{% endfor %}
</tr>
{% endfor %}
</table>
</div>
</div>
<script>
feather.replace()
</script>
</body>
</html>

View File

@@ -1,12 +1,14 @@
package org.koitharu.kotatsu.parsers
import okhttp3.*
import okhttp3.HttpUrl.Companion.toHttpUrl
import okhttp3.MediaType.Companion.toMediaType
import okhttp3.RequestBody.Companion.toRequestBody
import org.json.JSONObject
import org.jsoup.HttpStatusException
import org.koitharu.kotatsu.parsers.config.MangaSourceConfig
import org.koitharu.kotatsu.parsers.exception.GraphQLException
import org.koitharu.kotatsu.parsers.exception.NotFoundException
import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.util.await
import org.koitharu.kotatsu.parsers.util.parseJson
@@ -23,7 +25,7 @@ abstract class MangaLoaderContext {
* @param url
* @param headers an additional headers for request, may be null
*/
suspend fun httpGet(url: String, headers: Headers? = null): Response {
suspend fun httpGet(url: HttpUrl, headers: Headers? = null): Response {
val request = Request.Builder()
.get()
.url(url)
@@ -33,6 +35,11 @@ abstract class MangaLoaderContext {
return httpClient.newCall(request.build()).await().ensureSuccess()
}
suspend fun httpGet(url: String, headers: Headers? = null): Response {
return httpGet(url.toHttpUrl(), headers)
}
/**
* Do a HEAD http request to specific url
* @param url
@@ -140,8 +147,9 @@ abstract class MangaLoaderContext {
abstract fun getConfig(source: MangaSource): MangaSourceConfig
private fun Response.ensureSuccess() = apply {
private fun Response.ensureSuccess(): Response {
val exception: Exception? = when (code) { // Catch some error codes, not all
404 -> NotFoundException(message, request.url.toString())
in 500..599 -> HttpStatusException(message, code, request.url.toString())
else -> null
}
@@ -153,5 +161,6 @@ abstract class MangaLoaderContext {
}
throw exception
}
return this
}
}

View File

@@ -1,9 +1,14 @@
package org.koitharu.kotatsu.parsers
import androidx.annotation.CallSuper
import androidx.annotation.VisibleForTesting
import okhttp3.Headers
import okhttp3.HttpUrl
import org.jsoup.nodes.Element
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.exception.ParseException
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.FaviconParser
import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl
import java.util.*
@@ -20,6 +25,9 @@ abstract class MangaParser @InternalParsersApi constructor(val source: MangaSour
val config by lazy { context.getConfig(source) }
val sourceLocale: Locale?
get() = source.locale?.let { Locale(it) }
/**
* Provide default domain and available alternatives, if any.
*
@@ -27,6 +35,9 @@ abstract class MangaParser @InternalParsersApi constructor(val source: MangaSour
*/
protected abstract val configKeyDomain: ConfigKey.Domain
@VisibleForTesting(otherwise = VisibleForTesting.PROTECTED)
internal open val headers: Headers? = null
/**
* Used as fallback if value of `sortOrder` passed to [getList] is null
*/
@@ -60,7 +71,7 @@ abstract class MangaParser @InternalParsersApi constructor(val source: MangaSour
* @param offset starting from 0 and used for pagination.
* @param query search query
*/
suspend fun getList(offset: Int, query: String): List<Manga> {
open suspend fun getList(offset: Int, query: String): List<Manga> {
return getList(offset, query, null, defaultSortOrder)
}
@@ -72,7 +83,7 @@ abstract class MangaParser @InternalParsersApi constructor(val source: MangaSour
* @param tags genres for filtering, values from [getTags] and [Manga.tags]. May be null or empty
* @param sortOrder one of [sortOrders] or null for default value
*/
suspend fun getList(offset: Int, tags: Set<MangaTag>?, sortOrder: SortOrder?): List<Manga> {
open suspend fun getList(offset: Int, tags: Set<MangaTag>?, sortOrder: SortOrder?): List<Manga> {
return getList(offset, null, tags, sortOrder ?: defaultSortOrder)
}
@@ -102,8 +113,19 @@ abstract class MangaParser @InternalParsersApi constructor(val source: MangaSour
/**
* Returns direct link to the website favicon
*/
@Deprecated(
message = "Use parseFavicons() to get multiple favicons with different size",
replaceWith = ReplaceWith("parseFavicons()"),
)
open fun getFaviconUrl() = "https://${getDomain()}/favicon.ico"
/**
* Parse favicons from the main page of the source`s website
*/
open suspend fun getFavicons(): Favicons {
return FaviconParser(context, getDomain(), headers).parseFavicons()
}
@CallSuper
open fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) {
keys.add(configKeyDomain)
@@ -120,6 +142,12 @@ abstract class MangaParser @InternalParsersApi constructor(val source: MangaSour
return subdomain + "." + domain.removePrefix("www.")
}
fun urlBuilder(): HttpUrl.Builder {
return HttpUrl.Builder()
.scheme("https")
.host(getDomain())
}
/**
* Create a unique id for [Manga]/[MangaChapter]/[MangaPage].
* @param url must be relative url, without a domain
@@ -157,7 +185,16 @@ abstract class MangaParser @InternalParsersApi constructor(val source: MangaSour
}
@InternalParsersApi
protected fun parseFailed(message: String? = null): Nothing {
throw ParseException(message, null)
protected fun Element.parseFailed(message: String? = null): Nothing {
throw ParseException(message, ownerDocument()?.location() ?: baseUri(), null)
}
@InternalParsersApi
protected fun Set<MangaTag>?.oneOrThrowIfMany(): MangaTag? {
return when {
isNullOrEmpty() -> null
size == 1 -> first()
else -> throw IllegalArgumentException("Multiple genres are not supported by this source")
}
}
}

View File

@@ -0,0 +1,50 @@
package org.koitharu.kotatsu.parsers
import org.koitharu.kotatsu.parsers.model.Manga
import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.model.MangaTag
import org.koitharu.kotatsu.parsers.model.SortOrder
import org.koitharu.kotatsu.parsers.util.Paginator
@InternalParsersApi
abstract class PagedMangaParser(
source: MangaSource,
pageSize: Int,
searchPageSize: Int = pageSize,
) : MangaParser(source) {
protected val paginator = Paginator(pageSize)
protected val searchPaginator = Paginator(searchPageSize)
override suspend fun getList(offset: Int, query: String): List<Manga> {
return getList(searchPaginator, offset, query, null, defaultSortOrder)
}
override suspend fun getList(offset: Int, tags: Set<MangaTag>?, sortOrder: SortOrder?): List<Manga> {
return getList(paginator, offset, null, tags, sortOrder ?: defaultSortOrder)
}
@InternalParsersApi
@Deprecated("You should use getListPage for PagedMangaParser", level = DeprecationLevel.HIDDEN)
final override suspend fun getList(
offset: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> = throw UnsupportedOperationException("You should use getListPage for PagedMangaParser")
abstract suspend fun getListPage(page: Int, query: String?, tags: Set<MangaTag>?, sortOrder: SortOrder): List<Manga>
private suspend fun getList(
paginator: Paginator,
offset: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val page = paginator.getPage(offset)
val list = getListPage(page, query, tags, sortOrder)
paginator.onListReceived(offset, page, list.size)
return list
}
}

View File

@@ -0,0 +1,3 @@
package org.koitharu.kotatsu.parsers.exception
class ContentUnavailableException(message: String) : RuntimeException(message)

View File

@@ -0,0 +1,9 @@
package org.koitharu.kotatsu.parsers.exception
import org.jsoup.HttpStatusException
import java.net.HttpURLConnection
class NotFoundException(
message: String,
url: String,
) : HttpStatusException(message, HttpURLConnection.HTTP_NOT_FOUND, url)

View File

@@ -3,6 +3,7 @@ package org.koitharu.kotatsu.parsers.exception
import org.koitharu.kotatsu.parsers.InternalParsersApi
class ParseException @InternalParsersApi @JvmOverloads constructor(
message: String?,
val shortMessage: String?,
val url: String,
cause: Throwable? = null,
) : RuntimeException(message, cause)
) : RuntimeException("$shortMessage at $url", cause)

View File

@@ -0,0 +1,51 @@
package org.koitharu.kotatsu.parsers.model
import okhttp3.HttpUrl.Companion.toHttpUrl
class Favicon internal constructor(
val url: String,
val size: Int,
internal val rel: String?,
) : Comparable<Favicon> {
val type: String = url.toHttpUrl().pathSegments.last()
.substringAfterLast('.', "").lowercase()
override fun compareTo(other: Favicon): Int {
val res = size.compareTo(other.size)
if (res != 0) {
return res
}
return relWeightOf(rel).compareTo(relWeightOf(other.rel))
}
override fun equals(other: Any?): Boolean {
if (this === other) return true
if (javaClass != other?.javaClass) return false
other as Favicon
if (url != other.url) return false
if (size != other.size) return false
if (rel != other.rel) return false
return true
}
override fun hashCode(): Int {
var result = url.hashCode()
result = 31 * result + size
result = 31 * result + rel.hashCode()
return result
}
override fun toString(): String {
return "Favicon(size=$size, type='$type', rel='$rel', url='$url')"
}
private fun relWeightOf(rel: String?) = when (rel) {
"apple-touch-icon" -> 1 // Prefer apple-touch-icon because it has a better quality
"mask-icon" -> -1
else -> 0
}
}

View File

@@ -0,0 +1,45 @@
package org.koitharu.kotatsu.parsers.model
class Favicons internal constructor(
favicons: Collection<Favicon>,
val referer: String,
) : Collection<Favicon> {
private val icons = favicons.sortedDescending()
override val size: Int
get() = icons.size
override fun contains(element: Favicon): Boolean = icons.contains(element)
override fun containsAll(elements: Collection<Favicon>): Boolean = icons.containsAll(elements)
override fun isEmpty(): Boolean = icons.isEmpty()
override fun iterator(): Iterator<Favicon> = icons.iterator()
/**
* Finds a favicon whose size in pixels is greater than or equal to the specified size.
* If such icon is not available returns the largest icon
* @param size in pixels
* @param types supported file types, e.g. png, svg, ico. May be null but not empty
*/
@JvmOverloads
fun find(size: Int, types: Set<String>? = null): Favicon? {
if (icons.isEmpty()) {
return null
}
var result: Favicon? = null
for (icon in icons) {
if (types != null && icon.type !in types) {
continue
}
if (result == null || icon.size >= size) {
result = icon
} else {
break
}
}
return result
}
}

View File

@@ -74,7 +74,7 @@ class Manga(
* @see rating
*/
val hasRating: Boolean
get() = rating in 0f..1f
get() = rating > 0f && rating <= 1f
fun getChapters(branch: String?): List<MangaChapter>? {
return chapters?.filter { x -> x.branch == branch }

View File

@@ -5,9 +5,10 @@ import org.json.JSONArray
import org.json.JSONObject
import org.jsoup.nodes.Element
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.exception.ParseException
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import java.nio.charset.StandardCharsets
@@ -17,11 +18,12 @@ import javax.crypto.Cipher
import javax.crypto.spec.IvParameterSpec
import javax.crypto.spec.SecretKeySpec
private const val PAGE_SIZE = 60
private const val PAGE_SIZE_SEARCH = 20
@MangaSourceParser("BATOTO", "Bato.To")
internal class BatoToParser(override val context: MangaLoaderContext) : MangaParser(MangaSource.BATOTO) {
internal class BatoToParser(override val context: MangaLoaderContext) : PagedMangaParser(
source = MangaSource.BATOTO,
pageSize = 60,
searchPageSize = 20,
) {
override val sortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.NEWEST,
@@ -35,17 +37,15 @@ internal class BatoToParser(override val context: MangaLoaderContext) : MangaPar
arrayOf("bato.to", "mto.to", "mangatoto.com", "battwo.com", "batotwo.com", "comiko.net", "batotoo.com"),
)
override suspend fun getList(
offset: Int,
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
if (!query.isNullOrEmpty()) {
return search(offset, query)
return search(page, query)
}
val page = (offset / PAGE_SIZE) + 1
@Suppress("NON_EXHAUSTIVE_WHEN_STATEMENT")
val url = buildString {
append("https://")
@@ -54,6 +54,7 @@ internal class BatoToParser(override val context: MangaLoaderContext) : MangaPar
when (sortOrder) {
SortOrder.UPDATED,
-> append("update.za")
SortOrder.POPULARITY -> append("views_a.za")
SortOrder.NEWEST -> append("create.za")
SortOrder.ALPHABETICAL -> append("title.az")
@@ -70,8 +71,8 @@ internal class BatoToParser(override val context: MangaLoaderContext) : MangaPar
override suspend fun getDetails(manga: Manga): Manga {
val root = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml()
.getElementById("mainer") ?: parseFailed("Cannot find root")
val details = root.selectFirst(".detail-set") ?: parseFailed("Cannot find detail-set")
.requireElementById("mainer")
val details = root.selectFirstOrThrow(".detail-set")
val attrs = details.selectFirst(".attr-main")?.select(".attr-item")?.associate {
it.child(0).text().trim() to it.child(1)
}.orEmpty()
@@ -93,7 +94,7 @@ internal class BatoToParser(override val context: MangaLoaderContext) : MangaPar
?.selectFirst(".main")
?.children()
?.reversed()
?.mapIndexedNotNull { i, div ->
?.mapChapters { i, div ->
div.parseChapter(i)
}.orEmpty(),
)
@@ -104,7 +105,7 @@ internal class BatoToParser(override val context: MangaLoaderContext) : MangaPar
val scripts = context.httpGet(fullUrl).parseHtml().select("script")
for (script in scripts) {
val scriptSrc = script.html()
val p = scriptSrc.indexOf("const images =")
val p = scriptSrc.indexOf("const imgHttpLis =")
if (p == -1) continue
val start = scriptSrc.indexOf('[', p)
val end = scriptSrc.indexOf(';', start)
@@ -112,19 +113,19 @@ internal class BatoToParser(override val context: MangaLoaderContext) : MangaPar
continue
}
val images = JSONArray(scriptSrc.substring(start, end))
val batoJs = scriptSrc.substringBetweenFirst("batojs =", ";")?.trim(' ', '"', '\n')
?: parseFailed("Cannot find batojs")
val server = scriptSrc.substringBetweenFirst("server =", ";")?.trim(' ', '"', '\n')
?: parseFailed("Cannot find server")
val password = context.evaluateJs(batoJs)?.removeSurrounding('"')
?: parseFailed("Cannot evaluate batojs")
val serverDecrypted = decryptAES(server, password).removeSurrounding('"')
val batoPass = scriptSrc.substringBetweenFirst("batoPass =", ";")?.trim(' ', '"', '\n')
?: script.parseFailed("Cannot find batoPass")
val batoWord = scriptSrc.substringBetweenFirst("batoWord =", ";")?.trim(' ', '"', '\n')
?: script.parseFailed("Cannot find batoWord")
val password = context.evaluateJs(batoPass)?.removeSurrounding('"')
?: script.parseFailed("Cannot evaluate batoPass")
val args = JSONArray(decryptAES(batoWord, password))
val result = ArrayList<MangaPage>(images.length())
repeat(images.length()) { i ->
val url = images.getString(i)
result += MangaPage(
id = generateUid(url),
url = if (url.startsWith("http")) url else "$serverDecrypted$url",
url = url + "?" + args.getString(i),
referer = fullUrl,
preview = null,
source = source,
@@ -132,13 +133,13 @@ internal class BatoToParser(override val context: MangaLoaderContext) : MangaPar
}
return result
}
parseFailed("Cannot find images list")
throw ParseException("Cannot find images list", fullUrl)
}
override suspend fun getTags(): Set<MangaTag> {
val scripts = context.httpGet(
"https://${getDomain()}/browse",
).parseHtml().select("script")
).parseHtml().selectOrThrow("script")
for (script in scripts) {
val genres = script.html().substringBetweenFirst("const _genres =", ";") ?: continue
val jo = JSONObject(genres)
@@ -153,13 +154,12 @@ internal class BatoToParser(override val context: MangaLoaderContext) : MangaPar
}
return result
}
parseFailed("Cannot find gernes list")
throw ParseException("Cannot find gernes list", scripts[0].baseUri())
}
override fun getFaviconUrl(): String = "https://styles.amarkcdn.com/img/batoto/favicon.ico?v0"
private suspend fun search(offset: Int, query: String): List<Manga> {
val page = (offset / PAGE_SIZE_SEARCH) + 1
private suspend fun search(page: Int, query: String): List<Manga> {
val url = buildString {
append("https://")
append(getDomain())
@@ -174,7 +174,7 @@ internal class BatoToParser(override val context: MangaLoaderContext) : MangaPar
private fun getActivePage(body: Element): Int = body.select("nav ul.pagination > li.page-item.active")
.lastOrNull()
?.text()
?.toIntOrNull() ?: parseFailed("Cannot determine current page")
?.toIntOrNull() ?: body.parseFailed("Cannot determine current page")
private suspend fun parseList(url: String, page: Int): List<Manga> {
val body = context.httpGet(url).parseHtml().body()
@@ -185,11 +185,11 @@ internal class BatoToParser(override val context: MangaLoaderContext) : MangaPar
if (activePage != page) {
return emptyList()
}
val root = body.getElementById("series-list") ?: parseFailed("Cannot find root")
val root = body.requireElementById("series-list")
return root.children().map { div ->
val a = div.selectFirst("a") ?: parseFailed()
val a = div.selectFirstOrThrow("a")
val href = a.attrAsRelativeUrl("href")
val title = div.selectFirst(".item-title")?.text() ?: parseFailed("Title not found")
val title = div.selectFirstOrThrow(".item-title").text()
Manga(
id = generateUid(href),
title = title,

View File

@@ -1,8 +1,8 @@
package org.koitharu.kotatsu.parsers.site
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.exception.ParseException
import org.koitharu.kotatsu.parsers.model.*
@@ -13,7 +13,7 @@ import org.koitharu.kotatsu.parsers.util.json.mapJSONToSet
import java.util.*
@MangaSourceParser("DESUME", "Desu.me", "ru")
internal class DesuMeParser(override val context: MangaLoaderContext) : MangaParser(MangaSource.DESUME) {
internal class DesuMeParser(override val context: MangaLoaderContext) : PagedMangaParser(MangaSource.DESUME, 20) {
override val configKeyDomain = ConfigKey.Domain("desu.me", null)
@@ -24,13 +24,13 @@ internal class DesuMeParser(override val context: MangaLoaderContext) : MangaPar
SortOrder.ALPHABETICAL,
)
override suspend fun getList(
offset: Int,
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
if (query != null && offset != 0) {
if (query != null && page != searchPaginator.firstPage) {
return emptyList()
}
val domain = getDomain()
@@ -40,7 +40,7 @@ internal class DesuMeParser(override val context: MangaLoaderContext) : MangaPar
append("/manga/api/?limit=20&order=")
append(getSortKey(sortOrder))
append("&page=")
append((offset / 20) + 1)
append(page)
if (!tags.isNullOrEmpty()) {
append("&genres=")
appendAll(tags, ",") { it.key }
@@ -50,7 +50,8 @@ internal class DesuMeParser(override val context: MangaLoaderContext) : MangaPar
append(query)
}
}
val json = context.httpGet(url).parseJson().getJSONArray("response") ?: parseFailed("Invalid response")
val json = context.httpGet(url).parseJson().getJSONArray("response")
?: throw ParseException("Invalid response", url)
val total = json.length()
val list = ArrayList<Manga>(total)
for (i in 0 until total) {
@@ -83,7 +84,7 @@ internal class DesuMeParser(override val context: MangaLoaderContext) : MangaPar
override suspend fun getDetails(manga: Manga): Manga {
val url = manga.url.toAbsoluteUrl(getDomain())
val json = context.httpGet(url).parseJson().getJSONObject("response")
?: throw ParseException("Invalid response")
?: throw ParseException("Invalid response", url)
val baseChapterUrl = manga.url + "/chapter/"
val chaptersList = json.getJSONObject("chapters").getJSONArray("list")
val totalChapters = chaptersList.length()
@@ -119,7 +120,7 @@ internal class DesuMeParser(override val context: MangaLoaderContext) : MangaPar
val fullUrl = chapter.url.toAbsoluteUrl(getDomain())
val json = context.httpGet(fullUrl)
.parseJson()
.getJSONObject("response") ?: throw ParseException("Invalid response")
.getJSONObject("response") ?: throw ParseException("Invalid response", fullUrl)
return json.getJSONObject("pages").getJSONArray("list").mapJSON { jo ->
MangaPage(
id = generateUid(jo.getLong("id")),
@@ -133,17 +134,17 @@ internal class DesuMeParser(override val context: MangaLoaderContext) : MangaPar
override suspend fun getTags(): Set<MangaTag> {
val doc = context.httpGet("https://${getDomain()}/manga/").parseHtml()
val root = doc.body().getElementById("animeFilter")
?.selectFirst(".catalog-genres") ?: throw ParseException("Root not found")
val root = doc.body().requireElementById("animeFilter")
.selectFirstOrThrow(".catalog-genres")
return root.select("li").mapToSet {
val input = it.selectFirst("input") ?: parseFailed()
val input = it.selectFirstOrThrow("input")
MangaTag(
source = source,
key = input.attr("data-genre-slug").ifEmpty {
parseFailed("data-genre-slug is empty")
it.parseFailed("data-genre-slug is empty")
},
title = input.attr("data-genre-name").toTitleCase().ifEmpty {
parseFailed("data-genre-name is empty")
it.parseFailed("data-genre-name is empty")
},
)
}

View File

@@ -2,12 +2,11 @@ package org.koitharu.kotatsu.parsers.site
import org.jsoup.nodes.Element
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.MangaParserAuthProvider
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.exception.AuthRequiredException
import org.koitharu.kotatsu.parsers.exception.ParseException
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import java.util.*
@@ -19,7 +18,7 @@ private const val DOMAIN_AUTHORIZED = "exhentai.org"
@MangaSourceParser("EXHENTAI", "ExHentai")
internal class ExHentaiParser(
override val context: MangaLoaderContext,
) : MangaParser(MangaSource.EXHENTAI), MangaParserAuthProvider {
) : PagedMangaParser(MangaSource.EXHENTAI, pageSize = 25), MangaParserAuthProvider {
override val sortOrders: Set<SortOrder> = Collections.singleton(
SortOrder.NEWEST,
@@ -57,13 +56,12 @@ internal class ExHentaiParser(
context.cookieJar.insertCookies(DOMAIN_UNAUTHORIZED, "nw=1", "sl=dm_2")
}
override suspend fun getList(
offset: Int,
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val page = (offset / 25f).toIntUp()
var search = query?.urlEncoded().orEmpty()
val url = buildString {
append("https://")
@@ -95,19 +93,19 @@ internal class ExHentaiParser(
val root = body.selectFirst("table.itg")
?.selectFirst("tbody")
?: if (updateDm) {
parseFailed("Cannot find root")
body.parseFailed("Cannot find root")
} else {
updateDm = true
return getList(offset, query, tags, sortOrder)
return getListPage(page, query, tags, sortOrder)
}
updateDm = false
return root.children().mapNotNull { tr ->
if (tr.childrenSize() != 2) return@mapNotNull null
val (td1, td2) = tr.children()
val glink = td2.selectFirst("div.glink") ?: parseFailed("glink not found")
val a = glink.parents().select("a").first() ?: parseFailed("link not found")
val glink = td2.selectFirstOrThrow("div.glink")
val a = glink.parents().select("a").first() ?: glink.parseFailed("link not found")
val href = a.attrAsRelativeUrl("href")
val tagsDiv = glink.nextElementSibling() ?: parseFailed("tags div not found")
val tagsDiv = glink.nextElementSibling() ?: glink.parseFailed("tags div not found")
val mainTag = td2.selectFirst("div.cn")?.let { div ->
MangaTag(
title = div.text().toTitleCase(),
@@ -135,7 +133,7 @@ internal class ExHentaiParser(
override suspend fun getDetails(manga: Manga): Manga {
val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml()
val root = doc.body().selectFirst("div.gm") ?: parseFailed("Cannot find root")
val root = doc.body().selectFirstOrThrow("div.gm")
val cover = root.getElementById("gd1")?.children()?.first()
val title = root.getElementById("gd2")
val taglist = root.getElementById("taglist")
@@ -158,7 +156,7 @@ internal class ExHentaiParser(
a.text().toIntOrNull() != null
}?.let { a ->
val count = a.text().toInt()
val chapters = ArrayList<MangaChapter>(count)
val chapters = ChaptersListBuilder(count)
for (i in 1..count) {
val url = "${manga.url}?p=${i - 1}"
chapters += MangaChapter(
@@ -172,14 +170,14 @@ internal class ExHentaiParser(
branch = null,
)
}
chapters
chapters.toList()
},
)
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val doc = context.httpGet(chapter.url.toAbsoluteUrl(getDomain())).parseHtml()
val root = doc.body().getElementById("gdt") ?: parseFailed("Root not found")
val root = doc.body().requireElementById("gdt")
return root.select("a").map { a ->
val url = a.attrAsRelativeUrl("href")
MangaPage(
@@ -194,14 +192,12 @@ internal class ExHentaiParser(
override suspend fun getPageUrl(page: MangaPage): String {
val doc = context.httpGet(page.url.toAbsoluteUrl(getDomain())).parseHtml()
return doc.body().getElementById("img")?.absUrl("src")
?: parseFailed("Image not found")
return doc.body().requireElementById("img").attrAsAbsoluteUrl("src")
}
override suspend fun getTags(): Set<MangaTag> {
val doc = context.httpGet("https://${getDomain()}").parseHtml()
val root = doc.body().getElementById("searchbox")?.selectFirst("table")
?: parseFailed("Root not found")
val root = doc.body().requireElementById("searchbox").selectFirstOrThrow("table")
return root.select("div.cs").mapNotNullToSet { div ->
val id = div.id().substringAfterLast('_').toIntOrNull()
?: return@mapNotNullToSet null
@@ -222,7 +218,7 @@ internal class ExHentaiParser(
?: if (doc.getElementById("userlinksguest") != null) {
throw AuthRequiredException(source)
} else {
throw ParseException(null)
doc.parseFailed()
}
return username
}

View File

@@ -1,8 +1,8 @@
package org.koitharu.kotatsu.parsers.site
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
@@ -12,37 +12,40 @@ import java.util.*
private const val DEF_BRANCH_NAME = "Основний переклад"
@MangaSourceParser("MANGAINUA", "MANGA/in/UA", "uk")
class MangaInUaParser(override val context: MangaLoaderContext) : MangaParser(MangaSource.MANGAINUA) {
class MangaInUaParser(override val context: MangaLoaderContext) : PagedMangaParser(
source = MangaSource.MANGAINUA,
pageSize = 24,
searchPageSize = 10,
) {
override val sortOrders: Set<SortOrder>
get() = Collections.singleton(SortOrder.UPDATED)
override val configKeyDomain: ConfigKey.Domain = ConfigKey.Domain("manga.in.ua", null)
override suspend fun getList(
offset: Int,
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val page = (offset / 24f).toIntUp().inc()
val searchPage = (offset / 10f).toIntUp().inc()
val url = when {
!query.isNullOrEmpty() -> (
"/index.php?do=search" +
"&subaction=search" +
"&search_start=$searchPage" +
"&full_search=1" +
"&story=$query" +
"&titleonly=3"
).toAbsoluteUrl(getDomain())
"/index.php?do=search" +
"&subaction=search" +
"&search_start=$page" +
"&full_search=1" +
"&story=$query" +
"&titleonly=3"
).toAbsoluteUrl(getDomain())
tags.isNullOrEmpty() -> "/mangas/page/$page".toAbsoluteUrl(getDomain())
tags.size == 1 -> "${tags.first().key}/page/$page"
tags.size > 1 -> throw IllegalArgumentException("This source supports only 1 genre")
else -> "/mangas/page/$page".toAbsoluteUrl(getDomain())
}
val doc = context.httpGet(url).parseHtml()
val container = doc.body().getElementById("dle-content") ?: parseFailed("Container not found")
val container = doc.body().requireElementById("dle-content")
val items = container.select("div.col-6")
return items.mapNotNull { item ->
val href = item.selectFirst("a")?.attrAsRelativeUrl("href") ?: return@mapNotNull null
@@ -78,8 +81,7 @@ class MangaInUaParser(override val context: MangaLoaderContext) : MangaParser(Ma
override suspend fun getDetails(manga: Manga): Manga {
val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml()
val root =
doc.body().getElementById("dle-content") ?: parseFailed("Cannot find root")
val root = doc.body().requireElementById("dle-content")
val dateFormat = SimpleDateFormat("dd.MM.yyyy", Locale.US)
val chapterNodes = root.selectFirstOrThrow(".linkstocomics")
.select(".ltcitems")
@@ -90,16 +92,16 @@ class MangaInUaParser(override val context: MangaLoaderContext) : MangaParser(Ma
description = root.selectFirst("div.item__full-description")?.text(),
largeCoverUrl = root.selectFirst("div.item__full-sidebar--poster")?.selectFirst("img")
?.attrAsAbsoluteUrlOrNull("src"),
chapters = chapterNodes.mapNotNull { item ->
chapters = chapterNodes.mapChapters { _, item ->
val href = item?.selectFirst("a")?.attrAsRelativeUrlOrNull("href")
?: return@mapNotNull null
?: return@mapChapters null
val isAlternative = item.styleValueOrNull("background") != null
val name = item.selectFirst("a")?.text().orEmpty()
if (!isAlternative) i++
MangaChapter(
id = generateUid(href),
name = if (isAlternative) {
prevChapterName ?: return@mapNotNull null
prevChapterName ?: return@mapChapters null
} else {
prevChapterName = name
name
@@ -122,10 +124,9 @@ class MangaInUaParser(override val context: MangaLoaderContext) : MangaParser(Ma
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(getDomain())
val doc = context.httpGet(fullUrl).parseHtml()
val root =
doc.body().getElementById("comics")?.selectFirst("ul.xfieldimagegallery") ?: parseFailed("Root not found")
val root = doc.body().requireElementById("comics").selectFirstOrThrow("ul.xfieldimagegallery")
return root.select("li").map { ul ->
val img = ul.selectFirst("img") ?: parseFailed("Page image not found")
val img = ul.selectFirstOrThrow("img")
val url = img.attrAsAbsoluteUrl("data-src")
MangaPage(
id = generateUid(url),
@@ -140,8 +141,7 @@ class MangaInUaParser(override val context: MangaLoaderContext) : MangaParser(Ma
override suspend fun getTags(): Set<MangaTag> {
val domain = getDomain()
val doc = context.httpGet("https://$domain/mangas").parseHtml()
val root =
doc.body().getElementById("menu_1")?.selectFirst("div.menu__wrapper") ?: parseFailed("Cannot find root")
val root = doc.body().requireElementById("menu_1").selectFirstOrThrow("div.menu__wrapper")
return root.select("li").mapNotNullToSet { li ->
val a = li.selectFirst("a") ?: return@mapNotNullToSet null
MangaTag(

View File

@@ -4,7 +4,6 @@ import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.exception.ParseException
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import java.text.SimpleDateFormat
@@ -38,19 +37,21 @@ internal class MangaOwlParser(override val context: MangaLoaderContext) : MangaP
append("/search/$page?search=")
append(query.urlEncoded())
}
!tags.isNullOrEmpty() -> {
for (tag in tags) {
append(tag.key)
}
append("/$page?type=${getAlternativeSortKey(sortOrder)}")
}
else -> {
append("/${getSortKey(sortOrder)}/$page")
}
}
}
val doc = context.httpGet(link).parseHtml()
val slides = doc.body().select("ul.slides") ?: parseFailed("An error occurred while parsing")
val slides = doc.body().selectOrThrow("ul.slides")
val items = slides.select("div.col-md-2")
return items.mapNotNull { item ->
val href = item.selectFirst("h6 a")?.attrAsRelativeUrlOrNull("href") ?: return@mapNotNull null
@@ -78,12 +79,12 @@ internal class MangaOwlParser(override val context: MangaLoaderContext) : MangaP
override suspend fun getDetails(manga: Manga): Manga {
val doc = context.httpGet(manga.publicUrl).parseHtml()
val info = doc.body().selectFirst("div.single_detail") ?: parseFailed("An error occurred while parsing")
val table = doc.body().selectFirst("div.single-grid-right") ?: parseFailed("An error occurred while parsing")
val info = doc.body().selectFirstOrThrow("div.single_detail")
val table = doc.body().selectFirstOrThrow("div.single-grid-right")
val dateFormat = SimpleDateFormat("MM/dd/yyyy", Locale.US)
val trRegex = "window\\['tr'] = '([^']*)';".toRegex(RegexOption.IGNORE_CASE)
val trElement =
doc.getElementsByTag("script").find { trRegex.find(it.data()) != null } ?: parseFailed("Oops, tr not found")
val trElement = doc.getElementsByTag("script").find { trRegex.find(it.data()) != null }
?: doc.parseFailed("Oops, tr not found")
val tr = trRegex.find(trElement.data())!!.groups[1]!!.value
val s = context.encodeBase64(getDomain().toByteArray())
var isNsfw = manga.isNsfw
@@ -110,10 +111,10 @@ internal class MangaOwlParser(override val context: MangaLoaderContext) : MangaP
state = parseStatus(info.select("p.fexi_header_para:contains(status)").first()?.ownText()),
tags = manga.tags + parsedTags,
chapters = table.select("div.table.table-chapter-list").select("li.list-group-item.chapter_list")
.asReversed().mapIndexed { i, li ->
.asReversed().mapChapters { i, li ->
val a = li.select("a")
val href = a.attr("data-href").ifEmpty {
parseFailed("Link is missing")
li.parseFailed("Link is missing")
}
MangaChapter(
id = generateUid(href),
@@ -132,9 +133,9 @@ internal class MangaOwlParser(override val context: MangaLoaderContext) : MangaP
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(getDomain())
val doc = context.httpGet(fullUrl).parseHtml()
val root = doc.body().select("div.item img.owl-lazy") ?: throw ParseException("Root not found")
val root = doc.body().selectOrThrow("div.item img.owl-lazy")
return root.map { div ->
val url = div?.attrAsRelativeUrlOrNull("data-src") ?: parseFailed("Page image not found")
val url = div?.attrAsRelativeUrlOrNull("data-src") ?: doc.parseFailed("Page image not found")
MangaPage(
id = generateUid(url),
url = url,
@@ -156,7 +157,7 @@ internal class MangaOwlParser(override val context: MangaLoaderContext) : MangaP
val doc = context.httpGet("https://${getDomain()}/").parseHtml()
val root = doc.body().select("ul.dropdown-menu.multi-column.columns-3").select("li")
return root.mapToSet { p ->
val a = p.selectFirst("a") ?: parseFailed("a is null")
val a = p.selectFirstOrThrow("a")
MangaTag(
title = a.text().toTitleCase(),
key = a.attr("href"),

View File

@@ -4,7 +4,6 @@ import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.exception.ParseException
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import java.text.DateFormat
@@ -45,6 +44,7 @@ internal class MangaTownParser(override val context: MangaLoaderContext) : Manga
}
"/search?name=${query.urlEncoded()}".toAbsoluteUrl(getDomain())
}
tags.isNullOrEmpty() -> "/directory/$page.htm$sortKey".toAbsoluteUrl(getDomain())
tags.size == 1 -> "/directory/${tags.first().key}/$page.htm$sortKey".toAbsoluteUrl(getDomain())
else -> tags.joinToString(
@@ -54,8 +54,7 @@ internal class MangaTownParser(override val context: MangaLoaderContext) : Manga
}
}
val doc = context.httpGet(url).parseHtml()
val root = doc.body().selectFirst("ul.manga_pic_list")
?: throw ParseException("Root not found")
val root = doc.body().selectFirstOrThrow("ul.manga_pic_list")
return root.select("li").mapNotNull { li ->
val a = li.selectFirst("a.manga_cover")
val href = a?.attrAsRelativeUrlOrNull("href")
@@ -95,8 +94,8 @@ internal class MangaTownParser(override val context: MangaLoaderContext) : Manga
override suspend fun getDetails(manga: Manga): Manga {
val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml()
val root = doc.body().selectFirst("section.main")
?.selectFirst("div.article_content") ?: throw ParseException("Cannot find root")
val root = doc.body().selectFirstOrThrow("section.main")
.selectFirstOrThrow("div.article_content")
val info = root.selectFirst("div.detail_info")?.selectFirst("ul")
val chaptersList = root.selectFirst("div.chapter_content")
?.selectFirst("ul.chapter_list")?.select("li")?.asReversed()
@@ -112,9 +111,9 @@ internal class MangaTownParser(override val context: MangaLoaderContext) : Manga
)
}.orEmpty(),
description = info?.getElementById("show")?.ownText(),
chapters = chaptersList?.mapIndexedNotNull { i, li ->
chapters = chaptersList?.mapChapters { i, li ->
val href = li.selectFirst("a")?.attrAsRelativeUrlOrNull("href")
?: return@mapIndexedNotNull null
?: return@mapChapters null
val name = li.select("span")
.filter { x -> x.className().isEmpty() }
.joinToString(" - ") { it.text() }.trim()
@@ -138,9 +137,8 @@ internal class MangaTownParser(override val context: MangaLoaderContext) : Manga
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(getDomain())
val doc = context.httpGet(fullUrl).parseHtml()
val root = doc.body().selectFirst("div.page_select")
?: throw ParseException("Cannot find root")
return root.selectFirst("select")?.select("option")?.mapNotNull {
val root = doc.body().selectFirstOrThrow("div.page_select")
return root.selectFirstOrThrow("select").selectOrThrow("option").mapNotNull {
val href = it.attrAsRelativeUrlOrNull("value")
if (href == null || href.endsWith("featured.html")) {
return@mapNotNull null
@@ -152,12 +150,12 @@ internal class MangaTownParser(override val context: MangaLoaderContext) : Manga
referer = fullUrl,
source = MangaSource.MANGATOWN,
)
} ?: parseFailed("Pages list not found")
}
}
override suspend fun getPageUrl(page: MangaPage): String {
val doc = context.httpGet(page.url.toAbsoluteUrl(getDomain())).parseHtml()
return doc.getElementById("image")?.absUrl("src") ?: parseFailed("Image not found")
return doc.requireElementById("image").attrAsAbsoluteUrl("src")
}
override suspend fun getTags(): Set<MangaTag> {
@@ -165,7 +163,7 @@ internal class MangaTownParser(override val context: MangaLoaderContext) : Manga
val root = doc.body().selectFirst("aside.right")
?.getElementsContainingOwnText("Genres")
?.first()
?.nextElementSibling() ?: parseFailed("Root not found")
?.nextElementSibling() ?: doc.parseFailed("Root not found")
return root.select("li").mapNotNullToSet { li ->
val a = li.selectFirst("a") ?: return@mapNotNullToSet null
val key = a.attr("href").parseTagKey()

View File

@@ -6,8 +6,8 @@ import kotlinx.coroutines.awaitAll
import kotlinx.coroutines.coroutineScope
import org.jsoup.nodes.Element
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
@@ -15,7 +15,7 @@ import java.text.SimpleDateFormat
import java.util.*
@MangaSourceParser("NHENTAI", "N-Hentai")
class NHentaiParser(override val context: MangaLoaderContext) : MangaParser(MangaSource.NHENTAI) {
class NHentaiParser(override val context: MangaLoaderContext) : PagedMangaParser(MangaSource.NHENTAI, pageSize = 25) {
override val configKeyDomain: ConfigKey.Domain
get() = ConfigKey.Domain("nhentai.net", null)
@@ -23,17 +23,16 @@ class NHentaiParser(override val context: MangaLoaderContext) : MangaParser(Mang
override val sortOrders: Set<SortOrder>
get() = EnumSet.of(SortOrder.NEWEST, SortOrder.POPULARITY)
override suspend fun getList(
offset: Int,
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
if (query.isNullOrEmpty() && tags != null && tags.size > 1) {
return getList(offset, buildQuery(tags), emptySet(), sortOrder)
return getListPage(page, buildQuery(tags), emptySet(), sortOrder)
}
val domain = getDomain()
val page = (offset / 25) + 1
val url = buildString {
append("https://")
append(domain)
@@ -67,8 +66,8 @@ class NHentaiParser(override val context: MangaLoaderContext) : MangaParser(Mang
}
}
}
val root = context.httpGet(url).parseHtml().body().getElementById("content")
?.selectLast("div.index-container") ?: parseFailed("Root not found")
val root = context.httpGet(url).parseHtml().body().requireElementById("content")
.selectLastOrThrow("div.index-container")
val regexBrackets = Regex("\\[[^]]+]|\\([^)]+\\)")
val regexSpaces = Regex("\\s+")
return root.select(".gallery").map { div ->
@@ -101,7 +100,7 @@ class NHentaiParser(override val context: MangaLoaderContext) : MangaParser(Mang
override suspend fun getDetails(manga: Manga): Manga {
val root = context.httpGet(
url = manga.url.toAbsoluteUrl(getDomain())
url = manga.url.toAbsoluteUrl(getDomain()),
).parseHtml().body().requireElementById("bigcontainer")
val img = root.requireElementById("cover").selectFirstOrThrow("img")
val tagContainers = root.requireElementById("tags").select(".tag-container")
@@ -126,12 +125,12 @@ class NHentaiParser(override val context: MangaLoaderContext) : MangaParser(Mang
uploadDate = dateFormat.tryParse(
tagContainers.find { x -> x.ownText() == "Uploaded:" }
?.selectFirst("time")
?.attr("datetime")
?.attr("datetime"),
),
branch = null,
source = source,
)
)
),
),
)
}

View File

@@ -47,15 +47,18 @@ class NicovideoSeigaParser(override val context: MangaLoaderContext) :
val url = when {
!query.isNullOrEmpty() -> return if (offset == 0) getSearchList(query, page) else emptyList()
tags.isNullOrEmpty() -> "https://$domain/manga/list?page=$page&sort=${getSortKey(sortOrder)}"
tags.size == 1 -> "https://$domain/manga/list?category=${tags.first().key}&page=$page&sort=${getSortKey(sortOrder)}"
tags.size == 1 -> "https://$domain/manga/list?category=${tags.first().key}&page=$page" +
"&sort=${getSortKey(sortOrder)}"
tags.size > 1 -> throw IllegalArgumentException("This source supports only 1 category")
else -> "https://$domain/manga/list?page=$page&sort=${getSortKey(sortOrder)}"
}
val doc = context.httpGet(url).parseHtml()
val comicList = doc.body().select("#comic_list > ul > li") ?: parseFailed("Container not found")
val comicList = doc.body().select("#comic_list > ul > li") ?: doc.parseFailed("Container not found")
val items = comicList.select("div > .description > div > div")
return items.mapNotNull { item ->
val href = item.selectFirst(".comic_icon > div > a")?.attrAsRelativeUrlOrNull("href") ?: return@mapNotNull null
val href =
item.selectFirst(".comic_icon > div > a")?.attrAsRelativeUrlOrNull("href") ?: return@mapNotNull null
val statusText = item.selectFirst(".mg_description_header > .mg_icon > .content_status > span")?.text()
Manga(
id = generateUid(href),
@@ -86,21 +89,23 @@ class NicovideoSeigaParser(override val context: MangaLoaderContext) :
override suspend fun getDetails(manga: Manga): Manga {
val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain("seiga"))).parseHtml()
val contents = doc.body().selectFirst("#contents") ?: parseFailed("Cannot find root")
val contents = doc.body().selectFirstOrThrow("#contents")
val statusText = contents
.select("div.mg_work_detail > div > div:nth-child(2) > div.tip.content_status.status_series > span")
.text()
return manga.copy(
description = contents.selectFirst("div.mg_work_detail > div > div.row > div.description_text")?.html(),
largeCoverUrl = contents.selectFirst("div.primaries > div.main_visual > a > img")?.attrAsAbsoluteUrlOrNull("src"),
largeCoverUrl = contents.selectFirst("div.primaries > div.main_visual > a > img")
?.attrAsAbsoluteUrlOrNull("src"),
state = when (statusText) {
STATUS_ONGOING -> MangaState.ONGOING
STATUS_FINISHED -> MangaState.FINISHED
else -> null
},
isNsfw = contents.select(".icon_adult").isNotEmpty(),
chapters = contents.select("#episode_list > ul > li").mapIndexedNotNull { i, li ->
val href = li.selectFirst("div > div.description > div.title > a")?.attrAsRelativeUrl("href") ?: parseFailed()
chapters = contents.select("#episode_list > ul > li").mapChapters { i, li ->
val href = li.selectFirst("div > div.description > div.title > a")
?.attrAsRelativeUrl("href") ?: li.parseFailed()
MangaChapter(
id = generateUid(href),
name = li.select("div > div.description > div.title > a").text(),
@@ -135,9 +140,9 @@ class NicovideoSeigaParser(override val context: MangaLoaderContext) :
override suspend fun getTags(): Set<MangaTag> {
val doc = context.httpGet("https://${getDomain("seiga")}/manga/list").parseHtml()
val root = doc.body().select("#mg_category_list > ul > li") ?: parseFailed("Cannot find tags")
val root = doc.body().selectOrThrow("#mg_category_list > ul > li")
return root.mapToSet { li ->
val a = li.selectFirst("a") ?: parseFailed("a is null")
val a = li.selectFirstOrThrow("a")
MangaTag(
title = a.text(),
key = a.attrAsRelativeUrlOrNull("href").orEmpty(),
@@ -151,7 +156,8 @@ class NicovideoSeigaParser(override val context: MangaLoaderContext) :
val doc = context.httpGet("https://$domain/manga/search/?q=$query&page=$page&sort=score").parseHtml()
val root = doc.body().select(".search_result__item")
return root.mapNotNull { item ->
val href = item.selectFirst(".search_result__item__thumbnail > a")?.attrAsRelativeUrl("href") ?: parseFailed()
val href = item.selectFirst(".search_result__item__thumbnail > a")
?.attrAsRelativeUrl("href") ?: doc.parseFailed()
Manga(
id = generateUid(href),
url = href,

View File

@@ -3,22 +3,19 @@ package org.koitharu.kotatsu.parsers.site
import okhttp3.Headers
import okhttp3.HttpUrl.Companion.toHttpUrl
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.exception.ParseException
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import java.text.SimpleDateFormat
import java.util.*
private const val PAGE_SIZE = 26
internal abstract class NineMangaParser(
final override val context: MangaLoaderContext,
source: MangaSource,
defaultDomain: String,
) : MangaParser(source) {
) : PagedMangaParser(source, pageSize = 26) {
override val configKeyDomain = ConfigKey.Domain(defaultDomain, null)
@@ -26,7 +23,7 @@ internal abstract class NineMangaParser(
context.cookieJar.insertCookies(getDomain(), "ninemanga_template_desk=yes")
}
private val headers = Headers.Builder()
override val headers = Headers.Builder()
.add("Accept-Language", "en-US;q=0.7,en;q=0.3")
.build()
@@ -34,13 +31,12 @@ internal abstract class NineMangaParser(
SortOrder.POPULARITY,
)
override suspend fun getList(
offset: Int,
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val page = (offset / PAGE_SIZE.toFloat()).toIntUp() + 1
val url = buildString {
append("https://")
append(getDomain())
@@ -50,6 +46,7 @@ internal abstract class NineMangaParser(
append(query.urlEncoded())
append("&page=")
}
!tags.isNullOrEmpty() -> {
append("/search/?category_id=")
for (tag in tags) {
@@ -58,6 +55,7 @@ internal abstract class NineMangaParser(
}
append("&page=")
}
else -> {
append("/category/index_")
}
@@ -67,11 +65,11 @@ internal abstract class NineMangaParser(
}
val doc = context.httpGet(url, headers).parseHtml()
val root = doc.body().selectFirst("ul.direlist")
?: throw ParseException("Cannot find root")
?: doc.parseFailed("Cannot find root")
val baseHost = root.baseUri().toHttpUrl().host
return root.select("li").map { node ->
val href = node.selectFirst("a")?.absUrl("href")
?: parseFailed("Link not found")
?: node.parseFailed("Link not found")
val relUrl = href.toRelativeUrl(baseHost)
val dd = node.selectFirst("dd")
Manga(
@@ -97,10 +95,8 @@ internal abstract class NineMangaParser(
manga.url.toAbsoluteUrl(getDomain()) + "?waring=1",
headers,
).parseHtml()
val root = doc.body().selectFirst("div.manga")
?: throw ParseException("Cannot find root")
val infoRoot = root.selectFirst("div.bookintro")
?: throw ParseException("Cannot find info")
val root = doc.body().selectFirstOrThrow("div.manga")
val infoRoot = root.selectFirstOrThrow("div.bookintro")
return manga.copy(
tags = infoRoot.getElementsByAttributeValue("itemprop", "genre").first()
?.select("a")?.mapToSet { a ->
@@ -115,10 +111,10 @@ internal abstract class NineMangaParser(
description = infoRoot.getElementsByAttributeValue("itemprop", "description").first()
?.html()?.substringAfter("</b>"),
chapters = root.selectFirst("div.chapterbox")?.select("ul.sub_vol_ul > li")
?.asReversed()?.mapIndexed { i, li ->
?.asReversed()?.mapChapters { i, li ->
val a = li.selectFirst("a.chapter_list_a")
val href = a?.attrAsRelativeUrlOrNull("href")
?.replace("%20", " ") ?: parseFailed("Link not found")
?.replace("%20", " ") ?: li.parseFailed("Link not found")
MangaChapter(
id = generateUid(href),
name = a.text(),
@@ -144,14 +140,14 @@ internal abstract class NineMangaParser(
preview = null,
source = source,
)
} ?: throw ParseException("Pages list not found at ${chapter.url}")
} ?: doc.parseFailed("Pages list not found")
}
override suspend fun getPageUrl(page: MangaPage): String {
val doc = context.httpGet(page.url.toAbsoluteUrl(getDomain()), headers).parseHtml()
val root = doc.body()
return root.selectFirst("a.pic_download")?.absUrl("href")
?: throw ParseException("Page image not found")
?: doc.parseFailed("Page image not found")
}
override suspend fun getTags(): Set<MangaTag> {
@@ -166,7 +162,7 @@ internal abstract class NineMangaParser(
key = cateId,
source = source,
)
} ?: parseFailed("Root not found")
} ?: doc.parseFailed("Root not found")
}
private fun parseStatus(status: String) = when {

View File

@@ -6,7 +6,6 @@ import org.koitharu.kotatsu.parsers.MangaParserAuthProvider
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.exception.AuthRequiredException
import org.koitharu.kotatsu.parsers.exception.ParseException
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import java.text.SimpleDateFormat
@@ -59,12 +58,13 @@ internal class NudeMoonParser(
postfix = "&rowstart=$offset",
transform = { it.key.urlEncoded() },
)
else -> "https://$domain/all_manga?${getSortKey(sortOrder)}&rowstart=$offset"
}
val doc = context.httpGet(url).parseHtml()
val root = doc.body().run {
selectFirst("td.main-bg") ?: selectFirst("td.main-body")
} ?: parseFailed("Cannot find root")
} ?: doc.parseFailed("Cannot find root")
return root.select("table.news_pic2").mapNotNull { row ->
val a = row.selectFirst("td.bg_style1")?.selectFirst("a")
?: return@mapNotNull null
@@ -102,7 +102,7 @@ internal class NudeMoonParser(
override suspend fun getDetails(manga: Manga): Manga {
val body = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml().body()
val root = body.selectFirst("table.shoutbox")
?: parseFailed("Cannot find root")
?: body.parseFailed("Cannot find root")
val info = root.select("div.tbl2")
val lastInfo = info.last()
return manga.copy(
@@ -143,7 +143,7 @@ internal class NudeMoonParser(
val script = doc.select("script").firstNotNullOfOrNull {
it.html().takeIf { x -> x.contains(" images = new ") }
} ?: if (isAuthorized) {
parseFailed("Cannot find pages list")
doc.parseFailed("Cannot find pages list")
} else {
throw AuthRequiredException(source)
}
@@ -174,7 +174,7 @@ internal class NudeMoonParser(
val root = doc.body().getElementsContainingOwnText("Поиск манги по тегам")
.firstOrNull()?.parents()?.find { it.tag().normalName() == "tbody" }
?.selectFirst("td.textbox")?.selectFirst("td.small")
?: parseFailed("Tags root not found")
?: doc.parseFailed("Tags root not found")
return root.select("a").mapToSet {
MangaTag(
title = it.text().toTitleCase(),
@@ -197,7 +197,7 @@ internal class NudeMoonParser(
throw if (body.selectFirst("form[name=\"loginform\"]") != null) {
AuthRequiredException(source)
} else {
ParseException("Cannot find username")
body.parseFailed("Cannot find username")
}
}
}

View File

@@ -6,11 +6,12 @@ import org.json.JSONArray
import org.json.JSONException
import org.json.JSONObject
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.MangaParserAuthProvider
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.exception.AuthRequiredException
import org.koitharu.kotatsu.parsers.exception.ContentUnavailableException
import org.koitharu.kotatsu.parsers.exception.ParseException
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
@@ -31,7 +32,7 @@ private const val STATUS_FINISHED = 0
@MangaSourceParser("REMANGA", "Remanga", "ru")
internal class RemangaParser(
override val context: MangaLoaderContext,
) : MangaParser(MangaSource.REMANGA), MangaParserAuthProvider {
) : PagedMangaParser(MangaSource.REMANGA, PAGE_SIZE), MangaParserAuthProvider {
override val configKeyDomain = ConfigKey.Domain("remanga.org", null)
override val authUrl: String
@@ -53,8 +54,8 @@ internal class RemangaParser(
private val regexLastUrlPath = Regex("/[^/]+/?$")
override suspend fun getList(
offset: Int,
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
@@ -77,7 +78,7 @@ internal class RemangaParser(
}
urlBuilder
.append("&page=")
.append((offset / PAGE_SIZE) + 1)
.append(page)
.append("&count=")
.append(PAGE_SIZE)
val content = context.httpGet(urlBuilder.toString(), getApiHeaders()).parseJson()
@@ -113,18 +114,18 @@ internal class RemangaParser(
copyCookies()
val domain = getDomain()
val slug = manga.url.find(regexLastUrlPath)
?: throw ParseException("Cannot obtain slug from ${manga.url}")
?: throw ParseException("Cannot obtain slug from ${manga.url}", manga.publicUrl)
val data = context.httpGet(
url = "https://api.$domain/api/titles/$slug/",
url = "https://api.$domain/api/titles$slug/",
headers = getApiHeaders(),
).handle401().parseJson()
val content = try {
data.getJSONObject("content")
} catch (e: JSONException) {
throw ParseException(data.optString("msg"), e)
throw ParseException(data.optString("msg"), manga.publicUrl, e)
}
val branchId = content.getJSONArray("branches").optJSONObject(0)
?.getLong("id") ?: throw ParseException("No branches found")
?.getLong("id") ?: throw ParseException("No branches found", manga.publicUrl)
val chapters = grabChapters(domain, branchId)
val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.US)
return manga.copy(
@@ -141,7 +142,7 @@ internal class RemangaParser(
source = MangaSource.REMANGA,
)
},
chapters = chapters.mapIndexed { i, jo ->
chapters = chapters.mapChapters { i, jo ->
val id = jo.getLong("id")
val name = jo.getString("name").toTitleCase(Locale.ROOT)
val publishers = jo.optJSONArray("publishers")
@@ -182,9 +183,9 @@ internal class RemangaParser(
}
if (pubDate != null && pubDate > System.currentTimeMillis()) {
val at = SimpleDateFormat.getDateInstance(DateFormat.LONG).format(Date(pubDate))
parseFailed("Глава станет доступной $at")
throw ContentUnavailableException("Глава станет доступной $at")
} else {
parseFailed("Глава недоступна")
throw ContentUnavailableException("Глава недоступна")
}
}
val result = ArrayList<MangaPage>(pages.length())
@@ -192,7 +193,7 @@ internal class RemangaParser(
when (val item = pages.get(i)) {
is JSONObject -> result += parsePage(item, referer)
is JSONArray -> item.mapJSONTo(result) { parsePage(it, referer) }
else -> throw ParseException("Unknown json item $item")
else -> throw ParseException("Unknown json item $item", chapter.url)
}
}
return result

View File

@@ -0,0 +1,116 @@
package org.koitharu.kotatsu.parsers.site
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import java.text.SimpleDateFormat
import java.util.*
@MangaSourceParser("TUMANGAONLINE", "TuMangaOnline", "es")
class TuMangaOnlineParser(override val context: MangaLoaderContext) : PagedMangaParser(
source = MangaSource.TUMANGAONLINE,
pageSize = 24,
) {
override val configKeyDomain = ConfigKey.Domain("lectortmo.com", null)
override val sortOrders: EnumSet<SortOrder> = EnumSet.of(
SortOrder.POPULARITY,
SortOrder.ALPHABETICAL,
SortOrder.RATING,
SortOrder.NEWEST,
SortOrder.UPDATED,
)
init {
paginator.firstPage = 1
searchPaginator.firstPage = 0
}
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val url = (
"/library" +
"?order_item=likes_count" +
"&order_dir=desc" +
"&filter_by=title" +
"&_pg=1" +
"&page=$page"
).toAbsoluteUrl(getDomain())
val doc = context.httpGet(url).parseHtml()
val items = doc.body().select("div.element")
return items.mapNotNull { item ->
val href = item.selectFirst("a")?.attrAsRelativeUrl("href")?.substringAfter(" ") ?: return@mapNotNull null
Manga(
id = generateUid(href),
title = item.selectFirst("h4.text-truncate")?.text() ?: return@mapNotNull null,
coverUrl = item.select("style").toString().substringAfter("('").substringBeforeLast("')"),
altTitle = null,
author = null,
rating = item.selectFirst("span.score")
?.text()
?.toFloatOrNull()
?.div(10F) ?: RATING_UNKNOWN,
url = href,
isNsfw = item.select("i").hasClass("fas fa-heartbeat fa-2x"),
tags = emptySet(),
state = null,
publicUrl = href.toAbsoluteUrl(doc.host ?: getDomain()),
source = source,
)
}
}
override suspend fun getDetails(manga: Manga): Manga {
val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml()
val contents = doc.body().selectFirstOrThrow("section.element-header-content")
return manga.copy(
description = contents.selectFirst("p.element-description")?.html(),
largeCoverUrl = contents.selectFirst(".book-thumbnail")?.attrAsAbsoluteUrlOrNull("src"),
state = parseStatus(contents.select("span.book-status").text().orEmpty()),
author = contents.selectFirst("h5.card-title")?.attr("title")?.substringAfter(", "),
chapters = doc.body().select("div.chapters > ul.list-group li.p-0.list-group-item").asReversed()
.mapChapters { i, li ->
val href = li.selectFirst("div.row > .text-right > a")
?.attrAsRelativeUrl("href") ?: li.parseFailed()
MangaChapter(
id = generateUid(href),
name = li.select("div.col-10.text-truncate").text().substringAfter(": ").trim(),
number = i + 1,
url = href,
scanlator = li.select("div.col-md-6.text-truncate").text(),
branch = null,
uploadDate = li.select("span.badge.badge-primary.p-2").first()?.text()
?.let { parseChapterDate(it) }
?: 0,
source = source,
)
},
)
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
TODO("Not yet implemented")
}
override suspend fun getTags(): Set<MangaTag> {
TODO("Not yet implemented")
}
private fun parseStatus(status: String) = when {
status.contains("Publicándose") -> MangaState.ONGOING
status.contains("Finalizado") -> MangaState.FINISHED
else -> null
}
private fun parseChapterDate(date: String): Long =
SimpleDateFormat("yyyy-MM-dd", Locale.getDefault()).parse(date)?.time ?: 0
}

View File

@@ -0,0 +1,177 @@
package org.koitharu.kotatsu.parsers.site
import org.jsoup.nodes.Element
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import org.koitharu.kotatsu.parsers.util.json.getStringOrNull
import org.koitharu.kotatsu.parsers.util.json.mapJSON
import java.text.SimpleDateFormat
import java.util.*
@MangaSourceParser("UNION_MANGAS", "Union Mangás", "pt")
class UnionMangasParser(override val context: MangaLoaderContext) : PagedMangaParser(MangaSource.UNION_MANGAS, 40) {
override val sortOrders = EnumSet.of(
SortOrder.ALPHABETICAL,
SortOrder.POPULARITY,
)
override val configKeyDomain = ConfigKey.Domain("unionleitor.top", emptyArray())
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
if (!query.isNullOrEmpty()) {
return if (page == searchPaginator.firstPage) {
search(query)
} else {
emptyList()
}
}
val tag = tags.oneOrThrowIfMany()
val url = urlBuilder()
.addPathSegment("lista-mangas")
.addPathSegment(
when {
tag != null -> tag.key
sortOrder == SortOrder.ALPHABETICAL -> "a-z"
else -> "visualizacoes"
},
).addPathSegment(page.toString())
val doc = context.httpGet(url.build()).parseHtml()
val root = doc.selectFirstOrThrow("div.tamanho-bloco-perfil")
return root.select(".lista-mangas-novos").map { div ->
val a = div.selectFirstOrThrow("a")
val img = div.selectFirstOrThrow("img")
val href = a.attrAsRelativeUrl("href")
Manga(
id = generateUid(href),
url = href,
publicUrl = a.attrAsAbsoluteUrl("href"),
title = div.selectLastOrThrow("a").text(),
coverUrl = img.attrAsAbsoluteUrl("src"),
altTitle = null,
rating = RATING_UNKNOWN,
tags = emptySet(),
description = div.selectLast("div")?.ownText(),
state = null,
author = null,
isNsfw = false,
source = source,
)
}
}
override suspend fun getDetails(manga: Manga): Manga {
val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml()
val root = doc.selectFirstOrThrow(".perfil-manga")
val dateFormat = SimpleDateFormat("dd/MM/yyyy", Locale.ROOT)
return manga.copy(
rating = root.select("h2")
.find { it.ownText().startsWith('#') }
?.ownText()?.drop(1)?.toFloatOrNull()?.div(10f) ?: manga.rating,
largeCoverUrl = root.selectFirst("img.img-thumbnail")?.attrAsAbsoluteUrlOrNull("src"),
description = root.selectFirst(".panel-default")?.selectFirst(".panel-body")?.html(),
author = root.tableValue("Autor")?.ownText(),
altTitle = root.tableValue("Título(s) Alternativo(s)")?.ownText(),
state = when (root.tableValue("Status")?.selectLast(".label")?.text()) {
"Completo" -> MangaState.FINISHED
"Ativo" -> MangaState.ONGOING
else -> null
},
tags = root.tableValue("Gênero(s)")?.select("a")?.mapToSet {
it.toMangaTag()
} ?: manga.tags,
isNsfw = root.selectFirst(".alert-danger")?.html()?.contains("18 anos") == true,
chapters = root.select("div.row.capitulos").asReversed().mapChapters { i, div ->
val a = div.selectFirstOrThrow("a")
val href = a.attrAsRelativeUrl("href")
val title = a.text()
MangaChapter(
id = generateUid(href),
name = title,
number = i + 1,
url = href,
scanlator = div.selectLast("a")?.text()?.takeUnless { it == title },
uploadDate = dateFormat.tryParse(
a.nextElementSibling()?.text()?.removeSurrounding('(', ')'),
),
branch = null,
source = source,
)
},
)
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(getDomain())
val doc = context.httpGet(fullUrl).parseHtml()
val root = doc.body().selectFirstOrThrow("article")
return root.selectOrThrow("img[pag]").mapNotNull { img ->
val href = img.attrAsRelativeUrl("src")
if (href.startsWith("/images/banner")) {
return@mapNotNull null
}
MangaPage(
id = generateUid(href),
url = href,
referer = fullUrl,
preview = null,
source = source,
)
}
}
override suspend fun getTags(): Set<MangaTag> {
val doc = context.httpGet(urlBuilder().addPathSegment("lista-mangas").build()).parseHtml()
val ul = doc.body().selectFirstOrThrow(".nav-tabs").selectFirstOrThrow("ul.dropdown-menu")
return ul.select("li").mapToSet { li ->
li.selectFirstOrThrow("a").toMangaTag()
}
}
private suspend fun search(query: String): List<Manga> {
val domain = getDomain()
val json = context.httpGet(
urlBuilder()
.addPathSegments("assets/busca.php")
.addQueryParameter("nomeManga", query)
.build(),
).parseJson()
return json.getJSONArray("items").mapJSON { jo ->
val href = "/pagina-manga/" + jo.getString("url")
Manga(
id = generateUid(href),
url = href,
publicUrl = href.toAbsoluteUrl(domain),
title = jo.getString("titulo"),
rating = RATING_UNKNOWN,
tags = emptySet(),
author = jo.getStringOrNull("autor"),
coverUrl = jo.getString("imagem"),
state = null,
isNsfw = false,
altTitle = null,
source = source,
)
}
}
private fun Element.tableValue(title: String): Element? {
return select("h4.media-heading")
.find { it.selectFirst("label.subtit-manga")?.text()?.contains(title, ignoreCase = true) == true }
}
private fun Element.toMangaTag() = MangaTag(
title = text().toTitleCase(sourceLocale ?: Locale.ROOT),
key = attr("href").removeSuffix('/').substringAfterLast('/'),
source = source,
)
}

View File

@@ -1,10 +1,13 @@
package org.koitharu.kotatsu.parsers.site
package org.koitharu.kotatsu.parsers.site.grouple
import okhttp3.Headers
import okhttp3.HttpUrl.Companion.toHttpUrl
import okhttp3.Response
import org.json.JSONArray
import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.MangaParserAuthProvider
import org.koitharu.kotatsu.parsers.exception.AuthRequiredException
import org.koitharu.kotatsu.parsers.exception.ParseException
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import org.koitharu.kotatsu.parsers.util.json.mapJSON
@@ -16,9 +19,13 @@ private const val PAGE_SIZE_SEARCH = 50
private const val NSFW_ALERT = "сексуальные сцены"
private const val NOTHING_FOUND = "Ничего не найдено"
internal abstract class GroupleParser(source: MangaSource, userAgent: String) : MangaParser(source) {
internal abstract class GroupleParser(
source: MangaSource,
userAgent: String,
private val siteId: Int,
) : MangaParser(source), MangaParserAuthProvider {
private val headers = Headers.Builder()
override val headers = Headers.Builder()
.add("User-Agent", userAgent)
.build()
@@ -29,6 +36,15 @@ internal abstract class GroupleParser(source: MangaSource, userAgent: String) :
SortOrder.RATING,
)
override val authUrl: String
get() {
val targetUri = "https://${getDomain()}/".urlEncoded()
return "https://grouple.co/internal/auth/sso?siteId=$siteId&=targetUri=$targetUri"
}
override val isAuthorized: Boolean
get() = context.cookieJar.getCookies(getDomain()).any { it.name == "gwt" }
override suspend fun getList(
offset: Int,
query: String?,
@@ -45,29 +61,32 @@ internal abstract class GroupleParser(source: MangaSource, userAgent: String) :
),
headers,
)
tags.isNullOrEmpty() -> context.httpGet(
"https://$domain/list?sortType=${
getSortKey(sortOrder)
getSortKey(sortOrder)
}&offset=${offset upBy PAGE_SIZE}",
headers,
)
tags.size == 1 -> context.httpGet(
"https://$domain/list/genre/${tags.first().key}?sortType=${
getSortKey(sortOrder)
getSortKey(sortOrder)
}&offset=${offset upBy PAGE_SIZE}",
headers,
)
offset > 0 -> return emptyList()
else -> advancedSearch(domain, tags)
}.parseHtml().body()
val root = (doc.getElementById("mangaBox") ?: doc.getElementById("mangaResults"))
?: parseFailed("Cannot find root")
?: doc.parseFailed("Cannot find root")
val tiles = root.selectFirst("div.tiles.row") ?: if (
root.select(".alert").any { it.ownText() == NOTHING_FOUND }
) {
return emptyList()
} else {
parseFailed("No tiles found")
doc.parseFailed("No tiles found")
}
val baseHost = root.baseUri().toHttpUrl().host
return tiles.select("div.tile").mapNotNull { node ->
@@ -113,6 +132,7 @@ internal abstract class GroupleParser(source: MangaSource, userAgent: String) :
state = when {
node.selectFirst("div.tags")
?.selectFirst("span.mangaCompleted") != null -> MangaState.FINISHED
else -> null
},
source = source,
@@ -123,7 +143,7 @@ internal abstract class GroupleParser(source: MangaSource, userAgent: String) :
override suspend fun getDetails(manga: Manga): Manga {
val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain()), headers).parseHtml()
val root = doc.body().getElementById("mangaBox")?.selectFirst("div.leftContent")
?: parseFailed("Cannot find root")
?: doc.parseFailed("Cannot find root")
val dateFormat = SimpleDateFormat("dd.MM.yy", Locale.US)
val coverImg = root.selectFirst("div.subject-cover")?.selectFirst("img")
return manga.copy(
@@ -139,10 +159,11 @@ internal abstract class GroupleParser(source: MangaSource, userAgent: String) :
source = source,
)
},
author = root.selectFirst("a.person-link")?.text() ?: manga.author,
isNsfw = root.select(".alert-warning").any { it.ownText().contains(NSFW_ALERT) },
chapters = root.selectFirst("div.chapters-link")?.selectFirst("table")
?.select("tr:has(td > a)")?.asReversed()?.mapIndexedNotNull { i, tr ->
val a = tr.selectFirst("a") ?: return@mapIndexedNotNull null
?.select("tr:has(td > a)")?.asReversed()?.mapChapters { i, tr ->
val a = tr.selectFirst("a") ?: return@mapChapters null
val href = a.attrAsRelativeUrl("href")
var translators = ""
val translatorElement = a.attr("title")
@@ -198,7 +219,7 @@ internal abstract class GroupleParser(source: MangaSource, userAgent: String) :
)
}
}
parseFailed("Pages list not found at ${chapter.url}")
doc.parseFailed("Pages list not found at ${chapter.url}")
}
override suspend fun getPageUrl(page: MangaPage): String {
@@ -212,14 +233,14 @@ internal abstract class GroupleParser(source: MangaSource, userAgent: String) :
return url
}
}
val fallbackServer = servers.firstOrNull() ?: parseFailed("Cannot find any page url")
val fallbackServer = servers.firstOrNull() ?: throw ParseException("Cannot find any page url", page.url)
return fallbackServer + path
}
override suspend fun getTags(): Set<MangaTag> {
val doc = context.httpGet("https://${getDomain()}/list/genres/sort_name", headers).parseHtml()
val root = doc.body().getElementById("mangaBox")?.selectFirst("div.leftContent")
?.selectFirst("table.table") ?: parseFailed("Cannot find root")
?.selectFirst("table.table") ?: doc.parseFailed("Cannot find root")
return root.select("a.element-link").mapToSet { a ->
MangaTag(
title = a.text().toTitleCase(),
@@ -229,6 +250,15 @@ internal abstract class GroupleParser(source: MangaSource, userAgent: String) :
}
}
override suspend fun getUsername(): String {
val root = context.httpGet("https://grouple.co/").parseHtml().body()
val element = root.selectFirst("img.user-avatar") ?: throw AuthRequiredException(source)
val res = element.parent()?.text()
return if (res.isNullOrEmpty()) {
root.parseFailed("Cannot find username")
} else res
}
private fun getSortKey(sortOrder: SortOrder) =
when (sortOrder) {
SortOrder.ALPHABETICAL -> "name"
@@ -244,21 +274,21 @@ internal abstract class GroupleParser(source: MangaSource, userAgent: String) :
val tagsIndex = context.httpGet(url, headers).parseHtml()
.body().selectFirst("form.search-form")
?.select("div.form-group")
?.get(1) ?: parseFailed("Genres filter element not found")
?.get(1) ?: throw ParseException("Genres filter element not found", url)
val tagNames = tags.map { it.title.lowercase() }
val payload = HashMap<String, String>()
var foundGenres = 0
tagsIndex.select("li.property").forEach { li ->
val name = li.text().trim().lowercase()
val id = li.selectFirst("input")?.id()
?: parseFailed("Id for tag $name not found")
?: li.parseFailed("Id for tag $name not found")
payload[id] = if (name in tagNames) {
foundGenres++
"in"
} else ""
}
if (foundGenres != tags.size) {
parseFailed("Some genres are not found")
tagsIndex.parseFailed("Some genres are not found")
}
// Step 2: advanced search
payload["q"] = ""

View File

@@ -1,4 +1,4 @@
package org.koitharu.kotatsu.parsers.site
package org.koitharu.kotatsu.parsers.site.grouple
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
@@ -8,10 +8,14 @@ import org.koitharu.kotatsu.parsers.model.MangaSource
@MangaSourceParser("MINTMANGA", "MintManga", "ru")
internal class MintMangaParser(
override val context: MangaLoaderContext,
) : GroupleParser(MangaSource.MINTMANGA, "mintmangafun") {
) : GroupleParser(MangaSource.MINTMANGA, "mintmangafun", 2) {
override val configKeyDomain = ConfigKey.Domain(
"mintmanga.live",
arrayOf("mintmanga.live", "mintmanga.com"),
)
override fun getFaviconUrl(): String {
return "https://resmm.rmr.rocks/static/apple-touch-icon-8fff291039c140493adb0c7ba81065ad.png"
}
}

View File

@@ -1,4 +1,4 @@
package org.koitharu.kotatsu.parsers.site
package org.koitharu.kotatsu.parsers.site.grouple
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
@@ -8,10 +8,14 @@ import org.koitharu.kotatsu.parsers.model.MangaSource
@MangaSourceParser("READMANGA_RU", "ReadManga", "ru")
internal class ReadmangaParser(
override val context: MangaLoaderContext,
) : GroupleParser(MangaSource.READMANGA_RU, "readmangafun") {
) : GroupleParser(MangaSource.READMANGA_RU, "readmangafun", 1) {
override val configKeyDomain = ConfigKey.Domain(
"readmanga.io",
"readmanga.live",
arrayOf("readmanga.io", "readmanga.live", "readmanga.me"),
)
override fun getFaviconUrl(): String {
return "https://resrm.rmr.rocks/static/apple-touch-icon-3162037c9df9f28dca0f9a4092cb0f65.png"
}
}

View File

@@ -1,4 +1,4 @@
package org.koitharu.kotatsu.parsers.site
package org.koitharu.kotatsu.parsers.site.grouple
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
@@ -8,7 +8,11 @@ import org.koitharu.kotatsu.parsers.model.MangaSource
@MangaSourceParser("SELFMANGA", "SelfManga", "ru")
internal class SelfMangaParser(
override val context: MangaLoaderContext,
) : GroupleParser(MangaSource.SELFMANGA, "selfmangafun") {
) : GroupleParser(MangaSource.SELFMANGA, "selfmangafun", 3) {
override val configKeyDomain = ConfigKey.Domain("selfmanga.live", null)
override fun getFaviconUrl(): String {
return "https://ressm.rmr.rocks/static/apple-touch-icon-a769ea533d811b73ac3eedde658bb1d3.png"
}
}

View File

@@ -0,0 +1,187 @@
package org.koitharu.kotatsu.parsers.site.madara
import androidx.collection.arraySetOf
import org.jsoup.nodes.Element
import org.koitharu.kotatsu.parsers.InternalParsersApi
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import java.text.SimpleDateFormat
import java.util.*
abstract class Madara5Parser @InternalParsersApi constructor(
override val context: MangaLoaderContext,
source: MangaSource,
domain: String,
) : PagedMangaParser(source, pageSize = 22) {
protected open val datePattern = "MMMM dd, HH:mm"
protected open val tagPrefix = "/mangas/"
protected open val nsfwTags = arraySetOf("yaoi", "yuri", "mature")
override val sortOrders: Set<SortOrder> = EnumSet.of(SortOrder.UPDATED)
override val configKeyDomain = ConfigKey.Domain(domain, null)
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val domain = getDomain()
val url = buildString {
append("https://")
append(domain)
append("/search?s=")
if (!query.isNullOrEmpty()) {
append(query.urlEncoded())
}
append("&post_type=wp-manga")
if (!tags.isNullOrEmpty()) {
for (tag in tags) {
append("&genre%5B%5D=")
append(tag.key)
}
}
append("&op=1&author=&artist=&page=")
append(page)
}
val root = context.httpGet(url).parseHtml().body().selectFirstOrThrow(".search-wrap")
return root.select(".c-tabs-item__content").map { div ->
val a = div.selectFirstOrThrow("a")
val img = div.selectLastOrThrow("img")
val href = a.attrAsRelativeUrl("href")
val postContent = root.selectFirstOrThrow(".post-content")
val tagSet = postContent.getElementsContainingOwnText("Genre")
.firstOrNull()?.tableValue()
?.getElementsByAttributeValueContaining("href", tagPrefix)
?.mapToSet { it.asMangaTag() }.orEmpty()
Manga(
id = generateUid(href),
title = a.attr("title"),
altTitle = postContent.getElementsContainingOwnText("Alternative")
.firstOrNull()?.tableValue()?.text()?.trim(),
url = href,
publicUrl = a.attrAsAbsoluteUrl("href"),
coverUrl = img.src().orEmpty(),
author = postContent.getElementsContainingOwnText("Author")
.firstOrNull()?.tableValue()?.text()?.trim(),
state = postContent.getElementsContainingOwnText("Status")
.firstOrNull()?.tableValue()?.text()?.asMangaState(),
isNsfw = isNsfw(tagSet),
rating = div.selectFirstOrThrow(".score").text()
.toFloatOrNull()?.div(5f) ?: RATING_UNKNOWN,
tags = tagSet,
source = source,
)
}
}
override suspend fun getDetails(manga: Manga): Manga {
val root = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml().body()
.selectFirstOrThrow(".site-content")
val postContent = root.selectFirstOrThrow(".post-content")
val tags = postContent.getElementsContainingOwnText("Genre")
.firstOrNull()?.tableValue()
?.getElementsByAttributeValueContaining("href", tagPrefix)
?.mapToSet { a -> a.asMangaTag() } ?: manga.tags
val mangaId = root.getElementById("manga-chapters-holder")?.attr("data-id")?.toLongOrNull()
?: root.parseFailed("Cannot find mangaId")
return manga.copy(
description = (root.selectFirst(".detail-content")
?: root.selectFirstOrThrow(".description-summary")).html(),
author = postContent.getElementsContainingOwnText("Author")
.firstOrNull()?.tableValue()?.text()?.trim(),
state = postContent.getElementsContainingOwnText("Status")
.firstOrNull()?.tableValue()?.text()?.asMangaState(),
tags = tags,
isNsfw = isNsfw(tags),
chapters = loadChapters(mangaId),
)
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(getDomain())
val doc = context.httpGet(fullUrl).parseHtml()
val arrayData = doc.getElementById("arraydata") ?: doc.parseFailed("#arraydata not found")
return arrayData.html().split(',').map { url ->
MangaPage(
id = generateUid(url),
url = url,
referer = fullUrl,
preview = null,
source = source,
)
}
}
override suspend fun getTags(): Set<MangaTag> {
val doc = context.httpGet("http://${getDomain()}/").parseHtml().body()
return doc.getElementsByAttributeValueContaining("href", tagPrefix)
.mapToSet { it.asMangaTag() }
}
private suspend fun loadChapters(mangaId: Long): List<MangaChapter> {
val dateFormat = SimpleDateFormat(datePattern, sourceLocale ?: Locale.US)
val doc = context.httpGet("https://${getDomain()}/ajax-list-chapter?mangaID=$mangaId").parseHtml()
return doc.select("li.wp-manga-chapter").asReversed().mapChapters { i, li ->
val a = li.selectFirstOrThrow("a")
val href = a.attrAsRelativeUrl("href")
MangaChapter(
id = generateUid(href),
url = href,
name = a.text(),
number = i + 1,
branch = null,
uploadDate = dateFormat.tryParse(
li.selectFirst(".chapter-release-date")?.text()?.trim(),
),
scanlator = null,
source = source,
)
}
}
protected fun isNsfw(tags: Set<MangaTag>): Boolean {
return tags.any { it.key in nsfwTags }
}
private fun Element.src(): String? {
return absUrl("data-src").ifEmpty {
absUrl("src")
}.takeUnless { it.isEmpty() }
}
private fun Element.tableValue(): Element {
for (p in parents()) {
val children = p.children()
if (children.size == 2) {
return children[1]
}
}
parseFailed("Cannot find tableValue for node ${text()}")
}
private fun String.asMangaState() = when (trim().lowercase(sourceLocale ?: Locale.US)) {
"ongoing" -> MangaState.ONGOING
"completed" -> MangaState.FINISHED
else -> null
}
private fun Element.asMangaTag() = MangaTag(
title = ownText(),
key = attr("href").removeSuffix('/').substringAfterLast('/')
.replace('-', '+'),
source = source,
)
@MangaSourceParser("MANGAOWLS", "BeautyManga", "en")
class BeautyManga(context: MangaLoaderContext) : Madara5Parser(context, MangaSource.MANGAOWLS, "beautymanga.com") {
override fun getFaviconUrl() = "http://${getDomain()}/frontend/images/favico.png"
}
}

View File

@@ -0,0 +1,70 @@
package org.koitharu.kotatsu.parsers.site.madara
import kotlinx.coroutines.async
import kotlinx.coroutines.coroutineScope
import org.jsoup.nodes.Element
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import java.text.SimpleDateFormat
import java.util.*
internal abstract class Madara6Parser(
context: MangaLoaderContext,
source: MangaSource,
domain: String,
) : MadaraParser(context, source, domain) {
override val datePattern: String = "dd MMMM yyyy"
override suspend fun getDetails(manga: Manga): Manga {
return coroutineScope {
val chapters = async { loadChapters(manga.url) }
val body = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml().body()
parseDetails(manga, body, chapters.await())
}
}
protected fun Element.tableValue(): Element {
for (p in parents()) {
val children = p.children()
if (children.size == 2) {
return children[1]
}
}
parseFailed("Cannot find tableValue for node ${text()}")
}
protected abstract fun String.asMangaState(): MangaState?
protected fun Element.asMangaTag() = MangaTag(
title = ownText(),
key = attr("href").removeSuffix('/').substringAfterLast('/')
.replace('-', '+'),
source = source,
)
protected open suspend fun loadChapters(mangaUrl: String): List<MangaChapter> {
val url = mangaUrl.toAbsoluteUrl(getDomain()).removeSuffix('/') + "/ajax/chapters/"
val dateFormat = SimpleDateFormat(datePattern, sourceLocale ?: Locale.ROOT)
val doc = context.httpPost(url, emptyMap()).parseHtml()
return doc.select("li.wp-manga-chapter").asReversed().mapChapters { i, li ->
val a = li.selectFirstOrThrow("a")
val href = a.attrAsRelativeUrl("href")
MangaChapter(
id = generateUid(href),
url = href,
name = a.text(),
number = i + 1,
branch = null,
uploadDate = dateFormat.tryParse(
li.selectFirst(".chapter-release-date")?.text()?.trim(),
),
scanlator = null,
source = source,
)
}
}
protected abstract fun parseDetails(manga: Manga, body: Element, chapters: List<MangaChapter>): Manga
}

View File

@@ -1,9 +1,9 @@
package org.koitharu.kotatsu.parsers.site
package org.koitharu.kotatsu.parsers.site.madara
import org.jsoup.nodes.Element
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.exception.ParseException
import org.koitharu.kotatsu.parsers.model.*
@@ -12,13 +12,11 @@ import java.text.DateFormat
import java.text.SimpleDateFormat
import java.util.*
private const val PAGE_SIZE = 12
internal abstract class MadaraParser(
override val context: MangaLoaderContext,
source: MangaSource,
domain: String,
) : MangaParser(source) {
) : PagedMangaParser(source, pageSize = 12) {
override val configKeyDomain = ConfigKey.Domain(domain, null)
@@ -29,34 +27,36 @@ internal abstract class MadaraParser(
protected open val tagPrefix = "manga-genre/"
protected open val isNsfwSource = false
protected open val datePattern = "MMMM dd, yyyy"
override suspend fun getList(
offset: Int,
init {
paginator.firstPage = 0
searchPaginator.firstPage = 0
}
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
val tag = when {
tags.isNullOrEmpty() -> null
tags.size == 1 -> tags.first()
else -> throw IllegalArgumentException("Multiple genres are not supported by this source")
}
val tag = tags.oneOrThrowIfMany()
val payload = createRequestTemplate()
payload["page"] = (offset / PAGE_SIZE.toFloat()).toIntUp().toString()
payload["page"] = page.toString()
payload["vars[meta_key]"] = when (sortOrder) {
SortOrder.POPULARITY -> "_wp_manga_views"
SortOrder.UPDATED -> "_latest_update"
else -> "_wp_manga_views"
}
payload["vars[wp-manga-genre]"] = tag?.key.orEmpty()
payload["vars[s]"] = query.orEmpty()
payload["vars[s]"] = query?.urlEncoded().orEmpty()
val doc = context.httpPost(
"https://${getDomain()}/wp-admin/admin-ajax.php",
payload,
).parseHtml()
return doc.select("div.row.c-tabs-item__content").map { div ->
val href = div.selectFirst("a")?.attrAsRelativeUrlOrNull("href")
?: parseFailed("Link not found")
?: div.parseFailed("Link not found")
val summary = div.selectFirst(".tab-summary")
Manga(
id = generateUid(href),
@@ -95,7 +95,7 @@ internal abstract class MadaraParser(
val root1 = body.selectFirst("header")?.selectFirst("ul.second-menu")
val root2 = body.selectFirst("div.genres_wrap")?.selectFirst("ul.list-unstyled")
if (root1 == null && root2 == null) {
parseFailed("Root not found")
doc.parseFailed("Root not found")
}
val list = root1?.select("li").orEmpty() + root2?.select("li").orEmpty()
val keySet = HashSet<String>(list.size)
@@ -108,7 +108,9 @@ internal abstract class MadaraParser(
}
MangaTag(
key = href,
title = a.ownText().trim().toTitleCase(),
title = a.ownText().trim().ifEmpty {
a.selectFirst(".menu-image-title")?.text()?.trim() ?: return@mapNotNullToSet null
}.toTitleCase(),
source = source,
)
}
@@ -120,11 +122,11 @@ internal abstract class MadaraParser(
val root = doc.body().selectFirst("div.profile-manga")
?.selectFirst("div.summary_content")
?.selectFirst("div.post-content")
?: throw ParseException("Root not found")
?: throw ParseException("Root not found", fullUrl)
val root2 = doc.body().selectFirst("div.content-area")
?.selectFirst("div.c-page")
?: throw ParseException("Root2 not found")
val dateFormat = SimpleDateFormat("MMMM dd, yyyy", Locale.US)
?: throw ParseException("Root2 not found", fullUrl)
val dateFormat = SimpleDateFormat(datePattern, Locale.US)
return manga.copy(
tags = root.selectFirst("div.genres-content")?.select("a")
?.mapNotNullToSet { a ->
@@ -139,9 +141,9 @@ internal abstract class MadaraParser(
?.select("p")
?.filterNot { it.ownText().startsWith("A brief description") }
?.joinToString { it.html() },
chapters = root2.select("li").asReversed().mapIndexed { i, li ->
chapters = root2.select("li").asReversed().mapChapters { i, li ->
val a = li.selectFirst("a")
val href = a?.attrAsRelativeUrlOrNull("href") ?: parseFailed("Link is missing")
val href = a?.attrAsRelativeUrlOrNull("href") ?: li.parseFailed("Link is missing")
MangaChapter(
id = generateUid(href),
name = a.ownText(),
@@ -164,10 +166,10 @@ internal abstract class MadaraParser(
val doc = context.httpGet(fullUrl).parseHtml()
val root = doc.body().selectFirst("div.main-col-inner")
?.selectFirst("div.reading-content")
?: throw ParseException("Root not found")
?: throw ParseException("Root not found", fullUrl)
return root.select("div.page-break").map { div ->
val img = div.selectFirst("img") ?: parseFailed("Page image not found")
val url = img.src()?.toRelativeUrl(getDomain()) ?: parseFailed("Image src not found")
val img = div.selectFirst("img") ?: div.parseFailed("Page image not found")
val url = img.src()?.toRelativeUrl(getDomain()) ?: div.parseFailed("Image src not found")
MangaPage(
id = generateUid(url),
url = url,
@@ -202,6 +204,7 @@ internal abstract class MadaraParser(
set(Calendar.MILLISECOND, 0)
}.timeInMillis
}
date.startsWith("today", ignoreCase = true) -> {
Calendar.getInstance().apply {
set(Calendar.HOUR_OF_DAY, 0)
@@ -210,6 +213,7 @@ internal abstract class MadaraParser(
set(Calendar.MILLISECOND, 0)
}.timeInMillis
}
date.contains(Regex("""\d(st|nd|rd|th)""")) -> {
// Clean date (e.g. 5th December 2019 to 5 December 2019) before parsing it
date.split(" ").map {
@@ -221,6 +225,7 @@ internal abstract class MadaraParser(
}
.let { dateFormat.tryParse(it.joinToString(" ")) }
}
else -> dateFormat.tryParse(date)
}
}
@@ -240,24 +245,28 @@ internal abstract class MadaraParser(
"dia",
"day",
).anyWordIn(date) -> cal.apply { add(Calendar.DAY_OF_MONTH, -number) }.timeInMillis
WordSet("jam", "saat", "heure", "hora", "hour").anyWordIn(date) -> cal.apply {
add(
Calendar.HOUR,
-number,
)
}.timeInMillis
WordSet("menit", "dakika", "min", "minute", "minuto").anyWordIn(date) -> cal.apply {
add(
Calendar.MINUTE,
-number,
)
}.timeInMillis
WordSet("detik", "segundo", "second").anyWordIn(date) -> cal.apply {
add(
Calendar.SECOND,
-number,
)
}.timeInMillis
WordSet("month").anyWordIn(date) -> cal.apply { add(Calendar.MONTH, -number) }.timeInMillis
WordSet("year").anyWordIn(date) -> cal.apply { add(Calendar.YEAR, -number) }.timeInMillis
else -> 0
@@ -272,12 +281,12 @@ internal abstract class MadaraParser(
private fun createRequestTemplate() =
(
"action=madara_load_more&page=1&template=madara-core%2Fcontent%2Fcontent-search&vars%5Bs%5D=&vars%5B" +
"orderby%5D=meta_value_num&vars%5Bpaged%5D=1&vars%5Btemplate%5D=search&vars%5Bmeta_query" +
"%5D%5B0%5D%5Brelation%5D=AND&vars%5Bmeta_query%5D%5Brelation%5D=OR&vars%5Bpost_type" +
"%5D=wp-manga&vars%5Bpost_status%5D=publish&vars%5Bmeta_key%5D=_latest_update&vars%5Border" +
"%5D=desc&vars%5Bmanga_archives_item_layout%5D=default"
).split('&')
"action=madara_load_more&page=1&template=madara-core%2Fcontent%2Fcontent-search&vars%5Bs%5D=&vars%5B" +
"orderby%5D=meta_value_num&vars%5Bpaged%5D=1&vars%5Btemplate%5D=search&vars%5Bmeta_query" +
"%5D%5B0%5D%5Brelation%5D=AND&vars%5Bmeta_query%5D%5Brelation%5D=OR&vars%5Bpost_type" +
"%5D=wp-manga&vars%5Bpost_status%5D=publish&vars%5Bmeta_key%5D=_latest_update&vars%5Border" +
"%5D=desc&vars%5Bmanga_archives_item_layout%5D=default"
).split('&')
.map {
val pos = it.indexOf('=')
it.substring(0, pos) to it.substring(pos + 1)
@@ -291,6 +300,24 @@ internal abstract class MadaraParser(
@MangaSourceParser("MANGAWEEBS", "MangaWeebs", "en")
class MangaWeebs(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGAWEEBS, "mangaweebs.in")
@MangaSourceParser("KINGMANGA", "KingManga", "en")
class KingManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.KINGMANGA, "king-manga.com") {
override fun getFaviconUrl(): String {
return "https://${getDomain()}/wp-content/uploads/2022/05/cropped-logo-king-02-03-32x32.png"
}
}
@MangaSourceParser("MANGAHATACHI", "MangahaTachi", "ja")
class MangahaTachi(context: MangaLoaderContext) :
MadaraParser(context, MangaSource.MANGAHATACHI, "mangahatachi.com")
@MangaSourceParser("PIANMANGA", "PianManga", "en")
class PianManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.PIANMANGA, "pianmanga.com")
@MangaSourceParser("MANGAROSIE", "MangaRosie", "en")
class MangaRosie(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGAROSIE, "mangarosie.me")
@MangaSourceParser("READMANWHA", "ReadManwha", "en")
class ReadManwha(context: MangaLoaderContext) : MadaraParser(context, MangaSource.READMANWHA, "readmanwha.net")
@@ -311,12 +338,29 @@ internal abstract class MadaraParser(
@MangaSourceParser("MANGATX_OT", "MangaTx (ot)", "en")
class MangaTxOt(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGATX_OT, "manga-tx.com")
@MangaSourceParser("MANGALEK", "MangaLek", "ar")
class MangaLek(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGALEK, "mangalek.com") {
override fun getFaviconUrl(): String {
return "https://${getDomain()}/wp-content/logo192.png"
}
}
@MangaSourceParser("HARIMANGA", "HariManga", "en")
class HariManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.HARIMANGA, "harimanga.com")
@MangaSourceParser("KISSMANGA", "KissManga", "en")
class KissManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.KISSMANGA, "kissmanga.in")
@MangaSourceParser("MANGAROCK", "MangaRock", "en")
class MangaRock(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGAROCK, "mangarockteam.com")
@MangaSourceParser("ISEKAISCAN_EU", "IsekaiScan (eu)", "en")
class IsekaiScanEu(context: MangaLoaderContext) : MadaraParser(context, MangaSource.ISEKAISCAN_EU, "isekaiscan.eu")
@MangaSourceParser("FREEMANGA", "FreeManga", "en")
class FreeManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.FREEMANGA, "freemanga.me")
@MangaSourceParser("ISEKAISCAN", "IsekaiScan", "en")
class IsekaiScan(context: MangaLoaderContext) : MadaraParser(context, MangaSource.ISEKAISCAN, "isekaiscan.com") {
@@ -328,6 +372,18 @@ internal abstract class MadaraParser(
@MangaSourceParser("MANGA_KOMI", "MangaKomi", "en")
class MangaKomi(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGA_KOMI, "mangakomi.io")
@MangaSourceParser("READMANHWA", "ReadManhwa", "en")
class ReadManhwa(context: MangaLoaderContext) : MadaraParser(context, MangaSource.READMANHWA, "readmanhwa.info")
@MangaSourceParser("NEATMANGA", "NeatManga", "en")
class NeatManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.NEATMANGA, "neatmanga.com")
@MangaSourceParser("MANGAKIK", "MangaKik", "en")
class MangaKik(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGAKIK, "mangakik.net")
@MangaSourceParser("MANHWACLAN", "ManhwaClan", "en")
class ManhwaClan(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANHWACLAN, "manhwaclan.com")
@MangaSourceParser("MANGA_3S", "Manga3s", "en")
class Manga3s(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANGA_3S, "manga3s.com")
@@ -346,6 +402,15 @@ internal abstract class MadaraParser(
@MangaSourceParser("X2MANGA", "X2Manga", "en")
class X2Manga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.X2MANGA, "x2manga.com")
@MangaSourceParser("VINLOAD", "VinLoad", "en")
class VinLoad(context: MangaLoaderContext) : MadaraParser(context, MangaSource.VINLOAD, "vinload.com")
@MangaSourceParser("S2MANGA", "S2Manga", "en")
class S2Manga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.S2MANGA, "s2manga.com")
@MangaSourceParser("MANHWA", "Manhwa", "en")
class Manhwa(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANHWA, "manhwa.pics")
@MangaSourceParser("SKY_MANGA", "Sky Manga", "en")
class SkyManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.SKY_MANGA, "skymanga.xyz") {
@@ -408,6 +473,18 @@ internal abstract class MadaraParser(
}
}
@MangaSourceParser("CAT_300", "Cat300", "th")
class Cat300(context: MangaLoaderContext) : MadaraParser(context, MangaSource.CAT_300, "cat300.com") {
override val isNsfwSource = true
}
@MangaSourceParser("BIBIMANGA", "BibiManga", "en")
class BibiManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.BIBIMANGA, "bibimanga.com") {
override val isNsfwSource = true
}
@MangaSourceParser("MANHWA_CHILL", "Manhwa Chill", "en")
class ManhwaChill(context: MangaLoaderContext) : MadaraParser(context, MangaSource.MANHWA_CHILL, "manhwachill.me")
@@ -452,4 +529,4 @@ internal abstract class MadaraParser(
@MangaSourceParser("ZINMANGA", "ZINMANGA", "en")
class Zinmanga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.ZINMANGA, "zinmanga.com")
}
}

View File

@@ -0,0 +1,56 @@
package org.koitharu.kotatsu.parsers.site.madara
import org.jsoup.nodes.Element
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.model.Manga
import org.koitharu.kotatsu.parsers.model.MangaChapter
import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.model.MangaState
import org.koitharu.kotatsu.parsers.util.attrAsAbsoluteUrlOrNull
import org.koitharu.kotatsu.parsers.util.mapToSet
import org.koitharu.kotatsu.parsers.util.selectFirstOrThrow
import java.util.*
@MangaSourceParser("MANGAS_ORIGINES", "Mangas Origines", "fr")
internal class MangasOriginesParser(context: MangaLoaderContext) :
Madara6Parser(context, MangaSource.MANGAS_ORIGINES, "mangas-origines.fr") {
override val tagPrefix = "catalogues-genre/"
override fun getFaviconUrl(): String {
return "https://${getDomain()}/wp-content/uploads/2020/11/Mangas-150x150.png"
}
override fun parseDetails(manga: Manga, body: Element, chapters: List<MangaChapter>): Manga {
val root = body.selectFirstOrThrow(".site-content")
val postContent = root.selectFirstOrThrow(".post-content")
val tags = postContent.getElementsContainingOwnText("Genre")
.firstOrNull()?.tableValue()
?.getElementsByAttributeValueContaining("href", tagPrefix)
?.mapToSet { a -> a.asMangaTag() } ?: manga.tags
return manga.copy(
largeCoverUrl = root.selectFirst("picture")
?.selectFirst("img[data-src]")
?.attrAsAbsoluteUrlOrNull("data-src"),
description = (root.selectFirst(".detail-content")
?: root.selectFirstOrThrow(".manga-excerpt")).html(),
author = postContent.getElementsContainingOwnText("Auteur")
.firstOrNull()?.tableValue()?.text()?.trim(),
state = postContent.getElementsContainingOwnText("STATUS")
.firstOrNull()?.tableValue()?.text()?.asMangaState(),
tags = tags,
isNsfw = body.hasClass("adult-content"),
chapters = chapters,
)
}
override fun String.asMangaState() = when (trim().lowercase(Locale.FRANCE)) {
"en cours" -> MangaState.ONGOING
"abandonné",
"terminé",
-> MangaState.FINISHED
else -> null
}
}

View File

@@ -0,0 +1,57 @@
package org.koitharu.kotatsu.parsers.site.madara
import org.jsoup.nodes.Element
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.model.Manga
import org.koitharu.kotatsu.parsers.model.MangaChapter
import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.model.MangaState
import org.koitharu.kotatsu.parsers.util.attrAsAbsoluteUrlOrNull
import org.koitharu.kotatsu.parsers.util.mapToSet
import org.koitharu.kotatsu.parsers.util.selectFirstOrThrow
import java.util.*
@MangaSourceParser("PRISMA_SCANS", "Prisma Scans", "pt")
internal class PrismaScansParser(context: MangaLoaderContext) :
Madara6Parser(context, MangaSource.PRISMA_SCANS, "prismascans.net") {
override val tagPrefix = "manga-genre/"
override fun getFaviconUrl(): String {
return "https://${getDomain()}/wp-content/uploads/2022/07/cropped-branca-1-192x192.png"
}
override fun parseDetails(manga: Manga, body: Element, chapters: List<MangaChapter>): Manga {
val root = body.selectFirstOrThrow(".site-content")
val postContent = root.selectFirstOrThrow(".post-content")
val tags = postContent.getElementsContainingOwnText("Gênero")
.firstOrNull()?.tableValue()
?.getElementsByAttributeValueContaining("href", tagPrefix)
?.mapToSet { a -> a.asMangaTag() } ?: manga.tags
return manga.copy(
largeCoverUrl = root.selectFirst("picture")
?.selectFirst("img[data-src]")
?.attrAsAbsoluteUrlOrNull("data-src"),
description = root.selectFirstOrThrow(".description-summary").firstElementChild()?.html(),
author = postContent.getElementsContainingOwnText("Artista")
.firstOrNull()?.tableValue()?.text()?.trim(),
altTitle = postContent.getElementsContainingOwnText("Título Alternativo")
.firstOrNull()?.tableValue()?.text()?.trim(),
state = postContent.getElementsContainingOwnText("Status")
.firstOrNull()?.tableValue()?.text()?.asMangaState(),
tags = tags,
isNsfw = body.hasClass("adult-content"),
chapters = chapters,
)
}
override fun String.asMangaState() = when (trim().lowercase(sourceLocale ?: Locale.ROOT)) {
"em lançamento" -> MangaState.ONGOING
"completo",
"cancelado",
-> MangaState.FINISHED
else -> null
}
}

View File

@@ -0,0 +1,51 @@
package org.koitharu.kotatsu.parsers.site.madara
import org.jsoup.nodes.Element
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.attrAsAbsoluteUrlOrNull
import org.koitharu.kotatsu.parsers.util.mapToSet
import org.koitharu.kotatsu.parsers.util.selectFirstOrThrow
import java.util.*
@MangaSourceParser("TATAKAE_SCANS", "Tatakae Scans", "pt")
internal class TatakaeScansParser(context: MangaLoaderContext) :
Madara6Parser(context, MangaSource.TATAKAE_SCANS, "tatakaescan.com") {
override val datePattern: String = "dd 'de' MMMMM 'de' yyyy"
override fun parseDetails(manga: Manga, body: Element, chapters: List<MangaChapter>): Manga {
val root = body.selectFirstOrThrow(".site-content")
val postContent = root.selectFirstOrThrow(".post-content")
val tags = postContent.getElementsContainingOwnText("Gênero")
.firstOrNull()?.tableValue()
?.getElementsByAttributeValueContaining("href", tagPrefix)
?.mapToSet { a -> a.asMangaTag() } ?: manga.tags
return manga.copy(
largeCoverUrl = root.selectFirst("picture")
?.selectFirst("img[data-src]")
?.attrAsAbsoluteUrlOrNull("data-src"),
description = (root.selectFirst(".detail-content")
?: root.selectFirstOrThrow(".manga-excerpt")).html(),
author = postContent.getElementsContainingOwnText("Autor")
.firstOrNull()?.tableValue()?.text()?.trim(),
state = postContent.getElementsContainingOwnText("Status")
.firstOrNull()?.tableValue()?.text()?.asMangaState(),
tags = tags,
isNsfw = body.hasClass("adult-content"),
chapters = chapters,
)
}
override fun String.asMangaState() = when (trim().lowercase(Locale.ROOT)) {
"em lançamento" -> MangaState.ONGOING
else -> null
}
override fun getFaviconUrl(): String {
return "https://${getDomain()}/wp-content/uploads/2022/07/cropped-favicon-180x180.png"
}
}

View File

@@ -1,13 +1,14 @@
package org.koitharu.kotatsu.parsers.site
package org.koitharu.kotatsu.parsers.site.multichan
import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.exception.ParseException
import org.koitharu.kotatsu.parsers.MangaParserAuthProvider
import org.koitharu.kotatsu.parsers.exception.AuthRequiredException
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import java.text.SimpleDateFormat
import java.util.*
internal abstract class ChanParser(source: MangaSource) : MangaParser(source) {
internal abstract class ChanParser(source: MangaSource) : MangaParser(source), MangaParserAuthProvider {
override val sortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.NEWEST,
@@ -15,6 +16,12 @@ internal abstract class ChanParser(source: MangaSource) : MangaParser(source) {
SortOrder.ALPHABETICAL,
)
override val authUrl: String
get() = "https://${getDomain()}"
override val isAuthorized: Boolean
get() = context.cookieJar.getCookies(getDomain()).any { it.name == "dle_user_id" }
override suspend fun getList(
offset: Int,
query: String?,
@@ -29,16 +36,18 @@ internal abstract class ChanParser(source: MangaSource) : MangaParser(source) {
}
"https://$domain/?do=search&subaction=search&story=${query.urlEncoded()}"
}
!tags.isNullOrEmpty() -> tags.joinToString(
prefix = "https://$domain/tags/",
postfix = "&n=${getSortKey2(sortOrder)}?offset=$offset",
separator = "+",
) { tag -> tag.key }
else -> "https://$domain/${getSortKey(sortOrder)}?offset=$offset"
}
val doc = context.httpGet(url).parseHtml()
val root = doc.body().selectFirst("div.main_fon")?.getElementById("content")
?: parseFailed("Cannot find root")
?: doc.parseFailed("Cannot find root")
return root.select("div.content_row").mapNotNull { row ->
val a = row.selectFirst("div.manga_row1")?.selectFirst("h2")?.selectFirst("a")
?: return@mapNotNull null
@@ -74,15 +83,14 @@ internal abstract class ChanParser(source: MangaSource) : MangaParser(source) {
override suspend fun getDetails(manga: Manga): Manga {
val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml()
val root =
doc.body().getElementById("dle-content") ?: parseFailed("Cannot find root")
val root = doc.body().getElementById("dle-content") ?: doc.parseFailed("Cannot find root")
val dateFormat = SimpleDateFormat("yyyy-MM-dd", Locale.US)
return manga.copy(
description = root.getElementById("description")?.html()?.substringBeforeLast("<div"),
largeCoverUrl = root.getElementById("cover")?.absUrl("src"),
chapters = root.select("table.table_cha tr:gt(1)").reversed().mapIndexedNotNull { i, tr ->
chapters = root.select("table.table_cha tr:gt(1)").reversed().mapChapters { i, tr ->
val href = tr?.selectFirst("a")?.attrAsRelativeUrlOrNull("href")
?: return@mapIndexedNotNull null
?: return@mapChapters null
MangaChapter(
id = generateUid(href),
name = tr.selectFirst("a")?.text().orEmpty(),
@@ -125,16 +133,16 @@ internal abstract class ChanParser(source: MangaSource) : MangaParser(source) {
)
}
}
parseFailed("Pages list not found at ${chapter.url}")
doc.parseFailed("Pages list not found at ${chapter.url}")
}
override suspend fun getTags(): Set<MangaTag> {
val domain = getDomain()
val doc = context.httpGet("https://$domain/mostfavorites&sort=manga").parseHtml()
val root = doc.body().selectFirst("div.main_fon")?.getElementById("side")
?.select("ul")?.last() ?: throw ParseException("Cannot find root")
?.select("ul")?.last() ?: doc.parseFailed("Cannot find root")
return root.select("li.sidetag").mapToSet { li ->
val a = li.children().last() ?: throw ParseException("a is null")
val a = li.children().lastOrNull() ?: li.parseFailed("a is null")
MangaTag(
title = a.text().toTagName(),
key = a.attr("href").substringAfterLast('/'),
@@ -143,6 +151,14 @@ internal abstract class ChanParser(source: MangaSource) : MangaParser(source) {
}
}
override suspend fun getUsername(): String {
val doc = context.httpGet("https://${getDomain()}").parseHtml().body()
val root = doc.requireElementById("top_user")
val a = root.getElementsByAttributeValueContaining("href", "/user/").firstOrNull()
?: throw AuthRequiredException(source)
return a.attr("href").removeSuffix('/').substringAfterLast('/')
}
private fun getSortKey(sortOrder: SortOrder) =
when (sortOrder) {
SortOrder.ALPHABETICAL -> "catalog"

View File

@@ -1,21 +1,17 @@
package org.koitharu.kotatsu.parsers.site
package org.koitharu.kotatsu.parsers.site.multichan
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.exception.ParseException
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.mapToSet
import org.koitharu.kotatsu.parsers.util.parseHtml
import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl
import org.koitharu.kotatsu.parsers.util.toTitleCase
import org.koitharu.kotatsu.parsers.util.*
@MangaSourceParser("HENCHAN", "Хентай-тян", "ru")
internal class HenChanParser(override val context: MangaLoaderContext) : ChanParser(MangaSource.HENCHAN) {
override val configKeyDomain = ConfigKey.Domain(
"xxx.hentaichan.live",
arrayOf("xxx.hentaichan.live", "xx.hentaichan.live", "hentaichan.live", "hentaichan.pro"),
"y.hentaichan.live",
arrayOf("y.hentaichan.live", "xxx.hentaichan.live", "xx.hentaichan.live", "hentaichan.live", "hentaichan.pro"),
)
override suspend fun getList(
@@ -34,14 +30,13 @@ internal class HenChanParser(override val context: MangaLoaderContext) : ChanPar
override suspend fun getDetails(manga: Manga): Manga {
val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml()
val root =
doc.body().getElementById("dle-content") ?: throw ParseException("Cannot find root")
val root = doc.body().requireElementById("dle-content")
val readLink = manga.url.replace("manga", "online")
return manga.copy(
description = root.getElementById("description")?.html()?.substringBeforeLast("<div"),
largeCoverUrl = root.getElementById("cover")?.absUrl("src"),
tags = root.selectFirst("div.sidetags")?.select("li.sidetag")?.mapToSet {
val a = it.children().last() ?: parseFailed("Invalid tag")
val a = it.children().last() ?: doc.parseFailed("Invalid tag")
MangaTag(
title = a.text().toTitleCase(),
key = a.attr("href").substringAfterLast('/'),

View File

@@ -1,4 +1,4 @@
package org.koitharu.kotatsu.parsers.site
package org.koitharu.kotatsu.parsers.site.multichan
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser

View File

@@ -1,15 +1,12 @@
package org.koitharu.kotatsu.parsers.site
package org.koitharu.kotatsu.parsers.site.multichan
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.exception.ParseException
import org.koitharu.kotatsu.parsers.model.Manga
import org.koitharu.kotatsu.parsers.model.MangaChapter
import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.util.attrAsRelativeUrl
import org.koitharu.kotatsu.parsers.util.parseHtml
import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl
import org.koitharu.kotatsu.parsers.util.*
@MangaSourceParser("YAOICHAN", "Яой-тян", "ru")
internal class YaoiChanParser(override val context: MangaLoaderContext) : ChanParser(MangaSource.YAOICHAN) {
@@ -18,14 +15,13 @@ internal class YaoiChanParser(override val context: MangaLoaderContext) : ChanPa
override suspend fun getDetails(manga: Manga): Manga {
val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain())).parseHtml()
val root =
doc.body().getElementById("dle-content") ?: throw ParseException("Cannot find root")
val root = doc.body().requireElementById("dle-content")
return manga.copy(
description = root.getElementById("description")?.html()?.substringBeforeLast("<div"),
largeCoverUrl = root.getElementById("cover")?.absUrl("src"),
chapters = root.select("table.table_cha").flatMap { table ->
table.select("div.manga")
}.mapNotNull { it.selectFirst("a") }.reversed().mapIndexed { i, a ->
}.mapNotNull { it.selectFirst("a") }.reversed().mapChapters { i, a ->
val href = a.attrAsRelativeUrl("href")
MangaChapter(
id = generateUid(href),

View File

@@ -1,4 +1,4 @@
package org.koitharu.kotatsu.parsers.site
package org.koitharu.kotatsu.parsers.site.rulib
import org.jsoup.nodes.Document
import org.koitharu.kotatsu.parsers.MangaLoaderContext

View File

@@ -1,13 +1,13 @@
package org.koitharu.kotatsu.parsers.site
package org.koitharu.kotatsu.parsers.site.rulib
import androidx.collection.ArraySet
import org.json.JSONArray
import org.json.JSONObject
import org.jsoup.nodes.Document
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.MangaParserAuthProvider
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.exception.AuthRequiredException
import org.koitharu.kotatsu.parsers.exception.ParseException
@@ -22,7 +22,7 @@ import java.util.*
internal open class MangaLibParser(
override val context: MangaLoaderContext,
source: MangaSource,
) : MangaParser(source), MangaParserAuthProvider {
) : PagedMangaParser(source, pageSize = 60), MangaParserAuthProvider {
override val configKeyDomain = ConfigKey.Domain("mangalib.me", null)
@@ -37,16 +37,15 @@ internal open class MangaLibParser(
SortOrder.NEWEST,
)
override suspend fun getList(
offset: Int,
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder,
): List<Manga> {
if (!query.isNullOrEmpty()) {
return if (offset == 0) search(query) else emptyList()
return if (page == searchPaginator.firstPage) search(query) else emptyList()
}
val page = (offset / 60f).toIntUp()
val url = buildString {
append("https://")
append(getDomain())
@@ -60,7 +59,7 @@ internal open class MangaLibParser(
}
}
val doc = context.httpGet(url).parseHtml()
val root = doc.body().getElementById("manga-list") ?: throw ParseException("Root not found")
val root = doc.body().getElementById("manga-list") ?: doc.parseFailed("Root not found")
val items = root.selectFirst("div.media-cards-grid")?.select("div.media-card-wrap")
?: return emptyList()
return items.mapNotNull { card ->
@@ -86,13 +85,13 @@ internal open class MangaLibParser(
override suspend fun getDetails(manga: Manga): Manga {
val fullUrl = manga.url.toAbsoluteUrl(getDomain())
val doc = context.httpGet("$fullUrl?section=info").parseHtml()
val root = doc.body().getElementById("main-page") ?: throw ParseException("Root not found")
val root = doc.body().getElementById("main-page") ?: throw ParseException("Root not found", fullUrl)
val title = root.selectFirst("div.media-header__wrap")?.children()
val info = root.selectFirst("div.media-content")
val chaptersDoc = context.httpGet("$fullUrl?section=chapters").parseHtml()
val scripts = chaptersDoc.select("script")
val dateFormat = SimpleDateFormat("yyy-MM-dd", Locale.US)
var chapters: ArrayList<MangaChapter>? = null
var chapters: ChaptersListBuilder? = null
scripts@ for (script in scripts) {
val raw = script.html().lines()
for (line in raw) {
@@ -100,7 +99,7 @@ internal open class MangaLibParser(
val json = JSONObject(line.substringAfter('=').substringBeforeLast(';'))
val list = json.getJSONObject("chapters").getJSONArray("list")
val total = list.length()
chapters = ArrayList(total)
chapters = ChaptersListBuilder(total)
for (i in 0 until total) {
val item = list.getJSONObject(i)
val chapterId = item.getLong("chapter_id")
@@ -111,7 +110,6 @@ internal open class MangaLibParser(
append(item.getInt("chapter_volume"))
append("/c")
append(item.getString("chapter_number"))
@Suppress("BlockingMethodInNonBlockingContext") // lint issue
append('/')
append(item.optString("chapter_string"))
}
@@ -160,18 +158,18 @@ internal open class MangaLibParser(
} ?: manga.tags,
isNsfw = isNsfw(doc),
description = info?.selectFirst("div.media-description__text")?.html(),
chapters = chapters,
chapters = chapters?.toList(),
)
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(getDomain())
val doc = context.httpGet(fullUrl).parseHtml()
if (doc.location().endsWith("/register")) {
if (doc.location().substringBefore('?').endsWith("/register")) {
throw AuthRequiredException(source)
}
val scripts = doc.head().select("script")
val pg = (doc.body().getElementById("pg")?.html() ?: parseFailed("Element #pg not found"))
val pg = (doc.body().getElementById("pg")?.html() ?: doc.parseFailed("Element #pg not found"))
.substringAfter('=')
.substringBeforeLast(';')
val pages = JSONArray(pg)
@@ -201,7 +199,7 @@ internal open class MangaLibParser(
}
}
}
throw ParseException("Script with info not found")
throw ParseException("Script with info not found", fullUrl)
}
override suspend fun getTags(): Set<MangaTag> {
@@ -224,7 +222,7 @@ internal open class MangaLibParser(
return result
}
}
throw ParseException("Script with genres not found")
throw ParseException("Script with genres not found", url)
}
override val isAuthorized: Boolean
@@ -239,13 +237,13 @@ internal open class MangaLibParser(
if (body.baseUri().endsWith("/login")) {
throw AuthRequiredException(source)
}
return body.selectFirst(".profile-user__username")?.text() ?: parseFailed("Cannot find username")
return body.selectFirst(".profile-user__username")?.text() ?: body.parseFailed("Cannot find username")
}
protected open fun isNsfw(doc: Document): Boolean {
val sidebar = doc.body().run {
selectFirst(".media-sidebar") ?: selectFirst(".media-info")
} ?: parseFailed("Sidebar not found")
} ?: doc.parseFailed("Sidebar not found")
return sidebar.getElementsContainingOwnText("18+").isNotEmpty()
}

View File

@@ -0,0 +1,14 @@
package org.koitharu.kotatsu.parsers.site.rulib
import org.jsoup.nodes.Document
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.MangaSource
@MangaSourceParser("YAOILIB", "YaoiLib", "ru")
internal class YaoiLibParser(context: MangaLoaderContext) : MangaLibParser(context, MangaSource.YAOILIB) {
override val configKeyDomain = ConfigKey.Domain("yaoilib.me", null)
override fun isNsfw(doc: Document) = true
}

View File

@@ -0,0 +1,42 @@
package org.koitharu.kotatsu.parsers.util
import org.koitharu.kotatsu.parsers.InternalParsersApi
import org.koitharu.kotatsu.parsers.model.MangaChapter
@InternalParsersApi
inline fun <T> Iterable<T>.mapChapters(transform: (index: Int, T) -> MangaChapter?): List<MangaChapter> {
val builder = ChaptersListBuilder(collectionSize())
var index = 0
for (item in this) {
if (builder.add(transform(index, item))) {
index++
}
}
return builder.toList()
}
@PublishedApi
internal fun <T> Iterable<T>.collectionSize(): Int {
return if (this is Collection<*>) this.size else 10
}
@PublishedApi
internal class ChaptersListBuilder(initialSize: Int) {
private val ids = HashSet<Long>(initialSize)
private val list = ArrayList<MangaChapter>(initialSize)
fun add(chapter: MangaChapter?): Boolean {
return chapter != null && ids.add(chapter.id) && list.add(chapter)
}
operator fun plusAssign(chapter: MangaChapter?) {
add(chapter)
}
fun reverse() {
list.reverse()
}
fun toList(): List<MangaChapter> = list
}

View File

@@ -9,10 +9,7 @@ import okhttp3.HttpUrl
private const val SCHEME_HTTPS = "https"
fun CookieJar.insertCookies(domain: String, vararg cookies: String) {
val url = HttpUrl.Builder()
.scheme(SCHEME_HTTPS)
.host(domain)
.build()
val url = safeUrlOf(domain) ?: return
saveFromResponse(
url,
cookies.mapNotNull {
@@ -22,18 +19,12 @@ fun CookieJar.insertCookies(domain: String, vararg cookies: String) {
}
fun CookieJar.insertCookie(domain: String, cookie: Cookie) {
val url = HttpUrl.Builder()
.scheme(SCHEME_HTTPS)
.host(domain)
.build()
val url = safeUrlOf(domain) ?: return
saveFromResponse(url, listOf(cookie))
}
fun CookieJar.getCookies(domain: String): List<Cookie> {
val url = HttpUrl.Builder()
.scheme(SCHEME_HTTPS)
.host(domain)
.build()
val url = safeUrlOf(domain) ?: return emptyList()
return loadForRequest(url)
}
@@ -47,4 +38,13 @@ fun CookieJar.copyCookies(oldDomain: String, newDomain: String, names: Array<Str
}
url.host(newDomain)
saveFromResponse(url.build(), cookies)
}
private fun safeUrlOf(domain: String): HttpUrl? = try {
HttpUrl.Builder()
.scheme(SCHEME_HTTPS)
.host(domain)
.build()
} catch (_: IllegalArgumentException) {
null
}

View File

@@ -0,0 +1,94 @@
package org.koitharu.kotatsu.parsers.util
import okhttp3.Headers
import org.jsoup.nodes.Element
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.model.Favicon
import org.koitharu.kotatsu.parsers.model.Favicons
import org.koitharu.kotatsu.parsers.util.json.mapJSON
class FaviconParser(
private val context: MangaLoaderContext,
private val domain: String,
private val headers: Headers?,
) {
suspend fun parseFavicons(): Favicons {
val url = "https://$domain"
val doc = context.httpGet(url, headers).parseHtml()
val result = HashSet<Favicon>()
val manifestLink = doc.getElementsByAttributeValue("rel", "manifest").firstOrNull()
?.attrAsAbsoluteUrlOrNull("href")
if (manifestLink != null) {
result += parseManifest(manifestLink)
}
val links = doc.getElementsByAttributeValueContaining("rel", "icon")
links.mapNotNullTo(result) { link ->
parseLink(link)
}
if (result.isEmpty()) {
result.add(createFallback())
}
return Favicons(result, url)
}
private fun parseLink(link: Element): Favicon? {
val href = link.attrAsAbsoluteUrlOrNull("href")
if (href == null || href.endsWith('/')) {
return null
}
val sizes = link.attr("sizes")
return Favicon(
url = href,
size = parseSize(sizes),
rel = link.attrOrNull("rel"),
)
}
private fun parseSize(sizes: String): Int {
if (sizes.isEmpty() || sizes == "any") {
return 0
}
return sizes.substringBefore(' ')
.split('x', 'X', '*')
.firstNotNullOfOrNull { it.toIntOrNull() }
?: 0
}
private suspend fun parseManifest(url: String): List<Favicon> {
val json = context.httpGet(url, headers).parseJson()
val icons = json.getJSONArray("icons")
return icons.mapJSON { jo ->
Favicon(
url = jo.getString("src").resolveLink(),
size = parseSize(jo.getString("sizes")),
rel = null,
)
}
}
private fun createFallback(): Favicon {
val href = "https://$domain/favicon.ico"
return Favicon(
url = href,
size = 0,
rel = null,
)
}
private fun String.resolveLink(): String {
return when {
startsWith("http:") || startsWith("https:") -> {
this
}
startsWith('/') -> {
"https://$domain$this"
}
else -> {
"https://$domain/$this"
}
}
}
}

View File

@@ -4,6 +4,7 @@ package org.koitharu.kotatsu.parsers.util
import okhttp3.HttpUrl.Companion.toHttpUrlOrNull
import org.jsoup.nodes.Element
import org.jsoup.select.Elements
import org.jsoup.select.Selector
import org.koitharu.kotatsu.parsers.exception.ParseException
@@ -90,14 +91,27 @@ fun Element.styleValueOrNull(property: String): String? {
return css.substringAfter(':').removeSuffix(';').trim()
}
/**
* Like a `expectFirst` but with detailed error message
*/
fun Element.selectFirstOrThrow(cssQuery: String): Element {
return Selector.selectFirst(cssQuery, this) ?: throw ParseException("Cannot find \"$cssQuery\"")
return Selector.selectFirst(cssQuery, this) ?: throw ParseException("Cannot find \"$cssQuery\"", baseUri())
}
fun Element.selectOrThrow(cssQuery: String): Elements {
return Selector.select(cssQuery, this).ifEmpty {
throw ParseException("Empty result for \"$cssQuery\"", baseUri())
}
}
fun Element.requireElementById(id: String): Element {
return getElementById(id) ?: throw ParseException("Cannot find \"#$id\"")
return getElementById(id) ?: throw ParseException("Cannot find \"#$id\"", baseUri())
}
fun Element.selectLast(cssQuery: String): Element? {
return select(cssQuery).lastOrNull()
}
fun Element.selectLastOrThrow(cssQuery: String): Element {
return selectLast(cssQuery) ?: throw ParseException("Cannot find \"$cssQuery\"", baseUri())
}

View File

@@ -0,0 +1,25 @@
package org.koitharu.kotatsu.parsers.util
import androidx.collection.SparseArrayCompat
import androidx.collection.set
class Paginator constructor(private val initialPageSize: Int) {
var firstPage = 1
private var pages = SparseArrayCompat<Int>()
fun getPage(offset: Int): Int {
if (offset == 0) { // just an optimization
return firstPage
}
pages[offset]?.let { return it }
val pageSize = initialPageSize
val intPage = offset / pageSize
val tail = offset % pageSize
return intPage + firstPage + if (tail == 0) 0 else 1
}
fun onListReceived(offset: Int, page: Int, count: Int) {
pages[offset + count] = if (count > 0) page + 1 else page
}
}

View File

@@ -17,7 +17,7 @@ class CloudFlareInterceptor : Interceptor {
if (response.code == HTTP_FORBIDDEN || response.code == HTTP_UNAVAILABLE) {
if (response.header(HEADER_SERVER)?.startsWith(SERVER_CLOUDFLARE) == true) {
response.closeQuietly()
throw CloudFlareProtectedException(chain.request().url.toString())
throw CloudFlareProtectedException(response.request.url.toString())
}
}
return response

View File

@@ -1,6 +1,7 @@
package org.koitharu.kotatsu.parsers
import com.koushikdutta.quack.QuackContext
import okhttp3.Headers
import okhttp3.OkHttpClient
import okhttp3.Request
import okhttp3.Response
@@ -45,10 +46,13 @@ internal class MangaLoaderContextMock : MangaLoaderContext() {
return SourceConfigMock()
}
suspend fun doRequest(url: String, referer: String? = null): Response {
suspend fun doRequest(url: String, referer: String? = null, extraHeaders: Headers? = null): Response {
val request = Request.Builder()
.get()
.url(url)
if (extraHeaders != null) {
request.headers(extraHeaders)
}
if (referer != null) {
request.header("Referer", referer)
}

View File

@@ -15,12 +15,13 @@ import org.koitharu.kotatsu.test_util.isDistinctBy
import org.koitharu.kotatsu.test_util.isUrlAbsolute
import org.koitharu.kotatsu.test_util.maxDuplicates
@ExtendWith(AuthCheckExtension::class)
internal class MangaParserTest {
private val context = MangaLoaderContextMock()
@ParameterizedTest
@ParameterizedTest(name = "{index}|list|{0}")
@MangaSources
fun list(source: MangaSource) = runTest {
val parser = source.newParser(context)
@@ -29,7 +30,19 @@ internal class MangaParserTest {
assert(list.all { it.source == source })
}
@ParameterizedTest
@ParameterizedTest(name = "{index}|pagination|{0}")
@MangaSources
fun pagination(source: MangaSource) = runTest {
val parser = source.newParser(context)
val page1 = parser.getList(0, sortOrder = null, tags = null)
val page2 = parser.getList(page1.size, sortOrder = null, tags = null)
val intersection = page1.intersect(page2.toSet())
assert(intersection.isEmpty()) {
"Pages are intersected: " + intersection.joinToString { it.publicUrl }
}
}
@ParameterizedTest(name = "{index}|search|{0}")
@MangaSources
fun search(source: MangaSource) = runTest {
val parser = source.newParser(context)
@@ -46,7 +59,7 @@ internal class MangaParserTest {
assert(list.all { it.source == source })
}
@ParameterizedTest
@ParameterizedTest(name = "{index}|tags|{0}")
@MangaSources
fun tags(source: MangaSource) = runTest {
val parser = source.newParser(context)
@@ -66,7 +79,7 @@ internal class MangaParserTest {
assert(list.all { it.source == source })
}
@ParameterizedTest
@ParameterizedTest(name = "{index}|details|{0}")
@MangaSources
fun details(source: MangaSource) = runTest {
val parser = source.newParser(context)
@@ -95,7 +108,7 @@ internal class MangaParserTest {
}
}
@ParameterizedTest
@ParameterizedTest(name = "{index}|pages|{0}")
@MangaSources
fun pages(source: MangaSource) = runTest {
val parser = source.newParser(context)
@@ -115,16 +128,23 @@ internal class MangaParserTest {
checkImageRequest(pageUrl, page.referer)
}
@ParameterizedTest
@ParameterizedTest(name = "{index}|favicon|{0}")
@MangaSources
fun favicon(source: MangaSource) = runTest {
val parser = source.newParser(context)
val faviconUrl = parser.getFaviconUrl()
assert(faviconUrl.isUrlAbsolute())
checkImageRequest(faviconUrl, null)
val favicons = parser.getFavicons()
val types = setOf("png", "svg", "ico", "gif", "jpg", "jpeg")
assert(favicons.isNotEmpty())
favicons.forEach {
assert(it.url.isUrlAbsolute()) { "Favicon url is not absolute: ${it.url}" }
assert(it.type in types) { "Unknown icon type: ${it.type}" }
}
val favicon = favicons.find(24)
checkNotNull(favicon)
checkImageRequest(favicon.url, favicons.referer)
}
@ParameterizedTest
@ParameterizedTest(name = "{index}|domain|{0}")
@MangaSources
fun domain(source: MangaSource) = runTest {
val parser = source.newParser(context)
@@ -133,7 +153,7 @@ internal class MangaParserTest {
.host(defaultDomain)
.scheme("https")
.toString()
val response = context.doRequest(url)
val response = context.doRequest(url, extraHeaders = parser.headers)
val realUrl = response.request.url
val realDomain = realUrl.topPrivateDomain()
val realHost = realUrl.host
@@ -142,7 +162,7 @@ internal class MangaParserTest {
}
}
@ParameterizedTest
@ParameterizedTest(name = "{index}|authorization|{0}")
@MangaSources
@Disabled
fun authorization(source: MangaSource) = runTest {

View File

@@ -0,0 +1,42 @@
package org.koitharu.kotatsu.parsers.util
import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.Test
internal class PaginatorTest {
@Test
fun singlePaginationTest() {
val paginator = Paginator(24)
assertEquals(1, paginator.getPage(0))
assertEquals(2, paginator.getPage(24))
assertEquals(3, paginator.getPage(48))
}
@Test
fun adaptivePaginationTest() {
val paginator = Paginator(12)
assertEquals(1, paginator.getPage(0))
paginator.onListReceived(0, 1, 24)
assertEquals(2, paginator.getPage(24))
paginator.onListReceived(24, 2, 18)
assertEquals(3, paginator.getPage(42))
}
@Test
fun endReachPaginationTest() {
val pageSize = 24
val paginator = Paginator(pageSize)
var size = 0
repeat(5) { i ->
val offset = i * pageSize
assertEquals(i + 1, paginator.getPage(offset))
paginator.onListReceived(offset, i + 1, pageSize)
size += pageSize
}
val nextPage = paginator.getPage(size)
assertEquals(nextPage, paginator.getPage(size))
paginator.onListReceived(size, nextPage, 0)
assertEquals(nextPage, paginator.getPage(size))
}
}

View File

@@ -0,0 +1,2 @@
junit.jupiter.execution.parallel.enabled = true
junit.jupiter.execution.parallel.mode.default = concurrent