From 0731d2487adcea87fff15e6259e3cf4c5aae5984 Mon Sep 17 00:00:00 2001 From: devi Date: Sun, 5 Nov 2023 18:07:12 +0100 Subject: [PATCH 1/5] Minor daily update --- .../kotatsu/parsers/site/en/BeeToon.kt | 136 ++++++++++++++++++ .../kotatsu/parsers/site/en/ManhwasMen.kt | 4 +- .../kotatsu/parsers/site/tr/TrWebtoon.kt | 4 +- 3 files changed, 140 insertions(+), 4 deletions(-) create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/BeeToon.kt diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/BeeToon.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/BeeToon.kt new file mode 100644 index 00000000..9fa1266a --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/BeeToon.kt @@ -0,0 +1,136 @@ +package org.koitharu.kotatsu.parsers.site.en + +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.PagedMangaParser +import org.koitharu.kotatsu.parsers.config.ConfigKey +import org.koitharu.kotatsu.parsers.model.* +import org.koitharu.kotatsu.parsers.util.* +import java.text.SimpleDateFormat +import java.util.* + +@MangaSourceParser("BEETOON", "BeeToon.net", "en") +internal class BeeToon(context: MangaLoaderContext) : + PagedMangaParser(context, MangaSource.BEETOON, pageSize = 30) { + + override val sortOrders: Set = EnumSet.of(SortOrder.UPDATED, SortOrder.POPULARITY) + + override val configKeyDomain = ConfigKey.Domain("ww7.beetoon.net") + + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + val tag = tags.oneOrThrowIfMany() + val url = buildString { + append("https://") + append(domain) + when { + !query.isNullOrEmpty() -> { + if (page > 1) { + return emptyList() + } + append("/?s=") + append(query.urlEncoded()) + } + + !tags.isNullOrEmpty() -> { + append("/genre/") + append(tag?.key.orEmpty()) + append("/page-") + append(page) + append("/") + } + + else -> { + when (sortOrder) { + SortOrder.UPDATED -> append("/latest-update/") + SortOrder.POPULARITY -> append("/popular-manga/") + else -> append("/latest-update/") + } + append("page-") + append(page) + append("/") + } + } + } + val doc = webClient.httpGet(url).parseHtml() + return doc.select(".comics-grid .entry").map { div -> + val href = div.selectFirstOrThrow("a").attrAsRelativeUrl("href") + Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + coverUrl = div.selectFirst("img")?.src().orEmpty(), + title = div.selectFirst(".name")?.text().orEmpty(), + altTitle = null, + rating = div.selectFirst(".counter")?.text()?.toFloatOrNull()?.div(5f) ?: RATING_UNKNOWN, + tags = emptySet(), + author = null, + state = when (div.selectLastOrThrow(".status span").text()) { + "Ongoing" -> MangaState.ONGOING + "Completed" -> MangaState.FINISHED + else -> null + }, + source = source, + isNsfw = isNsfwSource, + ) + } + } + + override suspend fun getTags(): Set { + val doc = webClient.httpGet("https://$domain/").parseHtml() + return doc.requireElementById("menu-item-3").select("ul.sub-menu li a").mapNotNullToSet { + MangaTag( + key = it.attr("href").removeSuffix('/').substringAfterLast('/'), + title = it.text(), + source = source, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + return manga.copy( + description = doc.getElementById("desc")?.text().orEmpty(), + rating = doc.selectFirst(".counter")?.text()?.toFloatOrNull()?.div(5f) ?: RATING_UNKNOWN, + tags = doc.body().select(".info .genre a").mapNotNullToSet { + MangaTag( + key = it.attr("href").removeSuffix('/').substringAfterLast('/'), + title = it.text(), + source = source, + ) + }, + author = doc.selectFirst(".info .author a")?.text(), + chapters = doc.select(".items-chapters a").mapChapters(reversed = true) { i, a -> + val url = a.attrAsRelativeUrl("href").toAbsoluteUrl(domain) + MangaChapter( + id = generateUid(url), + name = a.selectFirstOrThrow(".chap").text(), + number = i + 1, + url = url, + scanlator = null, + uploadDate = SimpleDateFormat("MM/dd/yyyy HH:mm:ss", Locale.ENGLISH) + .tryParse(a.selectFirst(".chapter-date")?.attr("title") ?: "0"), + branch = null, + source = source, + ) + }, + ) + } + + override suspend fun getPages(chapter: MangaChapter): List { + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + return doc.select(".chapter-content-inner center img").map { img -> + val urlPage = img.src()?.toRelativeUrl(domain) ?: img.parseFailed("Image src not found") + MangaPage( + id = generateUid(urlPage), + url = urlPage, + preview = null, + source = source, + ) + } + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/ManhwasMen.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/ManhwasMen.kt index 16d82661..f150d9d1 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/ManhwasMen.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/ManhwasMen.kt @@ -77,9 +77,9 @@ class ManhwasMen(context: MangaLoaderContext) : } } - override suspend fun getDetails(manga: Manga): Manga = coroutineScope { + override suspend fun getDetails(manga: Manga): Manga { val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - manga.copy( + return manga.copy( tags = doc.body().select(".genres a").mapNotNullToSet { a -> MangaTag( key = a.attr("href").substringAfterLast('='), diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tr/TrWebtoon.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tr/TrWebtoon.kt index 0139c318..8c8e129b 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tr/TrWebtoon.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tr/TrWebtoon.kt @@ -89,9 +89,9 @@ class TrWebtoon(context: MangaLoaderContext) : } } - override suspend fun getDetails(manga: Manga): Manga = coroutineScope { + override suspend fun getDetails(manga: Manga): Manga { val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - manga.copy( + return manga.copy( tags = doc.body().select("li.movie__year a").mapNotNullToSet { a -> MangaTag( key = a.attr("href").substringAfterLast('='), From 9d3671e117ba46a606e8fe8f61adf03b692e9955 Mon Sep 17 00:00:00 2001 From: devi Date: Sun, 5 Nov 2023 18:07:37 +0100 Subject: [PATCH 2/5] Minor daily update --- .../kotlin/org/koitharu/kotatsu/parsers/site/en/ManhwasMen.kt | 1 - .../kotlin/org/koitharu/kotatsu/parsers/site/tr/TrWebtoon.kt | 1 - 2 files changed, 2 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/ManhwasMen.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/ManhwasMen.kt index f150d9d1..11904ea1 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/ManhwasMen.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/ManhwasMen.kt @@ -1,6 +1,5 @@ package org.koitharu.kotatsu.parsers.site.en -import kotlinx.coroutines.coroutineScope import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaSourceParser import org.koitharu.kotatsu.parsers.PagedMangaParser diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tr/TrWebtoon.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tr/TrWebtoon.kt index 8c8e129b..525322b3 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tr/TrWebtoon.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tr/TrWebtoon.kt @@ -1,6 +1,5 @@ package org.koitharu.kotatsu.parsers.site.tr -import kotlinx.coroutines.coroutineScope import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaSourceParser import org.koitharu.kotatsu.parsers.PagedMangaParser From 3e69b8851872640115be0f0f03ef0e1f27891940 Mon Sep 17 00:00:00 2001 From: Koitharu Date: Mon, 6 Nov 2023 10:42:04 +0200 Subject: [PATCH 3/5] Update dependencies --- build.gradle | 22 +++++++++++----------- kotatsu-parsers-ksp/build.gradle | 2 +- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/build.gradle b/build.gradle index 3a47f839..418aef94 100644 --- a/build.gradle +++ b/build.gradle @@ -2,8 +2,8 @@ import tasks.ReportGenerateTask plugins { id 'java-library' - id 'org.jetbrains.kotlin.jvm' version '1.9.0' - id 'com.google.devtools.ksp' version '1.9.0-1.0.13' + id 'org.jetbrains.kotlin.jvm' version '1.9.20' + id 'com.google.devtools.ksp' version '1.9.20-1.0.14' id 'maven-publish' } @@ -54,18 +54,18 @@ afterEvaluate { dependencies { implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-core:1.7.3' - implementation 'com.squareup.okhttp3:okhttp:4.11.0' - implementation 'com.squareup.okio:okio:3.3.0' - api 'org.jsoup:jsoup:1.16.1' - implementation 'org.json:json:20230618' - implementation 'androidx.collection:collection-ktx:1.2.0' + implementation 'com.squareup.okhttp3:okhttp:4.12.0' + implementation 'com.squareup.okio:okio:3.6.0' + api 'org.jsoup:jsoup:1.16.2' + implementation 'org.json:json:20231013' + implementation 'androidx.collection:collection-ktx:1.3.0' ksp project(':kotatsu-parsers-ksp') - testImplementation 'org.junit.jupiter:junit-jupiter-api:5.9.3' - testImplementation 'org.junit.jupiter:junit-jupiter-engine:5.9.3' - testImplementation 'org.junit.jupiter:junit-jupiter-params:5.9.3' - testImplementation 'org.jetbrains.kotlinx:kotlinx-coroutines-test:1.7.1' + testImplementation 'org.junit.jupiter:junit-jupiter-api:5.10.0' + testImplementation 'org.junit.jupiter:junit-jupiter-engine:5.10.0' + testImplementation 'org.junit.jupiter:junit-jupiter-params:5.10.0' + testImplementation 'org.jetbrains.kotlinx:kotlinx-coroutines-test:1.7.3' testImplementation 'io.webfolder:quickjs:1.1.0' } diff --git a/kotatsu-parsers-ksp/build.gradle b/kotatsu-parsers-ksp/build.gradle index 4e291ba1..87e55c15 100644 --- a/kotatsu-parsers-ksp/build.gradle +++ b/kotatsu-parsers-ksp/build.gradle @@ -3,5 +3,5 @@ plugins { } dependencies { - implementation 'com.google.devtools.ksp:symbol-processing-api:1.9.0-1.0.13' + implementation 'com.google.devtools.ksp:symbol-processing-api:1.9.20-1.0.14' } From 58c9d14d42afa20bdd94cfd2d12faecc8506e413 Mon Sep 17 00:00:00 2001 From: devi Date: Mon, 6 Nov 2023 20:24:12 +0100 Subject: [PATCH 4/5] Minor daily update --- .../{en/TempleScan.kt => es/TempleScanEsp.kt} | 13 ++++++------- .../kotatsu/parsers/site/heancms/HeanCms.kt | 8 +++++--- .../kotatsu/parsers/site/heancms/en/TempleScan.kt | 12 ++++++++++++ .../parsers/site/madara/es/TempleScanEsp.kt | 15 --------------- 4 files changed, 23 insertions(+), 25 deletions(-) rename src/main/kotlin/org/koitharu/kotatsu/parsers/site/{en/TempleScan.kt => es/TempleScanEsp.kt} (91%) create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/heancms/en/TempleScan.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/es/TempleScanEsp.kt diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/TempleScan.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/es/TempleScanEsp.kt similarity index 91% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/TempleScan.kt rename to src/main/kotlin/org/koitharu/kotatsu/parsers/site/es/TempleScanEsp.kt index 1c35a50b..6df4def3 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/TempleScan.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/es/TempleScanEsp.kt @@ -1,4 +1,4 @@ -package org.koitharu.kotatsu.parsers.site.en +package org.koitharu.kotatsu.parsers.site.es import kotlinx.coroutines.coroutineScope import okhttp3.Headers @@ -12,13 +12,13 @@ import org.koitharu.kotatsu.parsers.network.UserAgents import org.koitharu.kotatsu.parsers.util.* import java.util.* -@MangaSourceParser("TEMPLESCAN", "TempleScan", "en") -internal class TempleScan(context: MangaLoaderContext) : - PagedMangaParser(context, MangaSource.TEMPLESCAN, pageSize = 15) { +@MangaSourceParser("TEMPLESCANESP", "TempleScanEsp", "es", ContentType.HENTAI) +internal class TempleScanEsp(context: MangaLoaderContext) : + PagedMangaParser(context, MangaSource.TEMPLESCANESP, pageSize = 15) { override val sortOrders: Set = EnumSet.of(SortOrder.NEWEST, SortOrder.UPDATED) - override val configKeyDomain = ConfigKey.Domain("templescan.net") + override val configKeyDomain = ConfigKey.Domain("templescanesp.net") override val headers: Headers = Headers.Builder() .add("User-Agent", UserAgents.CHROME_DESKTOP) @@ -61,7 +61,7 @@ internal class TempleScan(context: MangaLoaderContext) : author = null, state = null, source = source, - isNsfw = false, + isNsfw = isNsfwSource, ) } } @@ -75,7 +75,6 @@ internal class TempleScan(context: MangaLoaderContext) : manga.copy( description = doc.requireElementById("section-sinopsis").html(), chapters = chaptersDeferred, - isNsfw = false, ) } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/heancms/HeanCms.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/heancms/HeanCms.kt index 18a9e23b..59966096 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/heancms/HeanCms.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/heancms/HeanCms.kt @@ -31,6 +31,8 @@ internal abstract class HeanCms( .add("User-Agent", UserAgents.CHROME_DESKTOP) .build() + protected open val pathManga = "series" + //For some sources, you need to send a json. For the moment, this part only works in Get. ( ex source need json gloriousscan.com , omegascans.org ) override suspend fun getListPage( page: Int, @@ -78,7 +80,7 @@ internal abstract class HeanCms( val json = webClient.httpGet(url).parseJson() return json.getJSONArray("data").mapJSON { j -> val slug = j.getString("series_slug") - val urlManga = "https://$domain/series/$slug" + val urlManga = "https://$domain/$pathManga/$slug" val cover = if (j.getString("thumbnail").contains('/')) { j.getString("thumbnail") } else { @@ -120,13 +122,13 @@ internal abstract class HeanCms( .drop(1) return manga.copy( - altTitle = root.selectFirstOrThrow("p.text-center.text-gray-400").text(), + altTitle = root.selectFirst("p.text-center.text-gray-400")?.text(), tags = emptySet(), author = root.select("div.flex.flex-col.gap-y-2 p:contains(Autor:) strong").text(), description = root.selectFirst("h5:contains(Desc) + .bg-gray-800")?.html(), chapters = chapter.mapChapters(reversed = true) { i, it -> val slugChapter = it.substringAfter("chapter_slug\":\"").substringBefore("\",\"") - val url = "https://$domain/series/$slug/$slugChapter" + val url = "https://$domain/$pathManga/$slug/$slugChapter" val date = it.substringAfter("created_at\":\"").substringBefore("\",\"").substringBefore("T") val name = slugChapter.replace("-", " ") MangaChapter( diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/heancms/en/TempleScan.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/heancms/en/TempleScan.kt new file mode 100644 index 00000000..e6ffc2e0 --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/heancms/en/TempleScan.kt @@ -0,0 +1,12 @@ +package org.koitharu.kotatsu.parsers.site.heancms.en + +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.model.MangaSource +import org.koitharu.kotatsu.parsers.site.heancms.HeanCms + +@MangaSourceParser("TEMPLESCAN", "TempleScan", "en") +internal class TempleScan(context: MangaLoaderContext) : + HeanCms(context, MangaSource.TEMPLESCAN, "templescan.net") { + override val pathManga = "comic" +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/es/TempleScanEsp.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/es/TempleScanEsp.kt deleted file mode 100644 index ffccc7d2..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/es/TempleScanEsp.kt +++ /dev/null @@ -1,15 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.madara.es - -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.ContentType -import org.koitharu.kotatsu.parsers.model.MangaSource -import org.koitharu.kotatsu.parsers.site.madara.MadaraParser - -@MangaSourceParser("TEMPLESCANESP", "TempleScanEsp", "es", ContentType.HENTAI) -internal class TempleScanEsp(context: MangaLoaderContext) : - MadaraParser(context, MangaSource.TEMPLESCANESP, "templescanesp.com") { - override val listUrl = "series/" - override val tagPrefix = "genero/" - override val datePattern = "dd.MM.yyyy" -} From 154ae09c6e44e60d5fb640313108622f729db73d Mon Sep 17 00:00:00 2001 From: Koitharu Date: Tue, 7 Nov 2023 18:33:08 +0200 Subject: [PATCH 5/5] [MangaReader] Add NetShield bypass (testing) #198 --- .../site/mangareader/MangaReaderParser.kt | 26 ++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt index 59f5560b..d34ad65c 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt @@ -1,8 +1,13 @@ package org.koitharu.kotatsu.parsers.site.mangareader import androidx.collection.ArrayMap +import kotlinx.coroutines.runBlocking import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock +import okhttp3.Cookie +import okhttp3.HttpUrl.Companion.toHttpUrl +import okhttp3.Interceptor +import okhttp3.Response import org.json.JSONObject import org.jsoup.nodes.Document import org.koitharu.kotatsu.parsers.MangaLoaderContext @@ -19,7 +24,7 @@ internal abstract class MangaReaderParser( domain: String, pageSize: Int, searchPageSize: Int, -) : PagedMangaParser(context, source, pageSize, searchPageSize) { +) : PagedMangaParser(context, source, pageSize, searchPageSize), Interceptor { override val configKeyDomain = ConfigKey.Domain(domain) @@ -296,4 +301,23 @@ internal abstract class MangaReaderParser( tagCache = tagMap return@withLock tagMap } + + override fun intercept(chain: Interceptor.Chain): Response { + val response = chain.proceed(chain.request()) + if (context.cookieJar.getCookies(domain).none { it.name.contains("NetShield") }) { + val cookie = runBlocking { response.parseHtml().getNetShieldCookie() } ?: return response + context.cookieJar.insertCookie(domain, cookie) + return chain.proceed(response.request.newBuilder().build()) + } + return response + } + + private suspend fun Document.getNetShieldCookie(): Cookie? { + val script = select("script").firstNotNullOfOrNull { s -> + s.html().takeIf { x -> x.contains("slowAES.decrypt") } + } ?: return null + val min = webClient.httpGet("https://$domain/min.js").parseRaw() + val res = context.evaluateJs(min + "\n\n" + script.replace("document.cookie =", "return")) + return Cookie.parse(baseUri().toHttpUrl(), res ?: return null) + } }