Compare commits

..

48 Commits

Author SHA1 Message Date
Koitharu 071f4f0911
Trigger rebuild 2 years ago
Koitharu 813c5236b3
[Grouple] Back to the old chapters parsing behavior with option 2 years ago
Koitharu b7613606c0
[TuMangaOnline] Fix regexp 2 years ago
Koitharu 0aa4ea01f7
[Grouple] Multiple translations support 2 years ago
Koitharu 20685598e3
[TuMangaOnline] Fix pages parsing 2 years ago
Koitharu b1fb1bdc6b
Merge pull request #615 from NagaYZ/feature-search-by-number
Feature search by number
2 years ago
Naga a80bdcc611 full title name 2 years ago
Naga 4571e3e001 gitignore 2 years ago
Naga 38d7d97167 added search by id 2 years ago
Koitharu 3913e95b53
Merge pull request #608 from nghduc97/fix-mangabox-parser-white-spaces
fix mangaboxparser white space coding in search query
2 years ago
Koitharu 954c926a9f
Merge pull request #588 from NagaYZ/feature-webtoon-original
Feature webtoon original
2 years ago
nghduc97 0973444c47
fix mangaboxparser white space coding in search query 2 years ago
Naga 368b61c21a fixed suggestions 2 years ago
Naga efc9b3502c fixed suggestions 2 years ago
Naga 2e86c480ec
Update src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt
Co-authored-by: Koitharu <nvasya95@gmail.com>
2 years ago
Naga 39eeaacdcb fixup! trying to improve speed 2 years ago
Naga ea45e91aa7 trying to improve speed 2 years ago
Naga a65e41e796 fix 2 years ago
Naga 91648756c9 added more language 2 years ago
Naga a225234800 fixed chapter date 2 years ago
Naga de58333dc4 fixup! fixup! fixed offset 2 years ago
Naga 24a25e3dbf fixup! fixed offset 2 years ago
Naga 8e5ae51067 fixed offset 2 years ago
Naga 86c981ae8e fixup! trying to speed up loading 2 years ago
Naga b716c35438 trying to speed up loading 2 years ago
Naga 9e984510d4 added caching for title list to speed up loading 2 years ago
Naga 3c08119ccc added caching for title list to speed up loading 2 years ago
Naga ada1b7b54c fixed sort descending 2 years ago
Naga a7ed4fbcc3 fix import 2 years ago
Naga 1330feaea5 added webtoon original english support 2 years ago
Koitharu 103f578c61
Merge pull request #579 from NagaYZ/master
Update domains name
2 years ago
Koitharu deee80ae71
Merge pull request #581 from NagaYZ/fix-popular-page-url
Fix popular page url
2 years ago
Naga 54350040ad fixed popular page url 2 years ago
Naga 3472b8ba00 Update ZinManga.kt 2 years ago
Naga f2dd9af66c Update MagusManga.kt
fixed domain name and listurl acces
2 years ago
Naga da46f7967a fixed bunch of domain name 2 years ago
Naga ed3d6c5f98 Replaced GloryScans/GloryScansX with LunarScans/LunarHentai
domain changed
2 years ago
Naga c4ea825df9 Replaced LightScans with Luascans , domain name changed 2 years ago
Naga 4120e3946c Update RagnarokScanlation.kt
new domain: ragnarokscanlation.net
2 years ago
Naga c84b6acff2 Update EmperorScan.kt
new domain : emperorscan.com
2 years ago
Naga 159110c032 Update ResetScans.kt
new domain: reset-scans.xyz
2 years ago
Naga 93e1b38da2 Update MangaEsp.kt
new domain : mangaesp.net
2 years ago
Koitharu 3ff028c4e9
Merge pull request #572 from geexuan/baozimh_details_fix
Baozimh: Fix for not getting all chapter details
2 years ago
geexuans 9b46963015 Baozimh: Fix for not getting all chapter details 2 years ago
Koitharu 09e1c14f37
Merge pull request #565 from sandeepjak2007/Issue##561
Issue #561 Leading to non existing page
2 years ago
Sandeep Karanam 58af01b90a Issue #561 Leading to non existing page
Issue #561 fixed
2 years ago
Koitharu e338237e2a
[NudeMoon] Fix pages loading 2 years ago
Koitharu 014ea5ef49
[Manganato] Update domain 2 years ago

1
.gitignore vendored

@ -4,6 +4,7 @@
.idea/**/usage.statistics.xml
.idea/**/dictionaries
.idea/**/shelf
.idea/**/copilot
# Generated files
.idea/**/contentModel.xml

@ -25,4 +25,8 @@ sealed class ConfigKey<T>(
class UserAgent(
override val defaultValue: String,
) : ConfigKey<String>("user_agent")
class SplitByTranslations(
override val defaultValue: Boolean,
) : ConfigKey<Boolean>("split_translations")
}

@ -109,7 +109,7 @@ class Manga(
largeCoverUrl = largeCoverUrl,
description = description,
chapters = chapters,
source = source,
source = source
)
override fun equals(other: Any?): Boolean {

@ -0,0 +1,358 @@
package org.koitharu.kotatsu.parsers.site.all
import kotlinx.coroutines.async
import kotlinx.coroutines.awaitAll
import kotlinx.coroutines.coroutineScope
import okhttp3.Headers
import okhttp3.HttpUrl
import okhttp3.HttpUrl.Companion.toHttpUrl
import org.json.JSONArray
import org.json.JSONObject
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.exception.NotFoundException
import org.koitharu.kotatsu.parsers.exception.ParseException
import org.koitharu.kotatsu.parsers.model.ContentType
import org.koitharu.kotatsu.parsers.model.Manga
import org.koitharu.kotatsu.parsers.model.MangaChapter
import org.koitharu.kotatsu.parsers.model.MangaListFilter
import org.koitharu.kotatsu.parsers.model.MangaPage
import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.model.MangaTag
import org.koitharu.kotatsu.parsers.model.RATING_UNKNOWN
import org.koitharu.kotatsu.parsers.model.SortOrder
import org.koitharu.kotatsu.parsers.util.SoftSuspendLazy
import org.koitharu.kotatsu.parsers.util.SuspendLazy
import org.koitharu.kotatsu.parsers.util.domain
import org.koitharu.kotatsu.parsers.util.generateUid
import org.koitharu.kotatsu.parsers.util.json.getBooleanOrDefault
import org.koitharu.kotatsu.parsers.util.json.getFloatOrDefault
import org.koitharu.kotatsu.parsers.util.json.getIntOrDefault
import org.koitharu.kotatsu.parsers.util.json.getStringOrNull
import org.koitharu.kotatsu.parsers.util.json.mapJSON
import org.koitharu.kotatsu.parsers.util.json.mapJSONIndexed
import org.koitharu.kotatsu.parsers.util.mapChapters
import org.koitharu.kotatsu.parsers.util.oneOrThrowIfMany
import org.koitharu.kotatsu.parsers.util.parseJson
import org.koitharu.kotatsu.parsers.util.splitTwoParts
import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl
import org.koitharu.kotatsu.parsers.util.urlEncoded
import java.util.Calendar
import java.util.EnumSet
import javax.crypto.Mac
import javax.crypto.spec.SecretKeySpec
internal abstract class WebtoonsParser(
context: MangaLoaderContext,
source: MangaSource,
) : MangaParser(context, source) {
override val isMultipleTagsSupported = false
private val signer by lazy {
WebtoonsUrlSigner("gUtPzJFZch4ZyAGviiyH94P99lQ3pFdRTwpJWDlSGFfwgpr6ses5ALOxWHOIT7R1")
}
// we don't __really__ support changing this domain because:
// 1. I don't think other websites have this exact API
// 2. most communication is done with other domains (hosting API and static content), which are not configurable
// 3. we rely on the HTTP client setting the referer header to webtoons.com
//
// This effectively means that changing the domain will break the source. Yikes
override val configKeyDomain = ConfigKey.Domain("webtoons.com")
private val apiDomain = "global.apis.naver.com"
private val staticDomain = "webtoon-phinf.pstatic.net"
override val availableSortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.POPULARITY, // views
SortOrder.RATING, // star rating
//SortOrder.LIKE, // likes
SortOrder.UPDATED,
)
override val headers: Headers
get() = Headers.Builder().add("User-Agent", "nApps (Android 12;; linewebtoon; 3.1.0)").build()
override suspend fun getPageUrl(page: MangaPage): String {
return page.url.toAbsoluteUrl(staticDomain)
}
// some language tags do not map perfectly to the ones used by the API
private val languageCode: String
get() = when (val tag = sourceLocale.toLanguageTag()) {
"in" -> "id"
"zh" -> "zh-hant"
else -> tag
}
private suspend fun fetchEpisodes(titleNo: Long): List<MangaChapter> = coroutineScope {
val firstResult =
makeRequest("/lineWebtoon/webtoon/episodeList.json?v=5&titleNo=$titleNo&startIndex=0&pageSize=30")
val totalEpisodeCount = firstResult.getJSONObject("episodeList").getInt("totalServiceEpisodeCount")
val episodes = firstResult.getJSONObject("episodeList").getJSONArray("episode").toJSONList().toMutableList()
val additionalEpisodes = (episodes.size until totalEpisodeCount step 30).map { startIndex ->
async {
makeRequest("/lineWebtoon/webtoon/episodeList.json?v=5&titleNo=$titleNo&startIndex=$startIndex&pageSize=30").getJSONObject(
"episodeList",
).getJSONArray("episode").toJSONList()
}
}.awaitAll().flatten()
episodes.addAll(additionalEpisodes)
// Optimize object creation and sorting
episodes.mapChapters { i, jo ->
MangaChapter(
id = generateUid("$titleNo-$i"),
name = jo.getString("episodeTitle"),
number = jo.getInt("episodeSeq"),
url = "$titleNo-${jo.get("episodeNo")}",
uploadDate = jo.getLong("registerYmdt"),
branch = null,
scanlator = null,
source = source,
)
}.sortedBy(MangaChapter::number)
}
private fun JSONArray.toJSONList(): List<JSONObject> {
val list = mutableListOf<JSONObject>()
for (i in 0 until length()) {
list.add(getJSONObject(i))
}
return list
}
override suspend fun getDetails(manga: Manga): Manga = coroutineScope {
val titleNo = manga.url.toLong()
val chaptersDeferred = async { fetchEpisodes(titleNo) }
val chapters = chaptersDeferred.await()
makeRequest("/lineWebtoon/webtoon/titleInfo.json?titleNo=${titleNo}&anyServiceStatus=false").getJSONObject("titleInfo")
.let { jo ->
MangaWebtoon(
Manga(
id = generateUid(titleNo),
title = jo.getString("title"),
altTitle = null,
url = "$titleNo",
publicUrl = "https://$domain/$languageCode/originals/a/list?title_no=${titleNo}",
rating = jo.getFloatOrDefault("starScoreAverage", -10f) / 10f,
isNsfw = jo.getBooleanOrDefault("ageGradeNotice", isNsfwSource),
coverUrl = jo.getString("thumbnail").toAbsoluteUrl(staticDomain),
largeCoverUrl = jo.getStringOrNull("thumbnailVertical")?.toAbsoluteUrl(staticDomain),
tags = setOf(parseTag(jo.getJSONObject("genreInfo"))),
author = jo.getStringOrNull("writingAuthorName"),
description = jo.getString("synopsis"),
// I don't think the API provides this info,
state = null,
chapters = chapters,
source = source,
),
date = jo.getLong("lastEpisodeRegisterYmdt"),
readCount = jo.getLong("readCount"),
//likeCount = jo.getLong("likeitCount"),
).manga
}
}
private val allGenreCache = SuspendLazy {
makeRequest("/lineWebtoon/webtoon/genreList.json").getJSONObject("genreList").getJSONArray("genres")
.mapJSON { jo -> parseTag(jo) }.associateBy { tag -> tag.key }
}
private val allTitleCache = SoftSuspendLazy {
makeRequest("/lineWebtoon/webtoon/titleList.json?").getJSONObject("titleList").getJSONArray("titles")
.mapJSON { jo ->
val titleNo = jo.getLong("titleNo")
MangaWebtoon(
Manga(
id = generateUid(titleNo),
url = titleNo.toString(),
publicUrl = "https://$domain/$languageCode/originals/a/list?title_no=$titleNo",
title = jo.getString("title"),
coverUrl = jo.getString("thumbnail").toAbsoluteUrl(staticDomain),
altTitle = null,
author = jo.getStringOrNull("writingAuthorName"),
isNsfw = jo.getBooleanOrDefault("ageGradeNotice", isNsfwSource),
rating = jo.getFloatOrDefault("starScoreAverage", -10f) / 10f,
tags = setOfNotNull(allGenreCache.get()[jo.getString("representGenre")]),
description = jo.getString("synopsis"),
state = null,
source = source,
),
date = jo.getLong("lastEpisodeRegisterYmdt"),
readCount = jo.getLong("readCount"),
//likeCount = jo.getLong("likeitCount"),
)
}
}
private suspend fun getAllGenreList(): Map<String, MangaTag> {
return allGenreCache.get()
}
private suspend fun getAllTitleList(): List<MangaWebtoon> {
return allTitleCache.get()
}
override suspend fun getList(offset: Int, filter: MangaListFilter?): List<Manga> {
val webtoons = when (filter) {
is MangaListFilter.Search -> {
makeRequest("/lineWebtoon/webtoon/searchWebtoon?query=${filter.query.urlEncoded()}").getJSONObject("webtoonSearch")
.getJSONArray("titleList").mapJSON { jo ->
val titleNo = jo.getLong("titleNo")
MangaWebtoon(
Manga(
id = generateUid(titleNo),
title = jo.getString("title"),
altTitle = null,
url = titleNo.toString(),
publicUrl = "https://$domain/$languageCode/originals/a/list?title_no=$titleNo",
rating = RATING_UNKNOWN,
isNsfw = isNsfwSource,
coverUrl = jo.getString("thumbnail").toAbsoluteUrl(staticDomain),
largeCoverUrl = null,
tags = emptySet(),
author = jo.getStringOrNull("writingAuthorName"),
description = null,
state = null,
source = source,
))
}
}
is MangaListFilter.Advanced -> {
val genre = filter.tags.oneOrThrowIfMany()?.key ?: "ALL"
val genres = getAllGenreList()
val result = getAllTitleList()
val sortedResult = when (filter.sortOrder) {
SortOrder.UPDATED -> result.sortedBy { it.date }
SortOrder.POPULARITY -> result.sortedByDescending { it.readCount }
SortOrder.RATING -> result.sortedByDescending { it.manga.rating }
//SortOrder.LIKE -> result.sortedBy { it.likeitCount }
else -> throw IllegalArgumentException("Unsupported sort order: ${filter.sortOrder}")
}
if (genre != "ALL") {
sortedResult.filter { it.manga.tags.contains(genres[genre]) }
} else {
sortedResult
}
}
else -> getAllTitleList()
}
return webtoons.map { it.manga }.subList(offset, (offset + 20).coerceAtMost(webtoons.size))
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val (titleNo, episodeNo) = requireNotNull(chapter.url.splitTwoParts('-'))
return makeRequest("/lineWebtoon/webtoon/episodeInfo.json?v=4&titleNo=$titleNo&episodeNo=$episodeNo").getJSONObject(
"episodeInfo",
).getJSONArray("imageInfo").mapJSONIndexed { i, jo ->
MangaPage(
id = generateUid("$titleNo-$episodeNo-$i"),
url = jo.getString("url"),
preview = null,
source = source,
)
}
}
private fun parseTag(jo: JSONObject): MangaTag {
return MangaTag(
title = jo.getString("name"),
key = jo.getString("code"),
source = source,
)
}
override suspend fun getAvailableTags(): Set<MangaTag> {
return getAllGenreList().values.toSet()
}
private suspend fun makeRequest(url: String): JSONObject {
val resp = webClient.httpGet(finalizeUrl(url))
val message: JSONObject? = resp.parseJson().optJSONObject("message")
return when (resp.code) {
in 200..299 -> checkNotNull(message).getJSONObject("result")
404 -> throw NotFoundException(message?.getStringOrNull("message").orEmpty(), url)
else -> {
val code = message?.getIntOrDefault("code", 0)
val errorMessage = message?.getStringOrNull("message")
throw ParseException("Api error (code=$code): $errorMessage", url)
}
}
}
private fun finalizeUrl(url: String): HttpUrl {
val httpUrl = url.toAbsoluteUrl(apiDomain).toHttpUrl()
val builder = httpUrl.newBuilder().addQueryParameter("serviceZone", "GLOBAL")
if (httpUrl.queryParameter("v") == null) {
builder.addQueryParameter("v", "1")
}
builder.addQueryParameter("language", languageCode).addQueryParameter("locale", "languageCode")
.addQueryParameter("platform", "APP_ANDROID")
signer.makeEncryptUrl(builder)
return builder.build()
}
@MangaSourceParser("WEBTOONS_EN", "Webtoons English", "en", type = ContentType.MANGA)
class English(context: MangaLoaderContext) : WebtoonsParser(context, MangaSource.WEBTOONS_EN)
@MangaSourceParser("WEBTOONS_ID", "Webtoons Indonesia", "id", type = ContentType.MANGA)
class Indonesian(context: MangaLoaderContext) : WebtoonsParser(context, MangaSource.WEBTOONS_ID)
@MangaSourceParser("WEBTOONS_ES", "Webtoons Spanish", "es", type = ContentType.MANGA)
class Spanish(context: MangaLoaderContext) : WebtoonsParser(context, MangaSource.WEBTOONS_ES)
@MangaSourceParser("WEBTOONS_FR", "Webtoons French", "fr", type = ContentType.MANGA)
class French(context: MangaLoaderContext) : WebtoonsParser(context, MangaSource.WEBTOONS_FR)
@MangaSourceParser("WEBTOONS_TH", "Webtoons Thai", "th", type = ContentType.MANGA)
class Thai(context: MangaLoaderContext) : WebtoonsParser(context, MangaSource.WEBTOONS_TH)
@MangaSourceParser("WEBTOONS_ZH", "Webtoons Chinese", "zh", type = ContentType.MANGA)
class Chinese(context: MangaLoaderContext) : LineWebtoonsParser(context, MangaSource.WEBTOONS_ZH)
@MangaSourceParser("WEBTOONS_DE", "Webtoons German", "de", type = ContentType.MANGA)
class German(context: MangaLoaderContext) : LineWebtoonsParser(context, MangaSource.WEBTOONS_DE)
private inner class WebtoonsUrlSigner(private val secret: String) {
private val mac = Mac.getInstance("HmacSHA1").apply {
this.init(SecretKeySpec(secret.encodeToByteArray(), "HmacSHA1"))
}
private fun getMessage(url: String, msgpad: String): String {
return url.substring(0, 0xFF.coerceAtMost(url.length)) + msgpad
}
private fun getMessageDigest(s: String): String {
val signedMessage = synchronized(mac) { mac.doFinal(s.toByteArray()) }
return context.encodeBase64(signedMessage)
}
fun makeEncryptUrl(urlBuilder: HttpUrl.Builder) {
val msgPad = Calendar.getInstance().timeInMillis.toString()
val digest = getMessageDigest(getMessage(urlBuilder.build().toString(), msgPad))
urlBuilder.addQueryParameter("msgpad", msgPad).addQueryParameter("md", digest)
// .addEncodedQueryParameter("md", digest.urlEncoded())
}
}
private inner class MangaWebtoon(
val manga: Manga,
@JvmField val date: Long? = null,
@JvmField val readCount: Long? = null,
)
}

@ -20,7 +20,7 @@ internal class TeamXNovel(context: MangaLoaderContext) : PagedMangaParser(contex
override val availableStates: Set<MangaState> =
EnumSet.of(MangaState.ONGOING, MangaState.FINISHED, MangaState.ABANDONED)
override val configKeyDomain = ConfigKey.Domain("teamxnovel.com")
override val configKeyDomain = ConfigKey.Domain("team11x11.com")
override val isMultipleTagsSupported = false
override suspend fun getListPage(page: Int, filter: MangaListFilter?): List<Manga> {

@ -210,7 +210,9 @@ class TuMangaOnlineParser(context: MangaLoaderContext) : PagedMangaParser(
if (script1 != null) {
val data = script1.data()
val regexParams = """\{uniqid:'(.+)',cascade:(.+)\}""".toRegex()
@Suppress("RegExpRedundantEscape")
val regexParams = """\{uniqid:\s*'(\S+)',\s*cascade:\s*(\S+)\}""".toRegex()
val regexAction = """form\.action\s?=\s?'(.+)'""".toRegex()
val params = regexParams.find(data)!!
val action = regexAction.find(data)!!.groupValues[1].toHttpUrl()

@ -1,6 +1,7 @@
package org.koitharu.kotatsu.parsers.site.galleryadults.all
import org.jsoup.internal.StringUtil
import org.jsoup.nodes.Document
import org.jsoup.nodes.Element
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
@ -35,9 +36,18 @@ internal class NHentaiParser(context: MangaLoaderContext) :
when (filter) {
is MangaListFilter.Search -> {
append("/search/?q=pages:>0 ")
append(filter.query.urlEncoded())
append("&")
// Check if the query is all numbers
val numericQuery = filter.query.trim()
if (numericQuery.matches("\\d+".toRegex())) {
val title = fetchMangaTitle("$this/g/$numericQuery/")
append("/search/?q=pages:>0 ")
append(title)
append("&")
} else {
append("/search/?q=pages:>0 ")
append(filter.query.urlEncoded())
append("&")
}
}
is MangaListFilter.Advanced -> {
@ -55,19 +65,21 @@ internal class NHentaiParser(context: MangaLoaderContext) :
}
append("/")
if (filter.sortOrder == SortOrder.POPULARITY) {
append("popular/")
append("popular")
}
if(page > 1){
append("?")
}
append("?")
} else if (filter.locale != null) {
append("/language/")
append(filter.locale.toLanguagePath())
append("/")
if (filter.sortOrder == SortOrder.POPULARITY) {
append("popular/")
append("popular")
}
if(page > 1){
append("?")
}
append("?")
} else {
if (filter.sortOrder == SortOrder.POPULARITY) {
append("/?sort=popular&")
@ -79,12 +91,39 @@ internal class NHentaiParser(context: MangaLoaderContext) :
null -> append("/?")
}
append("page=")
append(page.toString())
if(page > 1){
append("page=")
append(page.toString())
}
}
return parseMangaList(webClient.httpGet(url).parseHtml())
}
private suspend fun fetchMangaTitle(url: String): String {
val doc = webClient.httpGet(url).parseHtml()
return doc.selectFirstOrThrow("h1.title").text().trim()
}
override fun parseMangaList(doc: Document): List<Manga> {
return doc.select(selectGallery).map { div ->
val href = div.selectFirstOrThrow(selectGalleryLink).attrAsRelativeUrl("href")
Manga(
id = generateUid(href),
title = div.select(selectGalleryTitle).text().trim(),
altTitle = null,
url = href,
publicUrl = href.toAbsoluteUrl(domain),
rating = RATING_UNKNOWN,
isNsfw = isNsfwSource,
coverUrl = div.selectFirstOrThrow(selectGalleryImg).src().orEmpty(),
tags = emptySet(),
state = null,
author = null,
source = source,
)
}
}
override suspend fun getPageUrl(page: MangaPage): String {
val doc = webClient.httpGet(page.url.toAbsoluteUrl(domain)).parseHtml()
val root = doc.body()

@ -7,7 +7,7 @@ import org.koitharu.kotatsu.parsers.site.heancmsalt.HeanCmsAlt
@MangaSourceParser("MANGAESP", "MangaEsp", "es")
internal class MangaEsp(context: MangaLoaderContext) :
HeanCmsAlt(context, MangaSource.MANGAESP, "mangaesp.co", 15) {
HeanCmsAlt(context, MangaSource.MANGAESP, "mangaesp.net", 15) {
override val listUrl = "/comic"

@ -5,6 +5,6 @@ import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.site.likemanga.LikeMangaParser
@MangaSourceParser("ZINMANGA_IO", "ZinManga.io", "en")
@MangaSourceParser("ZINMANGA_COM", "ZinManga.com", "en")
internal class ZinManga(context: MangaLoaderContext) :
LikeMangaParser(context, MangaSource.ZINMANGA_IO, "zinmanga.io")
LikeMangaParser(context, MangaSource.ZINMANGA_COM, "zinmanga.com")

@ -11,5 +11,5 @@ internal class HuntersScanEn(context: MangaLoaderContext) :
override val withoutAjax = true
override val datePattern = "MM/dd/yyyy"
override val tagPrefix = "series-genre/"
override val listUrl = "series/"
override val listUrl = "manga/"
}

@ -9,10 +9,11 @@ import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.site.madara.MadaraParser
import org.koitharu.kotatsu.parsers.util.*
import java.text.SimpleDateFormat
import java.util.Collections.emptyMap
@MangaSourceParser("RESETSCANS", "ResetScans", "en")
internal class ResetScans(context: MangaLoaderContext) :
MadaraParser(context, MangaSource.RESETSCANS, "reset-scans.us", 18) {
MadaraParser(context, MangaSource.RESETSCANS, "reset-scans.xyz", 18) {
override val datePattern = "MMM dd"
override suspend fun loadChapters(mangaUrl: String, document: Document): List<MangaChapter> {

@ -7,4 +7,4 @@ import org.koitharu.kotatsu.parsers.site.madara.MadaraParser
@MangaSourceParser("EMPERORSCAN", "EmperorScan", "es")
internal class EmperorScan(context: MangaLoaderContext) :
MadaraParser(context, MangaSource.EMPERORSCAN, "emperorscan.net")
MadaraParser(context, MangaSource.EMPERORSCAN, "emperorscan.com")

@ -7,4 +7,4 @@ import org.koitharu.kotatsu.parsers.site.madara.MadaraParser
@MangaSourceParser("RAGNAROKSCANLATION", "RagnarokScanlation", "es")
internal class RagnarokScanlation(context: MangaLoaderContext) :
MadaraParser(context, MangaSource.RAGNAROKSCANLATION, "ragnarokscanlation.com")
MadaraParser(context, MangaSource.RAGNAROKSCANLATION, "ragnarokscanlation.net")

@ -6,8 +6,8 @@ import org.koitharu.kotatsu.parsers.model.ContentType
import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.site.madara.MadaraParser
@MangaSourceParser("GLORYSCANSX", "GloryScansX", "fr", ContentType.HENTAI)
internal class GloryScansX(context: MangaLoaderContext) :
MadaraParser(context, MangaSource.GLORYSCANSX, "x.gloryscans.fr") {
@MangaSourceParser("LUNARHENTAI", "LunarHentai", "fr", ContentType.HENTAI)
internal class LunarHentai(context: MangaLoaderContext) :
MadaraParser(context, MangaSource.LUNARHENTAI, "hentai.lunarscans.fr") {
override val datePattern = "dd MMMM yyyy"
}

@ -11,5 +11,5 @@ internal class HuntersScan(context: MangaLoaderContext) :
override val withoutAjax = true
override val datePattern = "MM/dd/yyyy"
override val tagPrefix = "series-genre/"
override val listUrl = "series/"
override val listUrl = "manga/"
}

@ -7,6 +7,6 @@ import org.koitharu.kotatsu.parsers.site.madara.MadaraParser
@MangaSourceParser("WICKEDWITCHSCAN", "WickedWitchScan", "pt")
internal class WickedWitchScan(context: MangaLoaderContext) :
MadaraParser(context, MangaSource.WICKEDWITCHSCAN, "wickedwitchscan.com", pageSize = 10) {
MadaraParser(context, MangaSource.WICKEDWITCHSCAN, "wicked-witch-scan.com", pageSize = 10) {
override val postReq = true
}

@ -59,7 +59,7 @@ internal abstract class MangaboxParser(
is MangaListFilter.Search -> {
append("&keyw=")
append(filter.query.urlEncoded())
append(filter.query.replace(" ", "_").urlEncoded())
}
is MangaListFilter.Advanced -> {

@ -9,6 +9,10 @@ import org.koitharu.kotatsu.parsers.site.mangabox.MangaboxParser
@MangaSourceParser("MANGANATO", "Manganato", "en")
internal class Manganato(context: MangaLoaderContext) :
MangaboxParser(context, MangaSource.MANGANATO) {
override val configKeyDomain = ConfigKey.Domain("chapmanganato.to", "chapmanganato.com", "manganato.com")
override val configKeyDomain = ConfigKey.Domain(
"manganato.com",
"chapmanganato.to",
"chapmanganato.com",
)
override val otherDomain = configKeyDomain.defaultValue
}

@ -5,8 +5,8 @@ import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.site.mangareader.MangaReaderParser
@MangaSourceParser("LIGHTSCANS", "LightScans", "en")
internal class LightScans(context: MangaLoaderContext) :
MangaReaderParser(context, MangaSource.LIGHTSCANS, "lightscans.fun", pageSize = 20, searchPageSize = 10) {
@MangaSourceParser("LUASCANS", "LuaScans", "en")
internal class LuaScans(context: MangaLoaderContext) :
MangaReaderParser(context, MangaSource.LUASCANS, "luascans.com", pageSize = 20, searchPageSize = 10) {
override val isTagsExclusionSupported = false
}

@ -7,7 +7,7 @@ import org.koitharu.kotatsu.parsers.site.mangareader.MangaReaderParser
@MangaSourceParser("LUMINOUSSCANS", "LuminousScans", "en")
internal class LuminousScans(context: MangaLoaderContext) :
MangaReaderParser(context, MangaSource.LUMINOUSSCANS, "luminousscans.net", pageSize = 20, searchPageSize = 10) {
MangaReaderParser(context, MangaSource.LUMINOUSSCANS, "lumitoon.com", pageSize = 20, searchPageSize = 10) {
override val listUrl = "/series"
override val isTagsExclusionSupported = false
}

@ -7,4 +7,6 @@ import org.koitharu.kotatsu.parsers.site.mangareader.MangaReaderParser
@MangaSourceParser("MAGUSMANGA", "MagusManga", "en")
internal class MagusManga(context: MangaLoaderContext) :
MangaReaderParser(context, MangaSource.MAGUSMANGA, "magusmanga.com", pageSize = 20, searchPageSize = 10)
MangaReaderParser(context, MangaSource.MAGUSMANGA, "vofeg.com", pageSize = 20, searchPageSize = 10){
override val listUrl = "/series"
}

@ -7,7 +7,7 @@ import org.koitharu.kotatsu.parsers.site.mangareader.MangaReaderParser
@MangaSourceParser("NIGHTSCANS", "NightScans", "en")
internal class Nightscans(context: MangaLoaderContext) :
MangaReaderParser(context, MangaSource.NIGHTSCANS, "nightscans.net", pageSize = 20, searchPageSize = 10) {
MangaReaderParser(context, MangaSource.NIGHTSCANS, "night-scans.com", pageSize = 20, searchPageSize = 10) {
override val listUrl = "/series"
override val selectMangaListImg = "img.ts-post-image, picture img"
override val isTagsExclusionSupported = false

@ -5,8 +5,8 @@ import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.site.mangareader.MangaReaderParser
@MangaSourceParser("GLORYSCANS", "GloryScans", "fr")
internal class GloryScans(context: MangaLoaderContext) :
MangaReaderParser(context, MangaSource.GLORYSCANS, "gloryscans.fr", pageSize = 20, searchPageSize = 10) {
@MangaSourceParser("LUNARSCANS", "LunarScans", "fr")
internal class LunarScans(context: MangaLoaderContext) :
MangaReaderParser(context, MangaSource.LUNARSCANS, "lunarscans.fr", pageSize = 20, searchPageSize = 10) {
override val isTagsExclusionSupported = false
}

@ -7,6 +7,6 @@ import org.koitharu.kotatsu.parsers.site.mangareader.MangaReaderParser
@MangaSourceParser("ROBINMANGA", "RobinManga", "tr")
internal class RobinManga(context: MangaLoaderContext) :
MangaReaderParser(context, MangaSource.ROBINMANGA, "www.robinmanga.com", pageSize = 20, searchPageSize = 25) {
MangaReaderParser(context, MangaSource.ROBINMANGA, "robinmanga.com", pageSize = 20, searchPageSize = 25) {
override val isTagsExclusionSupported = false
}

@ -7,7 +7,7 @@ import org.koitharu.kotatsu.parsers.site.mangareader.MangaReaderParser
@MangaSourceParser("SUMMERTOON", "SummerToon", "tr")
internal class SummerToon(context: MangaLoaderContext) :
MangaReaderParser(context, MangaSource.SUMMERTOON, "summertoon.com", pageSize = 10, searchPageSize = 10) {
MangaReaderParser(context, MangaSource.SUMMERTOON, "summertoon.net", pageSize = 10, searchPageSize = 10) {
override val isTagsExclusionSupported = false
}

@ -54,7 +54,13 @@ internal class NudeMoonParser(
): List<Manga> {
val domain = domain
val url = when {
!query.isNullOrEmpty() -> "https://$domain/search?stext=${query.urlEncoded()}&rowstart=$offset"
!query.isNullOrEmpty() -> {
if (!isAuthorized) {
throw AuthRequiredException(source)
}
"https://$domain/search?stext=${query.urlEncoded()}&rowstart=$offset"
}
!tags.isNullOrEmpty() -> tags.joinToString(
separator = "_",
prefix = "https://$domain/tags/",
@ -96,7 +102,7 @@ internal class NudeMoonParser(
override suspend fun getDetails(manga: Manga): Manga {
val body = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml().body()
val root = body.selectFirstOrThrow("table.news_pic2")
val dateFormat = SimpleDateFormat("dd/MM/yyyy")
val dateFormat = SimpleDateFormat("dd MMMM yyyy", sourceLocale)
return manga.copy(
largeCoverUrl = body.selectFirstOrThrow("img[data-src]").attrAsAbsoluteUrl("data-src"),
description = root.selectFirst(".description")?.html() ?: manga.description,
@ -114,12 +120,14 @@ internal class NudeMoonParser(
id = manga.id,
url = manga.url,
source = source,
number = 1,
number = 0f,
volume = 0,
name = manga.title,
scanlator = root.getElementsByAttributeValueContaining("href", "/perevod/").firstOrNull()
?.textOrNull(),
uploadDate = dateFormat.tryParse(
root.getElementsMatchingOwnText("\\d{1,2}/\\d{2}/\\d{4}").firstOrNull()?.text(),
root.getElementsByAttributeValueEnding("src", "ico/time.png").firstOrNull()
?.nextElementSibling()?.text(),
),
branch = null,
),
@ -142,10 +150,6 @@ internal class NudeMoonParser(
}.toList()
}
override suspend fun getPageUrl(page: MangaPage): String {
return page.url.toAbsoluteUrl("img.$domain")
}
override suspend fun getAvailableTags(): Set<MangaTag> {
val domain = domain
val doc = webClient.httpGet("https://$domain/tags").parseHtml()

@ -4,6 +4,7 @@ import kotlinx.coroutines.flow.channelFlow
import kotlinx.coroutines.flow.first
import kotlinx.coroutines.launch
import okhttp3.Headers
import okhttp3.HttpUrl.Companion.toHttpUrl
import okhttp3.HttpUrl.Companion.toHttpUrlOrNull
import okhttp3.Interceptor
import okhttp3.Response
@ -18,6 +19,7 @@ import org.koitharu.kotatsu.parsers.exception.AuthRequiredException
import org.koitharu.kotatsu.parsers.exception.ParseException
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import org.koitharu.kotatsu.parsers.util.json.getStringOrNull
import org.koitharu.kotatsu.parsers.util.json.mapJSON
import java.text.SimpleDateFormat
import java.util.*
@ -43,6 +45,7 @@ internal abstract class GroupleParser(
private val userAgentKey = ConfigKey.UserAgent(
"Mozilla/5.0 (X11; U; UNICOS lcLinux; en-US) Gecko/20140730 (KHTML, like Gecko, Safari/419.3) Arora/0.8.0",
)
private val splitTranslationsKey = ConfigKey.SplitByTranslations(false)
override val headers: Headers = Headers.Builder().add("User-Agent", config[userAgentKey]).build()
@ -113,6 +116,15 @@ internal abstract class GroupleParser(
val root = doc.body().requireElementById("mangaBox").selectFirstOrThrow("div.leftContent")
val dateFormat = SimpleDateFormat("dd.MM.yy", Locale.US)
val coverImg = root.selectFirst("div.subject-cover")?.selectFirst("img")
val translations = if (config[splitTranslationsKey]) {
root.selectFirst("div.translator-selection")
?.select(".translator-selection-item")
?.associate {
it.id().removePrefix("tr-").toLong() to it.selectFirst(".translator-selection-name")?.textOrNull()
}
} else {
null
}
return manga.copy(
description = root.selectFirst("div.manga-description")?.html(),
largeCoverUrl = coverImg?.attr("data-full"),
@ -128,24 +140,48 @@ internal abstract class GroupleParser(
author = root.selectFirst("a.person-link")?.text() ?: manga.author,
isNsfw = manga.isNsfw || root.select(".alert-warning").any { it.ownText().contains(NSFW_ALERT) },
chapters = root.requireElementById("chapters-list").select("a.chapter-link")
.mapChapters(reversed = true) { i, a ->
val tr = a.selectFirstParent("tr") ?: return@mapChapters null
.flatMapChapters(reversed = true) { a ->
val tr = a.selectFirstParent("tr") ?: return@flatMapChapters emptyList()
val href = a.attrAsRelativeUrl("href")
var translators = ""
val translatorElement = a.attr("title")
if (!translatorElement.isNullOrBlank()) {
translators = translatorElement.replace("(Переводчик),", "&").removeSuffix(" (Переводчик)")
val number = tr.attr("data-num").toFloatOrNull()?.div(10f) ?: 0f
val volume = tr.attr("data-vol").toIntOrNull() ?: 0
if (translations.isNullOrEmpty() || a.attr("data-translations").isEmpty()) {
var translators = ""
val translatorElement = a.attr("title")
if (!translatorElement.isNullOrBlank()) {
translators = translatorElement.replace("(Переводчик),", "&").removeSuffix(" (Переводчик)")
}
listOf(
MangaChapter(
id = generateUid(href),
name = a.text().removePrefix(manga.title).trim(),
number = number,
volume = volume,
url = href,
uploadDate = dateFormat.tryParse(tr.selectFirst("td.date")?.text()),
scanlator = translators,
source = source,
branch = null,
),
)
} else {
val translationData = JSONArray(a.attr("data-translations"))
translationData.mapJSON { jo ->
val personId = jo.getLong("personId")
val link = href.setQueryParam("tran", personId.toString())
MangaChapter(
id = generateUid(link),
name = a.text().removePrefix(manga.title).trim(),
number = number,
volume = volume,
url = link,
uploadDate = dateFormat.tryParse(jo.getStringOrNull("dateCreated")),
scanlator = null,
source = source,
branch = translations[personId],
)
}
}
MangaChapter(
id = generateUid(href),
name = a.text().removePrefix(manga.title).trim(),
number = i + 1,
url = href,
uploadDate = dateFormat.tryParse(tr.selectFirst("td.date")?.text()),
scanlator = translators,
source = source,
branch = null,
)
},
)
}
@ -246,6 +282,7 @@ internal abstract class GroupleParser(
override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) {
super.onCreateConfig(keys)
keys.add(userAgentKey)
keys.add(splitTranslationsKey)
}
override suspend fun getRelatedManga(seed: Manga): List<Manga> {
@ -402,4 +439,14 @@ internal abstract class GroupleParser(
)
}
}
private fun String.setQueryParam(name: String, value: String): String {
return toAbsoluteUrl(domain)
.toHttpUrl()
.newBuilder()
.setQueryParameter(name, value)
.build()
.toString()
.toRelativeUrl(domain)
}
}

@ -7,4 +7,4 @@ import org.koitharu.kotatsu.parsers.site.zeistmanga.ZeistMangaParser
@MangaSourceParser("ROCKSMANGA_COM", "RocksManga.com", "ar")
internal class RocksManga(context: MangaLoaderContext) :
ZeistMangaParser(context, MangaSource.ROCKSMANGA_COM, "www.rocks-manga.com")
ZeistMangaParser(context, MangaSource.ROCKSMANGA_COM, "rocks-manga.com")

@ -163,8 +163,9 @@ internal class Baozimh(context: MangaLoaderContext) :
else -> null
},
tags = tags,
chapters = doc.requireElementById("chapter-items").select("div.comics-chapters a")
.mapChapters(reversed = true) { i, a ->
chapters = (doc.requireElementById("chapter-items").select("div.comics-chapters a")
+ doc.requireElementById("chapters_other_list").select("div.comics-chapters a"))
.mapChapters { i, a ->
val url = a.attrAsRelativeUrl("href").toAbsoluteUrl(domain)
MangaChapter(
id = generateUid(url),

@ -19,6 +19,19 @@ inline fun <T> List<T>.mapChapters(
return builder.toList()
}
@InternalParsersApi
inline fun <T> List<T>.flatMapChapters(
reversed: Boolean = false,
transform: (T) -> Iterable<MangaChapter?>,
): List<MangaChapter> {
val builder = ChaptersListBuilder(collectionSize())
val elements = if (reversed) this.asReversed() else this
for (item in elements) {
builder.addAll(transform(item))
}
return builder.toList()
}
@PublishedApi
internal fun <T> Iterable<T>.collectionSize(): Int {
return if (this is Collection<*>) this.size else 10

@ -91,13 +91,15 @@ fun concatUrl(host: String, path: String): String {
}
fun DateFormat.tryParse(str: String?): Long = if (str.isNullOrEmpty()) {
assert(false) { "Date string is null or empty" }
// assert(false) { "Date string is null or empty" }
0L
} else {
runCatching {
parse(str)?.time ?: 0L
}.onFailure {
assert(false) { "Cannot parse date $str: ${it.message}" }
if (javaClass.desiredAssertionStatus()) {
throw AssertionError("Cannot parse date $str", it)
}
}.getOrDefault(0L)
}

@ -7,7 +7,12 @@ import okhttp3.Response
import org.koitharu.kotatsu.parsers.config.MangaSourceConfig
import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.util.await
import java.security.SecureRandom
import java.security.cert.X509Certificate
import java.util.concurrent.TimeUnit
import javax.net.ssl.SSLContext
import javax.net.ssl.SSLSocketFactory
import javax.net.ssl.X509TrustManager
internal object MangaLoaderContextMock : MangaLoaderContext() {
@ -15,6 +20,7 @@ internal object MangaLoaderContextMock : MangaLoaderContext() {
override val httpClient: OkHttpClient = OkHttpClient.Builder()
.cookieJar(cookieJar)
.permissiveSSL()
.addInterceptor(CommonHeadersInterceptor())
.addInterceptor(CloudFlareInterceptor())
.connectTimeout(20, TimeUnit.SECONDS)
@ -52,4 +58,24 @@ internal object MangaLoaderContextMock : MangaLoaderContext() {
cookieJar.loadFromStream(it)
} ?: println("No cookies loaded!")
}
private fun OkHttpClient.Builder.permissiveSSL() = also { builder ->
runCatching {
val trustAllCerts = object : X509TrustManager {
override fun checkClientTrusted(chain: Array<X509Certificate>, authType: String) = Unit
override fun checkServerTrusted(chain: Array<X509Certificate>, authType: String) = Unit
override fun getAcceptedIssuers(): Array<X509Certificate> = emptyArray()
}
val sslContext = SSLContext.getInstance("SSL")
sslContext.init(null, arrayOf(trustAllCerts), SecureRandom())
val sslSocketFactory: SSLSocketFactory = sslContext.socketFactory
builder.sslSocketFactory(sslSocketFactory, trustAllCerts)
builder.hostnameVerifier { _, _ -> true }
}.onFailure {
it.printStackTrace()
}
}
}

@ -3,11 +3,9 @@ package org.koitharu.kotatsu.parsers
import kotlinx.coroutines.test.runTest
import okhttp3.HttpUrl
import org.junit.jupiter.api.Disabled
import org.junit.jupiter.api.Test
import org.junit.jupiter.params.ParameterizedTest
import org.koitharu.kotatsu.parsers.model.Manga
import org.koitharu.kotatsu.parsers.model.MangaListFilter
import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.model.SortOrder
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.domain
import org.koitharu.kotatsu.parsers.util.medianOrNull
import org.koitharu.kotatsu.parsers.util.mimeType
@ -21,6 +19,14 @@ internal class MangaParserTest {
private val context = MangaLoaderContextMock
private val timeout = 2.minutes
@Test
fun singleTest() = runTest {
val manga = mangaOf(MangaSource.READMANGA_RU, "https://readmanga.live/podniatie_urovnia_v_odinochku__A5e4e")
val parser = context.newParserInstance(manga.source)
val details = parser.getDetails(manga)
assert(!details.chapters.isNullOrEmpty())
}
@ParameterizedTest(name = "{index}|list|{0}")
@MangaSources
fun list(source: MangaSource) = runTest(timeout = timeout) {
@ -162,8 +168,9 @@ internal class MangaParserTest {
assert(c.isDistinctBy { it.id }) {
"Chapters are not distinct by id: ${c.maxDuplicates { it.id }} for $publicUrl"
}
assert(c.isDistinctByNotNull { if (it.number > 0f) it.number to it.branch else null }) {
"Chapters are not distinct by number: ${c.maxDuplicates { it.number to it.branch }} for $publicUrl"
assert(c.isDistinctByNotNull { it.key() }) {
val dup = c.mapNotNull { it.key() }.maxDuplicates { it }
"Chapters are not distinct by branch/volume/number: $dup for $publicUrl"
}
assert(c.all { it.source == source })
checkImageRequest(coverUrl, source)
@ -266,4 +273,10 @@ internal class MangaParserTest {
private fun String.isCapitalized(): Boolean {
return !first().isLowerCase()
}
private fun MangaChapter.key(): Any? = when {
number > 0f && volume > 0 -> Triple(branch, volume, number)
number > 0f -> Pair(branch, number)
else -> null
}
}

Loading…
Cancel
Save