Batch parsers fixes

Koitharu 3 years ago
parent e8d299782b
commit 413f4a2f10
No known key found for this signature in database
GPG Key ID: 8E861F8CE6E7CE27

@ -12,115 +12,115 @@ import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl
import java.util.* import java.util.*
abstract class MangaParser @InternalParsersApi constructor( abstract class MangaParser @InternalParsersApi constructor(
@property:InternalParsersApi val context: MangaLoaderContext, @property:InternalParsersApi val context: MangaLoaderContext,
val source: MangaSource, val source: MangaSource,
) { ) {
/** /**
* Supported [SortOrder] variants. Must not be empty. * Supported [SortOrder] variants. Must not be empty.
* *
* For better performance use [EnumSet] for more than one item. * For better performance use [EnumSet] for more than one item.
*/ */
abstract val sortOrders: Set<SortOrder> abstract val sortOrders: Set<SortOrder>
val config by lazy { context.getConfig(source) } val config by lazy { context.getConfig(source) }
open val sourceLocale: Locale? open val sourceLocale: Locale
get() = source.locale?.let { Locale(it) } get() = source.locale?.let { Locale(it) } ?: Locale.ROOT
/** /**
* Provide default domain and available alternatives, if any. * Provide default domain and available alternatives, if any.
* *
* Never hardcode domain in requests, use [getDomain] instead. * Never hardcode domain in requests, use [getDomain] instead.
*/ */
@InternalParsersApi @InternalParsersApi
abstract val configKeyDomain: ConfigKey.Domain abstract val configKeyDomain: ConfigKey.Domain
open val headers: Headers? = null open val headers: Headers? = null
/** /**
* Used as fallback if value of `sortOrder` passed to [getList] is null * Used as fallback if value of `sortOrder` passed to [getList] is null
*/ */
protected open val defaultSortOrder: SortOrder protected open val defaultSortOrder: SortOrder
get() { get() {
val supported = sortOrders val supported = sortOrders
return SortOrder.values().first { it in supported } return SortOrder.values().first { it in supported }
} }
protected val webClient: WebClient = OkHttpWebClient(context.httpClient, source) protected val webClient: WebClient = OkHttpWebClient(context.httpClient, source)
/** /**
* Parse list of manga by specified criteria * Parse list of manga by specified criteria
* *
* @param offset starting from 0 and used for pagination. * @param offset starting from 0 and used for pagination.
* Note than passed value may not be divisible by internal page size, so you should adjust it manually. * Note than passed value may not be divisible by internal page size, so you should adjust it manually.
* @param query search query, may be null or empty if no search needed * @param query search query, may be null or empty if no search needed
* @param tags genres for filtering, values from [getTags] and [Manga.tags]. May be null or empty * @param tags genres for filtering, values from [getTags] and [Manga.tags]. May be null or empty
* @param sortOrder one of [sortOrders] or null for default value * @param sortOrder one of [sortOrders] or null for default value
*/ */
@JvmSynthetic @JvmSynthetic
@InternalParsersApi @InternalParsersApi
abstract suspend fun getList( abstract suspend fun getList(
offset: Int, offset: Int,
query: String?, query: String?,
tags: Set<MangaTag>?, tags: Set<MangaTag>?,
sortOrder: SortOrder, sortOrder: SortOrder,
): List<Manga> ): List<Manga>
/** /**
* Parse list of manga with search by text query * Parse list of manga with search by text query
* *
* @param offset starting from 0 and used for pagination. * @param offset starting from 0 and used for pagination.
* @param query search query * @param query search query
*/ */
open suspend fun getList(offset: Int, query: String): List<Manga> { open suspend fun getList(offset: Int, query: String): List<Manga> {
return getList(offset, query, null, defaultSortOrder) return getList(offset, query, null, defaultSortOrder)
} }
/** /**
* Parse list of manga by specified criteria * Parse list of manga by specified criteria
* *
* @param offset starting from 0 and used for pagination. * @param offset starting from 0 and used for pagination.
* Note than passed value may not be divisible by internal page size, so you should adjust it manually. * Note than passed value may not be divisible by internal page size, so you should adjust it manually.
* @param tags genres for filtering, values from [getTags] and [Manga.tags]. May be null or empty * @param tags genres for filtering, values from [getTags] and [Manga.tags]. May be null or empty
* @param sortOrder one of [sortOrders] or null for default value * @param sortOrder one of [sortOrders] or null for default value
*/ */
open suspend fun getList(offset: Int, tags: Set<MangaTag>?, sortOrder: SortOrder?): List<Manga> { open suspend fun getList(offset: Int, tags: Set<MangaTag>?, sortOrder: SortOrder?): List<Manga> {
return getList(offset, null, tags, sortOrder ?: defaultSortOrder) return getList(offset, null, tags, sortOrder ?: defaultSortOrder)
} }
/** /**
* Parse details for [Manga]: chapters list, description, large cover, etc. * Parse details for [Manga]: chapters list, description, large cover, etc.
* Must return the same manga, may change any fields excepts id, url and source * Must return the same manga, may change any fields excepts id, url and source
* @see Manga.copy * @see Manga.copy
*/ */
abstract suspend fun getDetails(manga: Manga): Manga abstract suspend fun getDetails(manga: Manga): Manga
/** /**
* Parse pages list for specified chapter. * Parse pages list for specified chapter.
* @see MangaPage for details * @see MangaPage for details
*/ */
abstract suspend fun getPages(chapter: MangaChapter): List<MangaPage> abstract suspend fun getPages(chapter: MangaChapter): List<MangaPage>
/** /**
* Fetch direct link to the page image. * Fetch direct link to the page image.
*/ */
open suspend fun getPageUrl(page: MangaPage): String = page.url.toAbsoluteUrl(domain) open suspend fun getPageUrl(page: MangaPage): String = page.url.toAbsoluteUrl(domain)
/** /**
* Fetch available tags (genres) for source * Fetch available tags (genres) for source
*/ */
abstract suspend fun getTags(): Set<MangaTag> abstract suspend fun getTags(): Set<MangaTag>
/** /**
* Parse favicons from the main page of the source`s website * Parse favicons from the main page of the source`s website
*/ */
open suspend fun getFavicons(): Favicons { open suspend fun getFavicons(): Favicons {
return FaviconParser(webClient, domain).parseFavicons() return FaviconParser(webClient, domain).parseFavicons()
} }
@CallSuper @CallSuper
open fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { open fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) {
keys.add(configKeyDomain) keys.add(configKeyDomain)
} }
} }

@ -10,94 +10,92 @@ import java.util.*
@MangaSourceParser("CLONEMANGA", "CloneManga", "en") @MangaSourceParser("CLONEMANGA", "CloneManga", "en")
internal class CloneMangaParser(context: MangaLoaderContext) : PagedMangaParser( internal class CloneMangaParser(context: MangaLoaderContext) : PagedMangaParser(
context, context,
MangaSource.CLONEMANGA, MangaSource.CLONEMANGA,
pageSize = 1, pageSize = 1,
) { ) {
override val sortOrders: Set<SortOrder> = Collections.singleton( override val sortOrders: Set<SortOrder> = Collections.singleton(
SortOrder.POPULARITY, SortOrder.POPULARITY,
) )
override val configKeyDomain = ConfigKey.Domain("manga.clone-army.org", null) override val configKeyDomain = ConfigKey.Domain("manga.clone-army.org", null)
override suspend fun getListPage( override suspend fun getListPage(
page: Int, page: Int,
query: String?, query: String?,
tags: Set<MangaTag>?, tags: Set<MangaTag>?,
sortOrder: SortOrder, sortOrder: SortOrder,
): List<Manga> { ): List<Manga> {
if (query != null || page > 1) { if (query != null || page > 1) {
return emptyList() return emptyList()
} }
val link = "https://${domain}/viewer_landing.php" val link = "https://${domain}/viewer_landing.php"
val doc = webClient.httpGet(link).parseHtml() val doc = webClient.httpGet(link).parseHtml()
val mangas = doc.getElementsByClass("comicPreviewContainer") val mangas = doc.getElementsByClass("comicPreviewContainer")
return mangas.mapNotNull { item -> return mangas.mapNotNull { item ->
val attr = item.getElementsByClass("comicPreview").attr("style") val background = item.selectFirstOrThrow(".comicPreview").styleValueOrNull("background")
val href = item.selectFirst("a")?.attrAsAbsoluteUrl("href") ?: return@mapNotNull null val href = item.selectFirst("a")?.attrAsAbsoluteUrl("href") ?: return@mapNotNull null
val cover = attr.substring(attr.indexOf("site/themes"), attr.indexOf(")")) val cover = background?.substring(background.indexOf("site/themes"), background.indexOf(")"))
Manga( Manga(
id = generateUid(href), id = generateUid(href),
title = item.selectFirst("h3")?.text() ?: return@mapNotNull null, title = item.selectFirst("h3")?.text() ?: return@mapNotNull null,
coverUrl = "https://${domain}/$cover", coverUrl = "https://${domain}/$cover",
altTitle = null, altTitle = null,
author = "Dan Kim", author = "Dan Kim",
rating = RATING_UNKNOWN, rating = RATING_UNKNOWN,
url = href, url = href,
isNsfw = false, isNsfw = false,
tags = emptySet(), tags = emptySet(),
state = null, state = null,
publicUrl = href.toAbsoluteUrl(domain), publicUrl = href.toAbsoluteUrl(domain),
source = source, source = source,
) )
} }
} }
override suspend fun getDetails(manga: Manga): Manga { override suspend fun getDetails(manga: Manga): Manga {
val doc = webClient.httpGet(manga.publicUrl).parseHtml() val doc = webClient.httpGet(manga.publicUrl).parseHtml()
val series = doc.location() val series = doc.location()
val numChapters = Regex( val numChapters = Regex(
pattern = "&page=(.*)&lang=", pattern = "&page=(.*)&lang=",
).findAll( ).findAll(
input = doc.getElementsByTag("script")[3].toString(), input = doc.getElementsByTag("script")[3].toString(),
) )
.elementAt(3).destructured.component1() .elementAt(3).destructured.component1()
.toInt() .toInt()
val chapters = ArrayList<MangaChapter>() val chapters = ArrayList<MangaChapter>()
for (i in 0..numChapters) { for (i in 0..numChapters) {
val chapter = MangaChapter( val chapter = MangaChapter(
id = generateUid("$series&page=$i"), id = generateUid("$series&page=$i"),
name = "Chapter ${i + 1}", name = "Chapter ${i + 1}",
number = i + 1, number = i + 1,
url = "$series&page=$i", url = "$series&page=$i",
scanlator = null, scanlator = null,
branch = null, branch = null,
uploadDate = 0L, uploadDate = 0L,
source = MangaSource.DUMMY, source = source,
) )
chapters.add(chapter) chapters.add(chapter)
} }
return manga.copy(chapters = chapters) return manga.copy(chapters = chapters)
} }
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml()
val imgUrl = doc.getElementsByClass("subsectionContainer")[0] val imgUrl = doc.getElementsByClass("subsectionContainer")[0]
.selectFirst("img") .selectFirst("img")
?.attrAsAbsoluteUrlOrNull("src") ?: doc.parseFailed("Something broken") ?.attrAsAbsoluteUrlOrNull("src") ?: doc.parseFailed("Something broken")
return listOf( return listOf(
MangaPage( MangaPage(
id = generateUid(imgUrl), id = generateUid(imgUrl),
url = imgUrl, url = imgUrl,
referer = imgUrl, referer = imgUrl,
preview = null, preview = null,
source = MangaSource.DUMMY, source = source,
), ),
) )
} }
override suspend fun getTags(): Set<MangaTag> { override suspend fun getTags(): Set<MangaTag> = emptySet()
return emptySet()
}
} }

@ -1,5 +1,6 @@
package org.koitharu.kotatsu.parsers.site package org.koitharu.kotatsu.parsers.site
import androidx.collection.ArrayMap
import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser import org.koitharu.kotatsu.parsers.PagedMangaParser
@ -10,176 +11,187 @@ import java.util.*
@MangaSourceParser("MANHWA18", "Manhwa18", "en") @MangaSourceParser("MANHWA18", "Manhwa18", "en")
class Manhwa18Parser(context: MangaLoaderContext) : class Manhwa18Parser(context: MangaLoaderContext) :
PagedMangaParser(context, MangaSource.MANHWA18, pageSize = 20, searchPageSize = 20) { PagedMangaParser(context, MangaSource.MANHWA18, pageSize = 20, searchPageSize = 20) {
override val configKeyDomain: ConfigKey.Domain override val configKeyDomain: ConfigKey.Domain
get() = ConfigKey.Domain("manhwa18.net", null) get() = ConfigKey.Domain("manhwa18.net", null)
override val sortOrders: Set<SortOrder> override val sortOrders: Set<SortOrder>
get() = EnumSet.of(SortOrder.UPDATED, SortOrder.POPULARITY, SortOrder.ALPHABETICAL) get() = EnumSet.of(SortOrder.UPDATED, SortOrder.POPULARITY, SortOrder.ALPHABETICAL, SortOrder.NEWEST)
override suspend fun getFavicons(): Favicons { private val tagsMap = SuspendLazy(::parseTags)
return Favicons(
listOf( override suspend fun getFavicons(): Favicons {
Favicon("https://${domain}/uploads/logos/logo-mini.png", 92, null), return Favicons(
), listOf(
domain, Favicon("https://${domain}/uploads/logos/logo-mini.png", 92, null),
) ),
} domain,
)
override suspend fun getDetails(manga: Manga): Manga { }
val docs = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml()
val cardInfoElement = docs.selectFirst(".card .manga-info") override suspend fun getDetails(manga: Manga): Manga {
val author = cardInfoElement?.selectFirst("b:contains(Author(s))")?.parent() val docs = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml()
?.select("a.btn") val cardInfoElement = docs.selectFirst(".card .manga-info")
?.joinToString(", ") { it.text() } val author = cardInfoElement?.selectFirst("b:contains(Author(s))")?.parent()
val tags = cardInfoElement?.selectFirst("b:contains(Genre(s))")?.parent() ?.select("a.btn")
?.select("a.btn") ?.joinToString(", ") { it.text() }
?.mapToSet { MangaTag(it.text(), it.text().lowercase(), MangaSource.MANHWA18) } val availableTags = tagsMap.get()
val state = cardInfoElement?.selectFirst("b:contains(Status)")?.parent() val tags = cardInfoElement?.selectFirst("b:contains(Genre(s))")?.parent()
?.selectFirst("a.btn") ?.select("a.btn")
?.let { ?.mapNotNullToSet { availableTags[it.text().lowercase(Locale.ENGLISH)] }
when (it.text()) { val state = cardInfoElement?.selectFirst("b:contains(Status)")?.parent()
"On going" -> MangaState.ONGOING ?.selectFirst("a.btn")
"Completed" -> MangaState.FINISHED ?.let {
else -> null when (it.text()) {
} "On going" -> MangaState.ONGOING
} "Completed" -> MangaState.FINISHED
else -> null
return manga.copy( }
altTitle = cardInfoElement?.selectFirst("b:contains(Other names)")?.parent()?.ownText()?.removePrefix(": "), }
author = author,
description = docs.selectFirst(".series-summary .summary-content")?.html(), return manga.copy(
tags = tags.orEmpty(), altTitle = cardInfoElement?.selectFirst("b:contains(Other names)")?.parent()?.ownText()?.removePrefix(": "),
state = state, author = author,
chapters = docs.select(".card-body > .list-chapters > a").asReversed().mapChapters { index, element -> description = docs.selectFirst(".series-summary .summary-content")?.html(),
// attrAsRelativeUrl only return page url without the '/' tags = tags.orEmpty(),
val chapterUrl = element.attrAsAbsoluteUrlOrNull("href")?.toRelativeUrl(domain) state = state,
?: return@mapChapters null chapters = docs.select(".card-body > .list-chapters > a").asReversed().mapChapters { index, element ->
val uploadDate = parseUploadDate(element.selectFirst(".chapter-time")?.text()) // attrAsRelativeUrl only return page url without the '/'
MangaChapter( val chapterUrl = element.attrAsAbsoluteUrlOrNull("href")?.toRelativeUrl(domain)
id = generateUid(chapterUrl), ?: return@mapChapters null
name = element.selectFirst(".chapter-name")?.text().orEmpty(), val uploadDate = parseUploadDate(element.selectFirst(".chapter-time")?.text())
number = index + 1, MangaChapter(
url = chapterUrl, id = generateUid(chapterUrl),
scanlator = null, name = element.selectFirst(".chapter-name")?.text().orEmpty(),
uploadDate = uploadDate, number = index + 1,
branch = null, url = chapterUrl,
source = MangaSource.MANHWA18, scanlator = null,
) uploadDate = uploadDate,
}, branch = null,
) source = MangaSource.MANHWA18,
} )
},
// 7 minutes ago )
// 5 hours ago }
// 2 days ago
// 2 weeks ago // 7 minutes ago
// 4 years ago // 5 hours ago
private fun parseUploadDate(timeStr: String?): Long { // 2 days ago
timeStr ?: return 0 // 2 weeks ago
// 4 years ago
val timeWords = timeStr.split(' ') private fun parseUploadDate(timeStr: String?): Long {
if (timeWords.size != 3) return 0 timeStr ?: return 0
val timeWord = timeWords[1]
val timeAmount = timeWords[0].toIntOrNull() ?: return 0 val timeWords = timeStr.split(' ')
val timeUnit = when (timeWord) { if (timeWords.size != 3) return 0
"minute", "minutes" -> Calendar.MINUTE val timeWord = timeWords[1]
"hour", "hours" -> Calendar.HOUR val timeAmount = timeWords[0].toIntOrNull() ?: return 0
"day", "days" -> Calendar.DAY_OF_YEAR val timeUnit = when (timeWord) {
"week", "weeks" -> Calendar.WEEK_OF_YEAR "minute", "minutes" -> Calendar.MINUTE
"month", "months" -> Calendar.MONTH "hour", "hours" -> Calendar.HOUR
"year", "years" -> Calendar.YEAR "day", "days" -> Calendar.DAY_OF_YEAR
else -> return 0 "week", "weeks" -> Calendar.WEEK_OF_YEAR
} "month", "months" -> Calendar.MONTH
val cal = Calendar.getInstance() "year", "years" -> Calendar.YEAR
cal.add(timeUnit, -timeAmount) else -> return 0
return cal.time.time }
} val cal = Calendar.getInstance()
cal.add(timeUnit, -timeAmount)
override suspend fun getListPage( return cal.time.time
page: Int, }
query: String?,
tags: Set<MangaTag>?, override suspend fun getListPage(
sortOrder: SortOrder, page: Int,
): List<Manga> { query: String?,
val sortQuery = when (sortOrder) { tags: Set<MangaTag>?,
SortOrder.ALPHABETICAL -> "name" sortOrder: SortOrder,
SortOrder.POPULARITY -> "views" ): List<Manga> {
SortOrder.UPDATED -> "last_update" val sortQuery = when (sortOrder) {
else -> "" SortOrder.ALPHABETICAL -> "az"
} SortOrder.POPULARITY -> "top"
SortOrder.UPDATED -> "update"
val sortType = if (sortOrder == SortOrder.ALPHABETICAL) "ASC" else "DESC" SortOrder.NEWEST -> "new"
val tagQuery = tags?.joinToString(",") { it.key }.orEmpty() else -> ""
val url = buildString { }
append("https://")
append(domain) val tagQuery = tags?.joinToString(",") { it.key }.orEmpty()
append("/manga-list.html?listType=pagination&page=") val url = buildString {
append(page) append("https://")
append("&artist=&author=&group=&m_status=&name=") append(domain)
append(query?.urlEncoded().orEmpty()) append("/tim-kiem?page=")
append("&genre=$tagQuery") append(page)
append("&ungenre=") if (!query.isNullOrEmpty()) {
append("&sort=") append("&q=")
append(sortQuery) append(query.urlEncoded())
append("&sort_type=") }
append(sortType) append("&accept_genres=$tagQuery")
} append("&sort=")
append(sortQuery)
val docs = webClient.httpGet(url).parseHtml() }
val actualPage = docs.selectFirst("ul.pagination a.active")?.text()?.toIntOrNull()
if (actualPage != page) { val docs = webClient.httpGet(url).parseHtml()
return emptyList()
} return docs.select(".card-body .thumb-item-flow")
.map {
return docs.select(".card-body .thumb-item-flow") val titleElement = it.selectFirstOrThrow(".thumb_attr.series-title > a")
.map { val absUrl = titleElement.attrAsAbsoluteUrl("href")
val titleElement = it.selectFirstOrThrow(".thumb_attr.series-title > a") Manga(
val absUrl = titleElement.attrAsAbsoluteUrl("href") id = generateUid(absUrl.toRelativeUrl(domain)),
Manga( title = titleElement.text(),
id = generateUid(absUrl.toRelativeUrl(domain)), altTitle = null,
title = titleElement.text(), url = absUrl.toRelativeUrl(domain),
altTitle = null, publicUrl = absUrl,
url = absUrl.toRelativeUrl(domain), rating = RATING_UNKNOWN,
publicUrl = absUrl, isNsfw = true,
rating = RATING_UNKNOWN, coverUrl = it.selectFirst("div.img-in-ratio")?.attrAsAbsoluteUrl("data-bg").orEmpty(),
isNsfw = true, tags = emptySet(),
coverUrl = it.selectFirst("div.img-in-ratio")?.attrAsAbsoluteUrl("data-bg").orEmpty(), state = null,
tags = emptySet(), author = null,
state = null, largeCoverUrl = null,
author = null, description = null,
largeCoverUrl = null, source = MangaSource.MANHWA18,
description = null, )
source = MangaSource.MANHWA18, }
) }
}
} override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val chapterUrl = chapter.url.toAbsoluteUrl(domain)
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { val doc = webClient.httpGet(chapterUrl).parseHtml()
val chapterUrl = chapter.url.toAbsoluteUrl(domain) return doc.requireElementById("chapter-content").select("img").mapNotNull {
return webClient.httpGet(chapterUrl).parseHtml() val url = it.attrAsRelativeUrlOrNull("data-src")
.select("div.chapter-content > img").map { ?: it.attrAsRelativeUrlOrNull("src")
val url = it.attrAsRelativeUrlOrNull("src").orEmpty() ?: return@mapNotNull null
MangaPage( MangaPage(
id = generateUid(url), id = generateUid(url),
url = url, url = url,
referer = chapterUrl, referer = chapterUrl,
preview = null, preview = null,
source = MangaSource.MANHWA18, source = MangaSource.MANHWA18,
) )
} }
} }
override suspend fun getTags(): Set<MangaTag> { override suspend fun getTags(): Set<MangaTag> {
return webClient.httpGet("https://${domain}/").parseHtml().selectFirstOrThrow(".genres-menu") return tagsMap.get().values.toSet()
.select("a.genres-item").orEmpty() }
.mapToSet {
MangaTag( private suspend fun parseTags(): Map<String, MangaTag> {
title = it.text(), val doc = webClient.httpGet("https://$domain/tim-kiem?q=").parseHtml()
key = it.text().lowercase(), val list = doc.getElementsByAttribute("data-genre-id")
source = MangaSource.MANHWA18, if (list.isEmpty()) {
) return emptyMap()
} }
} val result = ArrayMap<String, MangaTag>(list.size)
for (item in list) {
val id = item.attr("data-genre-id")
val name = item.text()
result[name.lowercase(Locale.ENGLISH)] = MangaTag(
title = name.toTitleCase(Locale.ENGLISH),
key = id,
source = source,
)
}
return result
}
} }

@ -15,163 +15,163 @@ import java.util.*
@MangaSourceParser("UNION_MANGAS", "Union Mangás", "pt") @MangaSourceParser("UNION_MANGAS", "Union Mangás", "pt")
class UnionMangasParser(context: MangaLoaderContext) : PagedMangaParser(context, MangaSource.UNION_MANGAS, 40) { class UnionMangasParser(context: MangaLoaderContext) : PagedMangaParser(context, MangaSource.UNION_MANGAS, 40) {
override val sortOrders = EnumSet.of( override val sortOrders = EnumSet.of(
SortOrder.ALPHABETICAL, SortOrder.ALPHABETICAL,
SortOrder.POPULARITY, SortOrder.POPULARITY,
) )
override val configKeyDomain = ConfigKey.Domain("unionleitor.top", emptyArray()) override val configKeyDomain = ConfigKey.Domain("unionleitor.top", emptyArray())
override suspend fun getListPage( override suspend fun getListPage(
page: Int, page: Int,
query: String?, query: String?,
tags: Set<MangaTag>?, tags: Set<MangaTag>?,
sortOrder: SortOrder, sortOrder: SortOrder,
): List<Manga> { ): List<Manga> {
if (!query.isNullOrEmpty()) { if (!query.isNullOrEmpty()) {
return if (page == searchPaginator.firstPage) { return if (page == searchPaginator.firstPage) {
search(query) search(query)
} else { } else {
emptyList() emptyList()
} }
} }
val tag = tags.oneOrThrowIfMany() val tag = tags.oneOrThrowIfMany()
val url = urlBuilder() val url = urlBuilder()
.addPathSegment("lista-mangas") .addPathSegment("lista-mangas")
.addPathSegment( .addPathSegment(
when { when {
tag != null -> tag.key tag != null -> tag.key
sortOrder == SortOrder.ALPHABETICAL -> "a-z" sortOrder == SortOrder.ALPHABETICAL -> "a-z"
else -> "visualizacoes" else -> "visualizacoes"
}, },
).addPathSegment(page.toString()) ).addPathSegment(page.toString())
val doc = webClient.httpGet(url.build()).parseHtml() val doc = webClient.httpGet(url.build()).parseHtml()
val root = doc.selectFirstOrThrow("div.tamanho-bloco-perfil") val root = doc.selectFirstOrThrow("div.tamanho-bloco-perfil")
return root.select(".lista-mangas-novos").map { div -> return root.select(".lista-mangas-novos").map { div ->
val a = div.selectFirstOrThrow("a") val a = div.selectFirstOrThrow("a")
val img = div.selectFirstOrThrow("img") val img = div.selectFirstOrThrow("img")
val href = a.attrAsRelativeUrl("href") val href = a.attrAsRelativeUrl("href")
Manga( Manga(
id = generateUid(href), id = generateUid(href),
url = href, url = href,
publicUrl = a.attrAsAbsoluteUrl("href"), publicUrl = a.attrAsAbsoluteUrl("href"),
title = div.selectLastOrThrow("a").text(), title = div.selectLastOrThrow("a").text(),
coverUrl = img.attrAsAbsoluteUrl("src"), coverUrl = img.attrAsAbsoluteUrl("src"),
altTitle = null, altTitle = null,
rating = RATING_UNKNOWN, rating = RATING_UNKNOWN,
tags = emptySet(), tags = emptySet(),
description = div.selectLast("div")?.ownText(), description = div.selectLast("div")?.ownText(),
state = null, state = null,
author = null, author = null,
isNsfw = false, isNsfw = false,
source = source, source = source,
) )
} }
} }
override suspend fun getDetails(manga: Manga): Manga { override suspend fun getDetails(manga: Manga): Manga {
val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml()
val root = doc.selectFirstOrThrow(".perfil-manga") val root = doc.selectFirstOrThrow(".perfil-manga")
val dateFormat = SimpleDateFormat("dd/MM/yyyy", Locale.ROOT) val dateFormat = SimpleDateFormat("dd/MM/yyyy", Locale.ROOT)
return manga.copy( return manga.copy(
rating = root.select("h2") rating = root.select("h2")
.find { it.ownText().startsWith('#') } .find { it.ownText().startsWith('#') }
?.ownText()?.drop(1)?.toFloatOrNull()?.div(10f) ?: manga.rating, ?.ownText()?.drop(1)?.toFloatOrNull()?.div(10f) ?: manga.rating,
largeCoverUrl = root.selectFirst("img.img-thumbnail")?.attrAsAbsoluteUrlOrNull("src"), largeCoverUrl = root.selectFirst("img.img-thumbnail")?.attrAsAbsoluteUrlOrNull("src"),
description = root.selectFirst(".panel-default")?.selectFirst(".panel-body")?.html(), description = root.selectFirst(".panel-default")?.selectFirst(".panel-body")?.html(),
author = root.tableValue("Autor")?.ownText(), author = root.tableValue("Autor")?.ownText(),
altTitle = root.tableValue("Título(s) Alternativo(s)")?.ownText(), altTitle = root.tableValue("Título(s) Alternativo(s)")?.ownText(),
state = when (root.tableValue("Status")?.selectLast(".label")?.text()) { state = when (root.tableValue("Status")?.selectLast(".label")?.text()) {
"Completo" -> MangaState.FINISHED "Completo" -> MangaState.FINISHED
"Ativo" -> MangaState.ONGOING "Ativo" -> MangaState.ONGOING
else -> null else -> null
}, },
tags = root.tableValue("Gênero(s)")?.select("a")?.mapToSet { tags = root.tableValue("Gênero(s)")?.select("a")?.mapToSet {
it.toMangaTag() it.toMangaTag()
} ?: manga.tags, } ?: manga.tags,
isNsfw = root.selectFirst(".alert-danger")?.html()?.contains("18 anos") == true, isNsfw = root.selectFirst(".alert-danger")?.html()?.contains("18 anos") == true,
chapters = root.select("div.row.capitulos").asReversed().mapChapters { i, div -> chapters = root.select("div.row.capitulos").asReversed().mapChapters { i, div ->
val a = div.selectFirstOrThrow("a") val a = div.selectFirstOrThrow("a")
val href = a.attrAsRelativeUrl("href") val href = a.attrAsRelativeUrl("href")
val title = a.text() val title = a.text()
MangaChapter( MangaChapter(
id = generateUid(href), id = generateUid(href),
name = title, name = title,
number = i + 1, number = i + 1,
url = href, url = href,
scanlator = div.selectLast("a")?.text()?.takeUnless { it == title }, scanlator = div.selectLast("a")?.text()?.takeUnless { it == title },
uploadDate = dateFormat.tryParse( uploadDate = dateFormat.tryParse(
a.nextElementSibling()?.text()?.removeSurrounding("(", ")"), a.nextElementSibling()?.text()?.removeSurrounding("(", ")"),
), ),
branch = null, branch = null,
source = source, source = source,
) )
}, },
) )
} }
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(domain) val fullUrl = chapter.url.toAbsoluteUrl(domain)
val doc = webClient.httpGet(fullUrl).parseHtml() val doc = webClient.httpGet(fullUrl).parseHtml()
val root = doc.body().selectFirstOrThrow("article") val root = doc.body().selectFirstOrThrow("article")
return root.selectOrThrow("img[pag]").mapNotNull { img -> return root.selectOrThrow("img[pag]").mapNotNull { img ->
val href = img.attrAsRelativeUrl("src") val href = img.attrAsRelativeUrl("src")
if (href.startsWith("/images/banner")) { if (href.startsWith("/images/banner")) {
return@mapNotNull null return@mapNotNull null
} }
MangaPage( MangaPage(
id = generateUid(href), id = generateUid(href),
url = href, url = href,
referer = fullUrl, referer = fullUrl,
preview = null, preview = null,
source = source, source = source,
) )
} }
} }
override suspend fun getTags(): Set<MangaTag> { override suspend fun getTags(): Set<MangaTag> {
val doc = webClient.httpGet(urlBuilder().addPathSegment("lista-mangas").build()).parseHtml() val doc = webClient.httpGet(urlBuilder().addPathSegment("lista-mangas").build()).parseHtml()
val ul = doc.body().selectFirstOrThrow(".nav-tabs").selectFirstOrThrow("ul.dropdown-menu") val ul = doc.body().selectFirstOrThrow(".nav-tabs").selectFirstOrThrow("ul.dropdown-menu")
return ul.select("li").mapToSet { li -> return ul.select("li").mapToSet { li ->
li.selectFirstOrThrow("a").toMangaTag() li.selectFirstOrThrow("a").toMangaTag()
} }
} }
private suspend fun search(query: String): List<Manga> { private suspend fun search(query: String): List<Manga> {
val domain = domain val domain = domain
val json = webClient.httpGet( val json = webClient.httpGet(
urlBuilder() urlBuilder()
.addPathSegments("assets/busca.php") .addPathSegments("assets/busca.php")
.addQueryParameter("nomeManga", query) .addQueryParameter("nomeManga", query)
.build(), .build(),
).parseJson() ).parseJson()
return json.getJSONArray("items").mapJSON { jo -> return json.getJSONArray("items").mapJSON { jo ->
val href = "/pagina-manga/" + jo.getString("url") val href = "/pagina-manga/" + jo.getString("url")
Manga( Manga(
id = generateUid(href), id = generateUid(href),
url = href, url = href,
publicUrl = href.toAbsoluteUrl(domain), publicUrl = href.toAbsoluteUrl(domain),
title = jo.getString("titulo"), title = jo.getString("titulo"),
rating = RATING_UNKNOWN, rating = RATING_UNKNOWN,
tags = emptySet(), tags = emptySet(),
author = jo.getStringOrNull("autor"), author = jo.getStringOrNull("autor"),
coverUrl = jo.getString("imagem"), coverUrl = jo.getString("imagem"),
state = null, state = null,
isNsfw = false, isNsfw = false,
altTitle = null, altTitle = null,
source = source, source = source,
) )
} }
} }
private fun Element.tableValue(title: String): Element? { private fun Element.tableValue(title: String): Element? {
return select("h4.media-heading") return select("h4.media-heading")
.find { it.selectFirst("label.subtit-manga")?.text()?.contains(title, ignoreCase = true) == true } .find { it.selectFirst("label.subtit-manga")?.text()?.contains(title, ignoreCase = true) == true }
} }
private fun Element.toMangaTag() = MangaTag( private fun Element.toMangaTag() = MangaTag(
title = text().toTitleCase(sourceLocale ?: Locale.ROOT), title = text().toTitleCase(sourceLocale),
key = attr("href").removeSuffix('/').substringAfterLast('/'), key = attr("href").removeSuffix('/').substringAfterLast('/'),
source = source, source = source,
) )
} }

@ -6,13 +6,14 @@ import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.model.Manga import org.koitharu.kotatsu.parsers.model.Manga
import org.koitharu.kotatsu.parsers.model.MangaChapter import org.koitharu.kotatsu.parsers.model.MangaChapter
import org.koitharu.kotatsu.parsers.model.MangaSource import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.model.MangaTag
import org.koitharu.kotatsu.parsers.util.* import org.koitharu.kotatsu.parsers.util.*
import java.text.SimpleDateFormat import java.text.SimpleDateFormat
import java.util.* import java.util.*
@MangaSourceParser("ISEKAISCAN_EU", "IsekaiScan (eu)", "en") @MangaSourceParser("ISEKAISCAN_EU", "IsekaiScan", "en")
internal class IsekaiScanEuParser(context: MangaLoaderContext) : internal class IsekaiScanEuParser(context: MangaLoaderContext) :
MadaraParser(context, MangaSource.ISEKAISCAN_EU, "isekaiscan.eu") { MadaraParser(context, MangaSource.ISEKAISCAN_EU, "isekaiscan.to") {
override val datePattern = "MM/dd/yyyy" override val datePattern = "MM/dd/yyyy"
@ -44,4 +45,29 @@ internal class IsekaiScanEuParser(context: MangaLoaderContext) :
) )
} }
} }
override suspend fun getTags(): Set<MangaTag> {
val doc = webClient.httpGet("https://$domain/mangax/").parseHtml()
val body = doc.body()
val root1 = body.selectFirst("header")?.selectFirst("ul.second-menu")
val root2 = body.selectFirst("div.genres_wrap")?.selectFirst("ul.list-unstyled")
if (root1 == null && root2 == null) {
doc.parseFailed("Root not found")
}
val list = root1?.select("li").orEmpty() + root2?.select("li").orEmpty()
val keySet = HashSet<String>(list.size)
return list.mapNotNullToSet { li ->
val a = li.selectFirst("a") ?: return@mapNotNullToSet null
val href = a.attr("href").removeSuffix("/")
.substringAfterLast(tagPrefix, "")
if (href.isEmpty() || !keySet.add(href)) {
return@mapNotNullToSet null
}
MangaTag(
key = href,
title = a.ownText().toTitleCase(Locale.ENGLISH),
source = source,
)
}
}
} }

@ -13,174 +13,174 @@ import java.text.SimpleDateFormat
import java.util.* import java.util.*
abstract class Madara5Parser @InternalParsersApi constructor( abstract class Madara5Parser @InternalParsersApi constructor(
context: MangaLoaderContext, context: MangaLoaderContext,
source: MangaSource, source: MangaSource,
domain: String, domain: String,
) : PagedMangaParser(context, source, pageSize = 22) { ) : PagedMangaParser(context, source, pageSize = 22) {
protected open val datePattern = "MMMM dd, HH:mm" protected open val datePattern = "MMMM dd, HH:mm"
protected open val tagPrefix = "/mangas/" protected open val tagPrefix = "/mangas/"
protected open val nsfwTags = arraySetOf("yaoi", "yuri", "mature") protected open val nsfwTags = arraySetOf("yaoi", "yuri", "mature")
override val sortOrders: Set<SortOrder> = EnumSet.of(SortOrder.UPDATED) override val sortOrders: Set<SortOrder> = EnumSet.of(SortOrder.UPDATED)
override val configKeyDomain = ConfigKey.Domain(domain, null) override val configKeyDomain = ConfigKey.Domain(domain, null)
override suspend fun getListPage( override suspend fun getListPage(
page: Int, page: Int,
query: String?, query: String?,
tags: Set<MangaTag>?, tags: Set<MangaTag>?,
sortOrder: SortOrder, sortOrder: SortOrder,
): List<Manga> { ): List<Manga> {
val domain = domain val domain = domain
val url = buildString { val url = buildString {
append("https://") append("https://")
append(domain) append(domain)
append("/search?s=") append("/search?s=")
if (!query.isNullOrEmpty()) { if (!query.isNullOrEmpty()) {
append(query.urlEncoded()) append(query.urlEncoded())
} }
append("&post_type=wp-manga") append("&post_type=wp-manga")
if (!tags.isNullOrEmpty()) { if (!tags.isNullOrEmpty()) {
for (tag in tags) { for (tag in tags) {
append("&genre%5B%5D=") append("&genre%5B%5D=")
append(tag.key) append(tag.key)
} }
} }
append("&op=1&author=&artist=&page=") append("&op=1&author=&artist=&page=")
append(page) append(page)
} }
val root = webClient.httpGet(url).parseHtml().body().selectFirstOrThrow(".search-wrap") val root = webClient.httpGet(url).parseHtml().body().selectFirstOrThrow(".search-wrap")
return root.select(".c-tabs-item__content").map { div -> return root.select(".c-tabs-item__content").map { div ->
val a = div.selectFirstOrThrow("a") val a = div.selectFirstOrThrow("a")
val img = div.selectLastOrThrow("img") val img = div.selectLastOrThrow("img")
val href = a.attrAsRelativeUrl("href") val href = a.attrAsRelativeUrl("href")
val postContent = root.selectFirstOrThrow(".post-content") val postContent = root.selectFirstOrThrow(".post-content")
val tagSet = postContent.getElementsContainingOwnText("Genre") val tagSet = postContent.getElementsContainingOwnText("Genre")
.firstOrNull()?.tableValue() .firstOrNull()?.tableValue()
?.getElementsByAttributeValueContaining("href", tagPrefix) ?.getElementsByAttributeValueContaining("href", tagPrefix)
?.mapToSet { it.asMangaTag() }.orEmpty() ?.mapToSet { it.asMangaTag() }.orEmpty()
Manga( Manga(
id = generateUid(href), id = generateUid(href),
title = a.attr("title"), title = a.attr("title"),
altTitle = postContent.getElementsContainingOwnText("Alternative") altTitle = postContent.getElementsContainingOwnText("Alternative")
.firstOrNull()?.tableValue()?.text()?.trim(), .firstOrNull()?.tableValue()?.text()?.trim(),
url = href, url = href,
publicUrl = a.attrAsAbsoluteUrl("href"), publicUrl = a.attrAsAbsoluteUrl("href"),
coverUrl = img.src().orEmpty(), coverUrl = img.src().orEmpty(),
author = postContent.getElementsContainingOwnText("Author") author = postContent.getElementsContainingOwnText("Author")
.firstOrNull()?.tableValue()?.text()?.trim(), .firstOrNull()?.tableValue()?.text()?.trim(),
state = postContent.getElementsContainingOwnText("Status") state = postContent.getElementsContainingOwnText("Status")
.firstOrNull()?.tableValue()?.text()?.asMangaState(), .firstOrNull()?.tableValue()?.text()?.asMangaState(),
isNsfw = isNsfw(tagSet), isNsfw = isNsfw(tagSet),
rating = div.selectFirstOrThrow(".score").text() rating = div.selectFirstOrThrow(".score").text()
.toFloatOrNull()?.div(5f) ?: RATING_UNKNOWN, .toFloatOrNull()?.div(5f) ?: RATING_UNKNOWN,
tags = tagSet, tags = tagSet,
source = source, source = source,
) )
} }
} }
override suspend fun getDetails(manga: Manga): Manga { override suspend fun getDetails(manga: Manga): Manga {
val root = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml().body() val root = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml().body()
.selectFirstOrThrow(".site-content") .selectFirstOrThrow(".site-content")
val postContent = root.selectFirstOrThrow(".post-content") val postContent = root.selectFirstOrThrow(".post-content")
val tags = postContent.getElementsContainingOwnText("Genre") val tags = postContent.getElementsContainingOwnText("Genre")
.firstOrNull()?.tableValue() .firstOrNull()?.tableValue()
?.getElementsByAttributeValueContaining("href", tagPrefix) ?.getElementsByAttributeValueContaining("href", tagPrefix)
?.mapToSet { a -> a.asMangaTag() } ?: manga.tags ?.mapToSet { a -> a.asMangaTag() } ?: manga.tags
val mangaId = root.getElementById("manga-chapters-holder")?.attr("data-id")?.toLongOrNull() val mangaId = root.getElementById("manga-chapters-holder")?.attr("data-id")?.toLongOrNull()
?: root.parseFailed("Cannot find mangaId") ?: root.parseFailed("Cannot find mangaId")
return manga.copy( return manga.copy(
description = (root.selectFirst(".detail-content") description = (root.selectFirst(".detail-content")
?: root.selectFirstOrThrow(".description-summary")).html(), ?: root.selectFirstOrThrow(".description-summary")).html(),
author = postContent.getElementsContainingOwnText("Author") author = postContent.getElementsContainingOwnText("Author")
.firstOrNull()?.tableValue()?.text()?.trim(), .firstOrNull()?.tableValue()?.text()?.trim(),
state = postContent.getElementsContainingOwnText("Status") state = postContent.getElementsContainingOwnText("Status")
.firstOrNull()?.tableValue()?.text()?.asMangaState(), .firstOrNull()?.tableValue()?.text()?.asMangaState(),
tags = tags, tags = tags,
isNsfw = isNsfw(tags), isNsfw = isNsfw(tags),
chapters = loadChapters(mangaId), chapters = loadChapters(mangaId),
) )
} }
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(domain) val fullUrl = chapter.url.toAbsoluteUrl(domain)
val doc = webClient.httpGet(fullUrl).parseHtml() val doc = webClient.httpGet(fullUrl).parseHtml()
val arrayData = doc.getElementById("arraydata") ?: doc.parseFailed("#arraydata not found") val arrayData = doc.getElementById("arraydata") ?: doc.parseFailed("#arraydata not found")
return arrayData.html().split(',').map { url -> return arrayData.html().split(',').map { url ->
MangaPage( MangaPage(
id = generateUid(url), id = generateUid(url),
url = url, url = url,
referer = fullUrl, referer = fullUrl,
preview = null, preview = null,
source = source, source = source,
) )
} }
} }
override suspend fun getTags(): Set<MangaTag> { override suspend fun getTags(): Set<MangaTag> {
val doc = webClient.httpGet("http://${domain}/").parseHtml().body() val doc = webClient.httpGet("http://${domain}/").parseHtml().body()
return doc.getElementsByAttributeValueContaining("href", tagPrefix) return doc.getElementsByAttributeValueContaining("href", tagPrefix)
.mapToSet { it.asMangaTag() } .mapToSet { it.asMangaTag() }
} }
private suspend fun loadChapters(mangaId: Long): List<MangaChapter> { private suspend fun loadChapters(mangaId: Long): List<MangaChapter> {
val dateFormat = SimpleDateFormat(datePattern, sourceLocale ?: Locale.US) val dateFormat = SimpleDateFormat(datePattern, sourceLocale)
val doc = webClient.httpGet("https://${domain}/ajax-list-chapter?mangaID=$mangaId").parseHtml() val doc = webClient.httpGet("https://${domain}/ajax-list-chapter?mangaID=$mangaId").parseHtml()
return doc.select("li.wp-manga-chapter").asReversed().mapChapters { i, li -> return doc.select("li.wp-manga-chapter").asReversed().mapChapters { i, li ->
val a = li.selectFirstOrThrow("a") val a = li.selectFirstOrThrow("a")
val href = a.attrAsRelativeUrl("href") val href = a.attrAsRelativeUrl("href")
MangaChapter( MangaChapter(
id = generateUid(href), id = generateUid(href),
url = href, url = href,
name = a.text(), name = a.text(),
number = i + 1, number = i + 1,
branch = null, branch = null,
uploadDate = dateFormat.tryParse( uploadDate = dateFormat.tryParse(
li.selectFirst(".chapter-release-date")?.text()?.trim(), li.selectFirst(".chapter-release-date")?.text()?.trim(),
), ),
scanlator = null, scanlator = null,
source = source, source = source,
) )
} }
} }
protected fun isNsfw(tags: Set<MangaTag>): Boolean { protected fun isNsfw(tags: Set<MangaTag>): Boolean {
return tags.any { it.key in nsfwTags } return tags.any { it.key in nsfwTags }
} }
private fun Element.src(): String? { private fun Element.src(): String? {
return absUrl("data-src").ifEmpty { return absUrl("data-src").ifEmpty {
absUrl("src") absUrl("src")
}.takeUnless { it.isEmpty() } }.takeUnless { it.isEmpty() }
} }
private fun Element.tableValue(): Element { private fun Element.tableValue(): Element {
for (p in parents()) { for (p in parents()) {
val children = p.children() val children = p.children()
if (children.size == 2) { if (children.size == 2) {
return children[1] return children[1]
} }
} }
parseFailed("Cannot find tableValue for node ${text()}") parseFailed("Cannot find tableValue for node ${text()}")
} }
private fun String.asMangaState() = when (trim().lowercase(sourceLocale ?: Locale.US)) { private fun String.asMangaState() = when (trim().lowercase(sourceLocale)) {
"ongoing" -> MangaState.ONGOING "ongoing" -> MangaState.ONGOING
"completed" -> MangaState.FINISHED "completed" -> MangaState.FINISHED
else -> null else -> null
} }
private fun Element.asMangaTag() = MangaTag( private fun Element.asMangaTag() = MangaTag(
title = ownText(), title = ownText(),
key = attr("href").removeSuffix('/').substringAfterLast('/') key = attr("href").removeSuffix('/').substringAfterLast('/')
.replace('-', '+'), .replace('-', '+'),
source = source, source = source,
) )
@MangaSourceParser("MANGAOWLS", "BeautyManga", "en") @MangaSourceParser("MANGAOWLS", "BeautyManga", "en")
class BeautyManga(context: MangaLoaderContext) : Madara5Parser(context, MangaSource.MANGAOWLS, "beautymanga.com") { class BeautyManga(context: MangaLoaderContext) : Madara5Parser(context, MangaSource.MANGAOWLS, "beautymanga.com") {
} }
} }

@ -7,64 +7,63 @@ import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.model.* import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.* import org.koitharu.kotatsu.parsers.util.*
import java.text.SimpleDateFormat import java.text.SimpleDateFormat
import java.util.*
internal abstract class Madara6Parser( internal abstract class Madara6Parser(
context: MangaLoaderContext, context: MangaLoaderContext,
source: MangaSource, source: MangaSource,
domain: String, domain: String,
) : MadaraParser(context, source, domain) { ) : MadaraParser(context, source, domain) {
override val datePattern: String = "dd MMMM yyyy" override val datePattern: String = "dd MMMM yyyy"
override suspend fun getDetails(manga: Manga): Manga { override suspend fun getDetails(manga: Manga): Manga {
return coroutineScope { return coroutineScope {
val chapters = async { loadChapters(manga.url) } val chapters = async { loadChapters(manga.url) }
val body = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml().body() val body = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml().body()
parseDetails(manga, body, chapters.await()) parseDetails(manga, body, chapters.await())
} }
} }
protected fun Element.tableValue(): Element { protected fun Element.tableValue(): Element {
for (p in parents()) { for (p in parents()) {
val children = p.children() val children = p.children()
if (children.size == 2) { if (children.size == 2) {
return children[1] return children[1]
} }
} }
parseFailed("Cannot find tableValue for node ${text()}") parseFailed("Cannot find tableValue for node ${text()}")
} }
protected abstract fun String.asMangaState(): MangaState? protected abstract fun String.asMangaState(): MangaState?
protected fun Element.asMangaTag() = MangaTag( protected fun Element.asMangaTag() = MangaTag(
title = ownText(), title = ownText(),
key = attr("href").removeSuffix('/').substringAfterLast('/') key = attr("href").removeSuffix('/').substringAfterLast('/')
.replace('-', '+'), .replace('-', '+'),
source = source, source = source,
) )
protected open suspend fun loadChapters(mangaUrl: String): List<MangaChapter> { protected open suspend fun loadChapters(mangaUrl: String): List<MangaChapter> {
val url = mangaUrl.toAbsoluteUrl(domain).removeSuffix('/') + "/ajax/chapters/" val url = mangaUrl.toAbsoluteUrl(domain).removeSuffix('/') + "/ajax/chapters/"
val dateFormat = SimpleDateFormat(datePattern, sourceLocale ?: Locale.ROOT) val dateFormat = SimpleDateFormat(datePattern, sourceLocale)
val doc = webClient.httpPost(url, emptyMap()).parseHtml() val doc = webClient.httpPost(url, emptyMap()).parseHtml()
return doc.select("li.wp-manga-chapter").asReversed().mapChapters { i, li -> return doc.select("li.wp-manga-chapter").asReversed().mapChapters { i, li ->
val a = li.selectFirstOrThrow("a") val a = li.selectFirstOrThrow("a")
val href = a.attrAsRelativeUrl("href") val href = a.attrAsRelativeUrl("href")
MangaChapter( MangaChapter(
id = generateUid(href), id = generateUid(href),
url = href, url = href,
name = a.text(), name = a.text(),
number = i + 1, number = i + 1,
branch = null, branch = null,
uploadDate = dateFormat.tryParse( uploadDate = dateFormat.tryParse(
li.selectFirst(".chapter-release-date")?.text()?.trim(), li.selectFirst(".chapter-release-date")?.text()?.trim(),
), ),
scanlator = null, scanlator = null,
source = source, source = source,
) )
} }
} }
protected abstract fun parseDetails(manga: Manga, body: Element, chapters: List<MangaChapter>): Manga protected abstract fun parseDetails(manga: Manga, body: Element, chapters: List<MangaChapter>): Manga
} }

@ -153,7 +153,7 @@ internal abstract class MadaraParser(
protected open suspend fun getChapters(manga: Manga, doc: Document): List<MangaChapter> { protected open suspend fun getChapters(manga: Manga, doc: Document): List<MangaChapter> {
val root2 = doc.body().selectFirstOrThrow("div.content-area") val root2 = doc.body().selectFirstOrThrow("div.content-area")
.selectFirstOrThrow("div.c-page") .selectFirstOrThrow("div.c-page")
val dateFormat = SimpleDateFormat(datePattern, sourceLocale ?: Locale.US) val dateFormat = SimpleDateFormat(datePattern, sourceLocale)
return root2.select("li").asReversed().mapChapters { i, li -> return root2.select("li").asReversed().mapChapters { i, li ->
val a = li.selectFirst("a") val a = li.selectFirst("a")
val href = a?.attrAsRelativeUrlOrNull("href") ?: li.parseFailed("Link is missing") val href = a?.attrAsRelativeUrlOrNull("href") ?: li.parseFailed("Link is missing")
@ -285,11 +285,11 @@ internal abstract class MadaraParser(
} }
} }
private fun Element.src(): String? { protected fun Element.src(): String? {
var result = absUrl("data-src") var result = absUrl("data-src")
if (result.isEmpty()) result = absUrl("data-cfsrc") if (result.isEmpty()) result = absUrl("data-cfsrc")
if (result.isEmpty()) result = absUrl("src") if (result.isEmpty()) result = absUrl("src")
return if (result.isEmpty()) null else result return result.ifEmpty { null }
} }
private fun createRequestTemplate() = private fun createRequestTemplate() =

@ -5,10 +5,7 @@ import kotlinx.coroutines.coroutineScope
import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.exception.ParseException import org.koitharu.kotatsu.parsers.exception.ParseException
import org.koitharu.kotatsu.parsers.model.Manga import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.model.MangaChapter
import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.model.MangaTag
import org.koitharu.kotatsu.parsers.util.* import org.koitharu.kotatsu.parsers.util.*
import java.text.SimpleDateFormat import java.text.SimpleDateFormat
import java.util.* import java.util.*
@ -16,6 +13,8 @@ import java.util.*
@MangaSourceParser("NEATMANGA", "NeatManga", "en") @MangaSourceParser("NEATMANGA", "NeatManga", "en")
internal class NeatManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.NEATMANGA, "neatmangas.com") { internal class NeatManga(context: MangaLoaderContext) : MadaraParser(context, MangaSource.NEATMANGA, "neatmangas.com") {
override val datePattern = "dd MMMM yyyy"
override suspend fun getDetails(manga: Manga): Manga = coroutineScope { override suspend fun getDetails(manga: Manga): Manga = coroutineScope {
val chaptersDeferred = async { getChapters(manga) } val chaptersDeferred = async { getChapters(manga) }
val fullUrl = manga.url.toAbsoluteUrl(domain) val fullUrl = manga.url.toAbsoluteUrl(domain)
@ -72,4 +71,26 @@ internal class NeatManga(context: MangaLoaderContext) : MadaraParser(context, Ma
) )
} }
} }
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.toAbsoluteUrl(domain)
val doc = webClient.httpGet(fullUrl).parseHtml()
val root = doc.body().selectFirst("div.main-col-inner")
?.selectFirst("div.reading-content")
?: throw ParseException("Root not found", fullUrl)
return root.select("div.page-break").mapNotNull { div ->
val img = div.selectFirst("img")
if (img == null || img.attr("id").isNullOrEmpty()) {
return@mapNotNull null
}
val url = img.src()?.toRelativeUrl(domain) ?: div.parseFailed("Image src not found")
MangaPage(
id = generateUid(url),
url = url,
preview = null,
referer = fullUrl,
source = source,
)
}
}
} }

@ -10,45 +10,44 @@ import org.koitharu.kotatsu.parsers.model.MangaState
import org.koitharu.kotatsu.parsers.util.attrAsAbsoluteUrlOrNull import org.koitharu.kotatsu.parsers.util.attrAsAbsoluteUrlOrNull
import org.koitharu.kotatsu.parsers.util.mapToSet import org.koitharu.kotatsu.parsers.util.mapToSet
import org.koitharu.kotatsu.parsers.util.selectFirstOrThrow import org.koitharu.kotatsu.parsers.util.selectFirstOrThrow
import java.util.*
@MangaSourceParser("PRISMA_SCANS", "Prisma Scans", "pt") @MangaSourceParser("PRISMA_SCANS", "Prisma Scans", "pt")
internal class PrismaScansParser(context: MangaLoaderContext) : internal class PrismaScansParser(context: MangaLoaderContext) :
Madara6Parser(context, MangaSource.PRISMA_SCANS, "prismascans.net") { Madara6Parser(context, MangaSource.PRISMA_SCANS, "prismascans.net") {
override val tagPrefix = "manga-genre/" override val tagPrefix = "manga-genre/"
override val datePattern = "MMM dd, yyyy" override val datePattern = "MMM dd, yyyy"
override fun parseDetails(manga: Manga, body: Element, chapters: List<MangaChapter>): Manga { override fun parseDetails(manga: Manga, body: Element, chapters: List<MangaChapter>): Manga {
val root = body.selectFirstOrThrow(".site-content") val root = body.selectFirstOrThrow(".site-content")
val postContent = root.selectFirstOrThrow(".post-content") val postContent = root.selectFirstOrThrow(".post-content")
val tags = postContent.getElementsContainingOwnText("Gênero") val tags = postContent.getElementsContainingOwnText("Gênero")
.firstOrNull()?.tableValue() .firstOrNull()?.tableValue()
?.getElementsByAttributeValueContaining("href", tagPrefix) ?.getElementsByAttributeValueContaining("href", tagPrefix)
?.mapToSet { a -> a.asMangaTag() } ?: manga.tags ?.mapToSet { a -> a.asMangaTag() } ?: manga.tags
return manga.copy( return manga.copy(
largeCoverUrl = root.selectFirst("picture") largeCoverUrl = root.selectFirst("picture")
?.selectFirst("img[data-src]") ?.selectFirst("img[data-src]")
?.attrAsAbsoluteUrlOrNull("data-src"), ?.attrAsAbsoluteUrlOrNull("data-src"),
description = root.selectFirstOrThrow(".manga-excerpt").firstElementChild()?.html(), description = root.selectFirstOrThrow(".manga-excerpt").firstElementChild()?.html(),
author = postContent.getElementsContainingOwnText("Artista") author = postContent.getElementsContainingOwnText("Artista")
.firstOrNull()?.tableValue()?.text()?.trim(), .firstOrNull()?.tableValue()?.text()?.trim(),
altTitle = postContent.getElementsContainingOwnText("Título Alternativo") altTitle = postContent.getElementsContainingOwnText("Título Alternativo")
.firstOrNull()?.tableValue()?.text()?.trim(), .firstOrNull()?.tableValue()?.text()?.trim(),
state = postContent.getElementsContainingOwnText("Status") state = postContent.getElementsContainingOwnText("Status")
.firstOrNull()?.tableValue()?.text()?.asMangaState(), .firstOrNull()?.tableValue()?.text()?.asMangaState(),
tags = tags, tags = tags,
isNsfw = body.hasClass("adult-content"), isNsfw = body.hasClass("adult-content"),
chapters = chapters, chapters = chapters,
) )
} }
override fun String.asMangaState() = when (trim().lowercase(sourceLocale ?: Locale.ROOT)) { override fun String.asMangaState() = when (trim().lowercase(sourceLocale)) {
"em lançamento" -> MangaState.ONGOING "em lançamento" -> MangaState.ONGOING
"completo", "completo",
"cancelado", "cancelado",
-> MangaState.FINISHED -> MangaState.FINISHED
else -> null else -> null
} }
} }

@ -223,6 +223,7 @@ internal abstract class MangaReaderParser(
private fun Element.imageUrl(): String { private fun Element.imageUrl(): String {
return attrAsAbsoluteUrlOrNull("src") return attrAsAbsoluteUrlOrNull("src")
?: attrAsAbsoluteUrlOrNull("data-src")
?: attrAsAbsoluteUrlOrNull("data-cfsrc") ?: attrAsAbsoluteUrlOrNull("data-cfsrc")
?: "" ?: ""
} }
@ -258,6 +259,7 @@ internal abstract class MangaReaderParser(
override val configKeyDomain: ConfigKey.Domain override val configKeyDomain: ConfigKey.Domain
get() = ConfigKey.Domain("manhwaindo.id", null) get() = ConfigKey.Domain("manhwaindo.id", null)
override val chapterDateFormat = SimpleDateFormat("MMMM dd, yyyy", Locale.ENGLISH)
override val listUrl: String get() = "/series" override val listUrl: String get() = "/series"
override val tableMode: Boolean get() = false override val tableMode: Boolean get() = false
} }
@ -499,7 +501,7 @@ internal abstract class MangaReaderParser(
override val tableMode: Boolean override val tableMode: Boolean
get() = false get() = false
override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH) override val chapterDateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", sourceLocale)
override suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List<MangaChapter>): Manga { override suspend fun parseInfoList(docs: Document, manga: Manga, chapters: List<MangaChapter>): Manga {
val infoElement = docs.selectFirst("div.infox") val infoElement = docs.selectFirst("div.infox")
@ -542,7 +544,7 @@ internal abstract class MangaReaderParser(
class KomikLokalParser(context: MangaLoaderContext) : class KomikLokalParser(context: MangaLoaderContext) :
MangaReaderParser(context, MangaSource.KOMIKLOKAL, pageSize = 20, searchPageSize = 10) { MangaReaderParser(context, MangaSource.KOMIKLOKAL, pageSize = 20, searchPageSize = 10) {
override val configKeyDomain: ConfigKey.Domain override val configKeyDomain: ConfigKey.Domain
get() = ConfigKey.Domain("komiklokal.pics", null) get() = ConfigKey.Domain("komikmirror.art", null)
override val listUrl: String override val listUrl: String
get() = "/manga" get() = "/manga"
@ -592,7 +594,7 @@ internal abstract class MangaReaderParser(
class KomikDewasaParser(context: MangaLoaderContext) : class KomikDewasaParser(context: MangaLoaderContext) :
MangaReaderParser(context, MangaSource.KOMIKDEWASA, pageSize = 20, searchPageSize = 10) { MangaReaderParser(context, MangaSource.KOMIKDEWASA, pageSize = 20, searchPageSize = 10) {
override val configKeyDomain: ConfigKey.Domain override val configKeyDomain: ConfigKey.Domain
get() = ConfigKey.Domain("komikdewasa.club", null) get() = ConfigKey.Domain("komikdewasa.info", null)
override val listUrl: String override val listUrl: String
get() = "/manga" get() = "/manga"
@ -656,7 +658,7 @@ internal abstract class MangaReaderParser(
class KomikMangaParser(context: MangaLoaderContext) : class KomikMangaParser(context: MangaLoaderContext) :
MangaReaderParser(context, MangaSource.KOMIKMANGA, pageSize = 20, searchPageSize = 10) { MangaReaderParser(context, MangaSource.KOMIKMANGA, pageSize = 20, searchPageSize = 10) {
override val configKeyDomain: ConfigKey.Domain override val configKeyDomain: ConfigKey.Domain
get() = ConfigKey.Domain("komikmanga.xyz", null) get() = ConfigKey.Domain("komikhentai.co", null)
override val listUrl: String override val listUrl: String
get() = "/project" get() = "/project"

@ -32,7 +32,7 @@ fun Element.attrOrNull(attributeKey: String) = attr(attributeKey).takeUnless { i
*/ */
fun Element.attrAsRelativeUrlOrNull(attributeKey: String): String? { fun Element.attrAsRelativeUrlOrNull(attributeKey: String): String? {
val attr = attr(attributeKey).trim() val attr = attr(attributeKey).trim()
if (attr.isEmpty()) { if (attr.isEmpty() || attr.startsWith("data:")) {
return null return null
} }
if (attr.startsWith("/")) { if (attr.startsWith("/")) {
@ -63,7 +63,7 @@ fun Element.attrAsRelativeUrl(attributeKey: String): String {
*/ */
fun Element.attrAsAbsoluteUrlOrNull(attributeKey: String): String? { fun Element.attrAsAbsoluteUrlOrNull(attributeKey: String): String? {
val attr = attr(attributeKey).trim() val attr = attr(attributeKey).trim()
if (attr.isEmpty()) { if (attr.isEmpty() || attr.startsWith("data:")) {
return null return null
} }
return (baseUri().toHttpUrlOrNull()?.newBuilder(attr) ?: return null).toString() return (baseUri().toHttpUrlOrNull()?.newBuilder(attr) ?: return null).toString()
@ -114,4 +114,4 @@ fun Element.selectLast(cssQuery: String): Element? {
fun Element.selectLastOrThrow(cssQuery: String): Element { fun Element.selectLastOrThrow(cssQuery: String): Element {
return selectLast(cssQuery) ?: throw ParseException("Cannot find \"$cssQuery\"", baseUri()) return selectLast(cssQuery) ?: throw ParseException("Cannot find \"$cssQuery\"", baseUri())
} }

@ -23,7 +23,7 @@ internal class MangaParserTest {
@MangaSources @MangaSources
fun list(source: MangaSource) = runTest { fun list(source: MangaSource) = runTest {
val parser = source.newParser(context) val parser = source.newParser(context)
val list = parser.getList(20, sortOrder = SortOrder.POPULARITY, tags = null) val list = parser.getList(0, sortOrder = SortOrder.POPULARITY, tags = null)
checkMangaList(list, "list") checkMangaList(list, "list")
assert(list.all { it.source == source }) assert(list.all { it.source == source })
} }
@ -46,7 +46,7 @@ internal class MangaParserTest {
@MangaSources @MangaSources
fun search(source: MangaSource) = runTest { fun search(source: MangaSource) = runTest {
val parser = source.newParser(context) val parser = source.newParser(context)
val subject = parser.getList(20, sortOrder = SortOrder.POPULARITY, tags = null).minByOrNull { val subject = parser.getList(0, sortOrder = SortOrder.POPULARITY, tags = null).minByOrNull {
it.title.length it.title.length
} ?: error("No manga found") } ?: error("No manga found")
val query = subject.title val query = subject.title
@ -84,7 +84,7 @@ internal class MangaParserTest {
@MangaSources @MangaSources
fun details(source: MangaSource) = runTest { fun details(source: MangaSource) = runTest {
val parser = source.newParser(context) val parser = source.newParser(context)
val list = parser.getList(20, sortOrder = SortOrder.POPULARITY, tags = null) val list = parser.getList(0, sortOrder = SortOrder.POPULARITY, tags = null)
val manga = list[3] val manga = list[3]
parser.getDetails(manga).apply { parser.getDetails(manga).apply {
assert(!chapters.isNullOrEmpty()) { "Chapters are null or empty" } assert(!chapters.isNullOrEmpty()) { "Chapters are null or empty" }
@ -113,7 +113,7 @@ internal class MangaParserTest {
@MangaSources @MangaSources
fun pages(source: MangaSource) = runTest { fun pages(source: MangaSource) = runTest {
val parser = source.newParser(context) val parser = source.newParser(context)
val list = parser.getList(20, sortOrder = SortOrder.POPULARITY, tags = null) val list = parser.getList(0, sortOrder = SortOrder.POPULARITY, tags = null)
val manga = list.first() val manga = list.first()
val chapter = parser.getDetails(manga).chapters?.firstOrNull() ?: error("Chapter is null") val chapter = parser.getDetails(manga).chapters?.firstOrNull() ?: error("Chapter is null")
val pages = parser.getPages(chapter) val pages = parser.getPages(chapter)

Loading…
Cancel
Save