[LibSocial] Rewrite parsers
parent
3e32a6280a
commit
952e9c39ac
GPG Key ID:
676DEE768C17A9D7
@ -1,14 +1,14 @@
|
||||
package org.koitharu.kotatsu.parsers.site.ru.rulib
|
||||
|
||||
import org.jsoup.nodes.Document
|
||||
import org.koitharu.kotatsu.parsers.MangaLoaderContext
|
||||
import org.koitharu.kotatsu.parsers.MangaSourceParser
|
||||
import org.koitharu.kotatsu.parsers.config.ConfigKey
|
||||
import org.koitharu.kotatsu.parsers.model.ContentType
|
||||
import org.koitharu.kotatsu.parsers.model.MangaSource
|
||||
|
||||
@MangaSourceParser("HENTAILIB", "HentaiLib", "ru", type = ContentType.HENTAI)
|
||||
internal class HentaiLibParser(context: MangaLoaderContext) : MangaLibParser(context, MangaSource.HENTAILIB) {
|
||||
override val configKeyDomain = ConfigKey.Domain("hentailib.me")
|
||||
override fun isNsfw(doc: Document) = true
|
||||
}
|
||||
internal class HentaiLibParser(context: MangaLoaderContext) : LibSocialParser(
|
||||
context = context,
|
||||
source = MangaSource.HENTAILIB,
|
||||
siteId = 4,
|
||||
siteDomain = "hentailib.me",
|
||||
)
|
||||
|
||||
@ -0,0 +1,307 @@
|
||||
package org.koitharu.kotatsu.parsers.site.ru.rulib
|
||||
|
||||
import androidx.collection.*
|
||||
import kotlinx.coroutines.async
|
||||
import kotlinx.coroutines.coroutineScope
|
||||
import org.json.JSONArray
|
||||
import org.json.JSONObject
|
||||
import org.koitharu.kotatsu.parsers.MangaLoaderContext
|
||||
import org.koitharu.kotatsu.parsers.PagedMangaParser
|
||||
import org.koitharu.kotatsu.parsers.config.ConfigKey
|
||||
import org.koitharu.kotatsu.parsers.model.*
|
||||
import org.koitharu.kotatsu.parsers.util.*
|
||||
import org.koitharu.kotatsu.parsers.util.json.*
|
||||
import java.text.SimpleDateFormat
|
||||
import java.util.*
|
||||
|
||||
internal abstract class LibSocialParser(
|
||||
context: MangaLoaderContext,
|
||||
source: MangaSource,
|
||||
protected val siteDomain: String,
|
||||
protected val siteId: Int,
|
||||
) : PagedMangaParser(context, source, pageSize = 60) {
|
||||
|
||||
override val availableSortOrders: Set<SortOrder> = EnumSet.allOf(SortOrder::class.java)
|
||||
|
||||
final override val configKeyDomain = ConfigKey.Domain("lib.social")
|
||||
override val availableStates: Set<MangaState> = EnumSet.allOf(MangaState::class.java)
|
||||
override val isMultipleTagsSupported = true
|
||||
override val isTagsExclusionSupported = true
|
||||
override val isSearchSupported = true
|
||||
|
||||
private val statesMap = intObjectMapOf(
|
||||
1, MangaState.ONGOING,
|
||||
2, MangaState.FINISHED,
|
||||
3, MangaState.UPCOMING,
|
||||
4, MangaState.PAUSED,
|
||||
5, MangaState.ABANDONED,
|
||||
)
|
||||
private val imageServers = SuspendLazy(::fetchServers)
|
||||
|
||||
override suspend fun getListPage(page: Int, filter: MangaListFilter?): List<Manga> {
|
||||
val urlBuilder = urlBuilder("api")
|
||||
.addPathSegment("api")
|
||||
.addPathSegment("manga")
|
||||
.addQueryParameter("site_id[]", siteId.toString())
|
||||
.addQueryParameter("fields[]", "rate")
|
||||
.addQueryParameter("fields[]", "rate_avg")
|
||||
.addQueryParameter("page", page.toString())
|
||||
when (filter) {
|
||||
is MangaListFilter.Advanced -> {
|
||||
for (state in filter.states) {
|
||||
urlBuilder.addQueryParameter("status[]", statesMap.keyOf(state).toString())
|
||||
}
|
||||
for (tag in filter.tags) {
|
||||
urlBuilder.addQueryParameter("${tag.typeKey()}[]", tag.key.drop(1))
|
||||
}
|
||||
for (tag in filter.tagsExclude) {
|
||||
urlBuilder.addQueryParameter("${tag.typeKey()}_exclude[]", tag.key.drop(1))
|
||||
}
|
||||
}
|
||||
|
||||
is MangaListFilter.Search -> {
|
||||
urlBuilder.addQueryParameter("q", filter.query)
|
||||
}
|
||||
|
||||
null -> Unit
|
||||
}
|
||||
val sortOrder = filter?.sortOrder ?: defaultSortOrder
|
||||
urlBuilder.addQueryParameter(
|
||||
"sort_by",
|
||||
when (sortOrder) {
|
||||
SortOrder.UPDATED -> "last_chapter_at"
|
||||
SortOrder.POPULARITY -> "views"
|
||||
SortOrder.RATING -> "rate_avg"
|
||||
SortOrder.NEWEST -> "created_at"
|
||||
SortOrder.ALPHABETICAL,
|
||||
SortOrder.ALPHABETICAL_DESC,
|
||||
-> "rus_name"
|
||||
},
|
||||
)
|
||||
urlBuilder.addQueryParameter(
|
||||
"sort_type",
|
||||
when (sortOrder) {
|
||||
SortOrder.UPDATED,
|
||||
SortOrder.POPULARITY,
|
||||
SortOrder.RATING,
|
||||
SortOrder.NEWEST,
|
||||
SortOrder.ALPHABETICAL_DESC,
|
||||
-> "desc"
|
||||
|
||||
SortOrder.ALPHABETICAL -> "asc"
|
||||
},
|
||||
)
|
||||
val json = webClient.httpGet(urlBuilder.build()).parseJson()
|
||||
val data = json.getJSONArray("data")
|
||||
return data.mapJSON(::parseManga)
|
||||
}
|
||||
|
||||
override suspend fun getDetails(manga: Manga): Manga = coroutineScope {
|
||||
val chapters = async { fetchChapters(manga) }
|
||||
val url = urlBuilder("api")
|
||||
.addPathSegment("api")
|
||||
.addPathSegment("manga")
|
||||
.addPathSegment(manga.url)
|
||||
.addQueryParameter("fields[]", "summary")
|
||||
.addQueryParameter("fields[]", "genres")
|
||||
.addQueryParameter("fields[]", "tags")
|
||||
.addQueryParameter("fields[]", "authors")
|
||||
.build()
|
||||
val json = webClient.httpGet(url).parseJson().getJSONObject("data")
|
||||
val genres = json.getJSONArray("genres").mapJSON { jo ->
|
||||
MangaTag(title = jo.getString("name"), key = "g" + jo.getInt("id"), source = source)
|
||||
}
|
||||
val tags = json.getJSONArray("genres").mapJSON { jo ->
|
||||
MangaTag(title = jo.getString("name"), key = "t" + jo.getInt("id"), source = source)
|
||||
}
|
||||
manga.copy(
|
||||
title = json.getStringOrNull("rus_name") ?: manga.title,
|
||||
altTitle = json.getString("name"),
|
||||
tags = tagsSetOf(tags, genres),
|
||||
author = json.getJSONArray("authors").optJSONObject(0)?.getStringOrNull("name"),
|
||||
description = json.getString("summary").nl2br(),
|
||||
chapters = chapters.await(),
|
||||
)
|
||||
}
|
||||
|
||||
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> = coroutineScope {
|
||||
val pages = async {
|
||||
webClient.httpGet(
|
||||
concatUrl("https://api.$domain/api/manga/", chapter.url),
|
||||
).parseJson().getJSONObject("data")
|
||||
}
|
||||
val servers = imageServers.get()
|
||||
val json = pages.await()
|
||||
val primaryServer =
|
||||
checkNotNull(servers[SERVER_MAIN] ?: servers[SERVER_DOWNLOAD] ?: servers[SERVER_SECONDARY]) {
|
||||
"No available images servers"
|
||||
}
|
||||
json.getJSONArray("pages").mapJSON { jo ->
|
||||
val url = jo.getString("url")
|
||||
MangaPage(
|
||||
id = generateUid(jo.getLong("id")),
|
||||
url = concatUrl(primaryServer, url),
|
||||
preview = servers[SERVER_COMPRESS]?.let { concatUrl(it, url) },
|
||||
source = source,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
override suspend fun getAvailableTags(): Set<MangaTag> = coroutineScope {
|
||||
val tags = async { fetchTags("tags") }
|
||||
val genres = async { fetchTags("genres") }
|
||||
tagsSetOf(tags.await(), genres.await())
|
||||
}
|
||||
|
||||
override suspend fun getRelatedManga(seed: Manga): List<Manga> {
|
||||
val json = webClient.httpGet(
|
||||
urlBuilder("api")
|
||||
.addPathSegment("api")
|
||||
.addPathSegment("manga")
|
||||
.addPathSegment(seed.url)
|
||||
.addPathSegment("similar")
|
||||
.build(),
|
||||
).parseJson().getJSONArray("data")
|
||||
return json.mapJSON { jo ->
|
||||
parseManga(jo.getJSONObject("media"))
|
||||
}
|
||||
}
|
||||
|
||||
private fun parseManga(jo: JSONObject): Manga {
|
||||
val cover = jo.getJSONObject("cover")
|
||||
return Manga(
|
||||
id = generateUid(jo.getLong("id")),
|
||||
title = jo.getString("rus_name").ifEmpty { jo.getString("name") },
|
||||
altTitle = jo.getString("name"),
|
||||
url = jo.getString("slug_url"),
|
||||
publicUrl = "https://$siteDomain/ru/manga/" + jo.getString("slug_url"),
|
||||
rating = jo.optJSONObject("rating")
|
||||
?.getFloatOrDefault("average", RATING_UNKNOWN * 10f)?.div(10f) ?: RATING_UNKNOWN,
|
||||
isNsfw = jo.getJSONObject("ageRestriction").getIntOrDefault("id", 0) >= 3,
|
||||
coverUrl = cover.getString("thumbnail"),
|
||||
tags = setOf(),
|
||||
state = statesMap[jo.optJSONObject("status")?.getIntOrDefault("id", -1) ?: -1],
|
||||
author = null,
|
||||
largeCoverUrl = cover.getString("default"),
|
||||
source = source,
|
||||
)
|
||||
}
|
||||
|
||||
private suspend fun fetchChapters(manga: Manga): List<MangaChapter> {
|
||||
val url = urlBuilder("api")
|
||||
.addPathSegment("api")
|
||||
.addPathSegment("manga")
|
||||
.addPathSegment(manga.url)
|
||||
.addPathSegment("chapters")
|
||||
.build()
|
||||
val json = webClient.httpGet(url).parseJson().getJSONArray("data")
|
||||
val builder = ChaptersListBuilder(json.length())
|
||||
val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.US)
|
||||
for (i in 0 until json.length()) {
|
||||
val jo = json.getJSONObject(i)
|
||||
val volume = jo.getIntOrDefault("volume", 0)
|
||||
val number = jo.getFloatOrDefault("number", 0f)
|
||||
val numberString = number.formatSimple()
|
||||
val name = jo.getStringOrNull("name") ?: buildString {
|
||||
if (volume > 0) append("Том ").append(volume).append(' ')
|
||||
append("Глава ").append(numberString)
|
||||
}
|
||||
val branches = jo.getJSONArray("branches")
|
||||
for (j in 0 until branches.length()) {
|
||||
val bjo = branches.getJSONObject(j)
|
||||
val id = bjo.getLong("id")
|
||||
val team = bjo.getJSONArray("teams").optJSONObject(0)?.getStringOrNull("name")
|
||||
builder += MangaChapter(
|
||||
id = generateUid(id),
|
||||
name = name,
|
||||
number = number,
|
||||
volume = volume,
|
||||
url = "${manga.url}/chapter?number=$numberString&volume=$volume",
|
||||
scanlator = team,
|
||||
uploadDate = dateFormat.tryParse(bjo.getStringOrNull("created_at")),
|
||||
branch = team,
|
||||
source = source,
|
||||
)
|
||||
}
|
||||
}
|
||||
return builder.toList()
|
||||
}
|
||||
|
||||
private suspend fun fetchTags(type: String): List<MangaTag> {
|
||||
val data = webClient.httpGet(
|
||||
urlBuilder("api").addPathSegment("api").addPathSegment(type).build(),
|
||||
).parseJson().getJSONArray("data")
|
||||
val prefix = type.first().toString()
|
||||
return data.mapJSONNotNull { jo ->
|
||||
val sites = jo.getJSONArray("site_ids").toIntSet()
|
||||
if (siteId !in sites) {
|
||||
return@mapJSONNotNull null
|
||||
}
|
||||
MangaTag(
|
||||
title = jo.getString("name"),
|
||||
key = prefix + jo.getInt("id"),
|
||||
source = source,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
private suspend fun fetchServers(): ScatterMap<String, String> {
|
||||
val json = webClient.httpGet(
|
||||
urlBuilder("api")
|
||||
.addPathSegment("api")
|
||||
.addPathSegment("constants")
|
||||
.addQueryParameter("fields[]", "imageServers")
|
||||
.build(),
|
||||
).parseJson().getJSONObject("data").getJSONArray("imageServers")
|
||||
val result = MutableScatterMap<String, String>()
|
||||
for (i in 0 until json.length()) {
|
||||
val jo = json.getJSONObject(i)
|
||||
val sites = jo.getJSONArray("site_ids").toIntSet()
|
||||
if (siteId !in sites) {
|
||||
continue
|
||||
}
|
||||
result[jo.getString("id")] = jo.getString("url")
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
private fun <V> IntObjectMap<V>.keyOf(value: V): Int {
|
||||
forEach { k, v ->
|
||||
if (v == value) {
|
||||
return k
|
||||
}
|
||||
}
|
||||
throw NoSuchElementException("No key associated with value $value")
|
||||
}
|
||||
|
||||
private fun JSONArray.toIntSet(): IntSet {
|
||||
val result = MutableIntSet(length())
|
||||
for (i in 0 until length()) {
|
||||
result.add(getInt(i))
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
private fun MangaTag.typeKey() = when (key.firstOrNull()) {
|
||||
'g' -> "genres"
|
||||
't' -> "tags"
|
||||
else -> throw IllegalArgumentException("Tag $key($title) is of unknown type")
|
||||
}
|
||||
|
||||
private fun tagsSetOf(tags: Collection<MangaTag>, genres: Collection<MangaTag>): Set<MangaTag> {
|
||||
val result = ArraySet<MangaTag>(tags.size + genres.size)
|
||||
val names = HashSet<String>(tags.size + genres.size)
|
||||
genres.forEach { x -> if (names.add(x.title)) result.add(x) }
|
||||
tags.forEach { x -> if (names.add(x.title)) result.add(x) }
|
||||
return result
|
||||
}
|
||||
|
||||
protected companion object {
|
||||
|
||||
const val SERVER_MAIN = "main"
|
||||
const val SERVER_SECONDARY = "secondary"
|
||||
const val SERVER_COMPRESS = "compress"
|
||||
const val SERVER_DOWNLOAD = "download"
|
||||
const val SERVER_CROP = "crop"
|
||||
}
|
||||
}
|
||||
@ -1,350 +1,15 @@
|
||||
package org.koitharu.kotatsu.parsers.site.ru.rulib
|
||||
|
||||
import androidx.collection.ArraySet
|
||||
import kotlinx.coroutines.withTimeoutOrNull
|
||||
import okhttp3.Response
|
||||
import okhttp3.internal.headersContentLength
|
||||
import org.json.JSONArray
|
||||
import org.json.JSONObject
|
||||
import org.jsoup.nodes.Document
|
||||
import org.koitharu.kotatsu.parsers.MangaLoaderContext
|
||||
import org.koitharu.kotatsu.parsers.MangaParserAuthProvider
|
||||
import org.koitharu.kotatsu.parsers.MangaSourceParser
|
||||
import org.koitharu.kotatsu.parsers.PagedMangaParser
|
||||
import org.koitharu.kotatsu.parsers.config.ConfigKey
|
||||
import org.koitharu.kotatsu.parsers.exception.AuthRequiredException
|
||||
import org.koitharu.kotatsu.parsers.exception.NotFoundException
|
||||
import org.koitharu.kotatsu.parsers.exception.ParseException
|
||||
import org.koitharu.kotatsu.parsers.model.*
|
||||
import org.koitharu.kotatsu.parsers.util.*
|
||||
import org.koitharu.kotatsu.parsers.util.json.*
|
||||
import java.text.SimpleDateFormat
|
||||
import java.util.*
|
||||
import org.koitharu.kotatsu.parsers.model.MangaSource
|
||||
|
||||
internal open class MangaLibParser(
|
||||
@MangaSourceParser("MANGALIB", "MangaLib", "ru")
|
||||
internal class MangaLibParser(
|
||||
context: MangaLoaderContext,
|
||||
source: MangaSource,
|
||||
) : PagedMangaParser(context, source, pageSize = 60), MangaParserAuthProvider {
|
||||
|
||||
override val configKeyDomain = ConfigKey.Domain("mangalib.me")
|
||||
|
||||
override val authUrl: String
|
||||
get() = "https://$domain/login"
|
||||
|
||||
override val availableSortOrders: Set<SortOrder> = EnumSet.of(
|
||||
SortOrder.RATING,
|
||||
SortOrder.ALPHABETICAL,
|
||||
SortOrder.POPULARITY,
|
||||
SortOrder.UPDATED,
|
||||
SortOrder.NEWEST,
|
||||
)
|
||||
|
||||
override suspend fun getListPage(
|
||||
page: Int,
|
||||
query: String?,
|
||||
tags: Set<MangaTag>?,
|
||||
tagsExclude: Set<MangaTag>?,
|
||||
sortOrder: SortOrder,
|
||||
): List<Manga> {
|
||||
if (!query.isNullOrEmpty()) {
|
||||
return if (page == searchPaginator.firstPage) search(query) else emptyList()
|
||||
}
|
||||
val url = buildString {
|
||||
append("https://")
|
||||
append(domain)
|
||||
append("/manga-list?dir=")
|
||||
append(getSortKey(sortOrder))
|
||||
append("&page=")
|
||||
append(page)
|
||||
tags?.forEach { tag ->
|
||||
append("&genres[include][]=")
|
||||
append(tag.key)
|
||||
}
|
||||
}
|
||||
val doc = webClient.httpGet(url).parseHtml()
|
||||
val root = doc.body().getElementById("manga-list") ?: doc.parseFailed("Root not found")
|
||||
val items = root.selectFirst("div.media-cards-grid")?.select("div.media-card-wrap") ?: return emptyList()
|
||||
return items.mapNotNull { card ->
|
||||
val a = card.selectFirst("a.media-card") ?: return@mapNotNull null
|
||||
val href = a.attrAsRelativeUrl("href")
|
||||
Manga(
|
||||
id = generateUid(href),
|
||||
title = card.selectFirst("h3")?.text().orEmpty(),
|
||||
coverUrl = a.absUrl("data-src"),
|
||||
altTitle = null,
|
||||
author = null,
|
||||
rating = RATING_UNKNOWN,
|
||||
url = href,
|
||||
publicUrl = href.toAbsoluteUrl(a.host ?: domain),
|
||||
tags = emptySet(),
|
||||
state = null,
|
||||
isNsfw = false,
|
||||
source = source,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
override suspend fun getDetails(manga: Manga): Manga {
|
||||
val fullUrl = manga.url.toAbsoluteUrl(domain)
|
||||
val doc = webClient.httpGet("$fullUrl?section=info").parseHtml()
|
||||
val root = doc.body().getElementById("main-page") ?: throw ParseException("Root not found", fullUrl)
|
||||
val title = root.selectFirst("div.media-header__wrap")?.children()
|
||||
val info = root.selectFirst("div.media-content")
|
||||
val chaptersDoc = webClient.httpGet("$fullUrl?section=chapters").parseHtml()
|
||||
val scripts = chaptersDoc.select("script")
|
||||
val dateFormat = SimpleDateFormat("yyy-MM-dd", Locale.US)
|
||||
var chapters: ChaptersListBuilder? = null
|
||||
scripts@ for (script in scripts) {
|
||||
val raw = script.html().lines()
|
||||
for (line in raw) {
|
||||
if (line.startsWith("window.__DATA__")) {
|
||||
val json = JSONObject(line.substringAfter('=').substringBeforeLast(';'))
|
||||
val list = json.getJSONObject("chapters").getJSONArray("list")
|
||||
val branches = json.getJSONObject("chapters").getJSONArray("branches").toJSONList()
|
||||
.associate { x ->
|
||||
x.getInt("id") to x.getJSONArray("teams").toJSONList().joinToString { it.getString("name") }
|
||||
}
|
||||
val id = json.optJSONObject("user")?.getLong("id")?.toString() ?: "not"
|
||||
val total = list.length()
|
||||
chapters = ChaptersListBuilder(total)
|
||||
for (i in (0 until total).reversed()) {
|
||||
val item = list.getJSONObject(i)
|
||||
val chapterId = item.getLong("chapter_id")
|
||||
val scanlator = item.getStringOrNull("username")
|
||||
val volume = item.getInt("chapter_volume")
|
||||
val number = item.getString("chapter_number")
|
||||
val url = buildString {
|
||||
append(manga.url)
|
||||
append("/v")
|
||||
append(volume)
|
||||
append("/c")
|
||||
append(number)
|
||||
if (isAuthorized) {
|
||||
append("?ui=")
|
||||
append(id)
|
||||
}
|
||||
}
|
||||
val nameChapter = item.getStringOrNull("chapter_name")
|
||||
val fullNameChapter = "Том $volume. Глава $number"
|
||||
val branchId = item.getIntOrDefault("branch_id", 0)
|
||||
chapters.add(
|
||||
MangaChapter(
|
||||
id = generateUid(chapterId),
|
||||
url = url,
|
||||
source = source,
|
||||
number = number.toFloatOrNull() ?: 0f,
|
||||
volume = volume,
|
||||
uploadDate = dateFormat.tryParse(
|
||||
item.getString("chapter_created_at").substringBefore(" "),
|
||||
),
|
||||
scanlator = scanlator,
|
||||
branch = branches[branchId],
|
||||
name = if (nameChapter.isNullOrBlank()) fullNameChapter else "$fullNameChapter - $nameChapter",
|
||||
),
|
||||
)
|
||||
}
|
||||
break@scripts
|
||||
}
|
||||
}
|
||||
}
|
||||
return manga.copy(
|
||||
title = title?.getOrNull(0)?.text()?.takeUnless(String::isBlank) ?: manga.title,
|
||||
altTitle = title?.getOrNull(1)?.text()?.substringBefore('/')?.trim(),
|
||||
rating = root.selectFirst("div.media-stats-item__score")?.selectFirst("span")?.text()?.toFloatOrNull()
|
||||
?.div(5f) ?: manga.rating,
|
||||
author = info?.getElementsMatchingOwnText("Автор")?.firstOrNull()?.nextElementSibling()?.text()
|
||||
?: manga.author,
|
||||
tags = info?.selectFirst("div.media-tags")?.select("a.media-tag-item")?.mapNotNullToSet { a ->
|
||||
val href = a.attr("href")
|
||||
if (href.contains("genres")) {
|
||||
MangaTag(
|
||||
title = a.text().toTitleCase(),
|
||||
key = href.substringAfterLast('='),
|
||||
source = source,
|
||||
)
|
||||
} else {
|
||||
null
|
||||
}
|
||||
} ?: manga.tags,
|
||||
isNsfw = isNsfw(doc),
|
||||
description = info?.selectFirst("div.media-description__text")?.html(),
|
||||
chapters = chapters?.toList(),
|
||||
)
|
||||
}
|
||||
|
||||
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
|
||||
val fullUrl = chapter.url.toAbsoluteUrl(domain)
|
||||
val doc = handle404 {
|
||||
webClient.httpGet(fullUrl).parseHtml()
|
||||
}
|
||||
if (doc.location().substringBefore('?').endsWith("/register")) {
|
||||
throw AuthRequiredException(source)
|
||||
}
|
||||
val scripts = doc.head().select("script")
|
||||
val pg =
|
||||
(doc.body().getElementById("pg")?.html() ?: doc.parseFailed("Element #pg not found")).substringAfter('=')
|
||||
.substringBeforeLast(';')
|
||||
val pages = JSONArray(pg)
|
||||
for (script in scripts) {
|
||||
val raw = script.html().trim()
|
||||
if (raw.contains("window.__info")) {
|
||||
val json = JSONObject(
|
||||
raw.substringAfter("window.__info").substringAfter('=').substringBeforeLast(';'),
|
||||
)
|
||||
val servers = json.getJSONObject("servers")
|
||||
val img = json.getJSONObject("img")
|
||||
val defaultServer = servers.getStringOrNull(img.getString("server"))
|
||||
val baseUrl = img.getString("url")
|
||||
val pageJson = JSONObject()
|
||||
pageJson.put("default", defaultServer)
|
||||
pageJson.put("servers", servers.jsonValues())
|
||||
return pages.mapJSON { x ->
|
||||
val pageUrl = concatUrl(baseUrl, x.getString("u"))
|
||||
pageJson.put("url", pageUrl)
|
||||
MangaPage(
|
||||
id = generateUid(pageUrl),
|
||||
url = pageJson.toString(),
|
||||
preview = null,
|
||||
source = source,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
throw ParseException("Script with info not found", fullUrl)
|
||||
}
|
||||
|
||||
override suspend fun getPageUrl(page: MangaPage): String {
|
||||
val json = JSONObject(page.url)
|
||||
val defaultServer = json.getString("default")
|
||||
val servers = json.getJSONArray("servers")
|
||||
val pageUrl = json.getString("url")
|
||||
return (0 until servers.length()).firstNotNullOfOrNull { i ->
|
||||
val server = servers.getString(i)
|
||||
concatUrl(server, pageUrl).takeIf {
|
||||
tryHeadImage(it)
|
||||
}
|
||||
} ?: concatUrl(defaultServer, pageUrl)
|
||||
}
|
||||
|
||||
override suspend fun getAvailableTags(): Set<MangaTag> {
|
||||
val url = "https://$domain/manga-list"
|
||||
val doc = webClient.httpGet(url).parseHtml()
|
||||
val scripts = doc.body().select("script")
|
||||
for (script in scripts) {
|
||||
val raw = script.html().trim()
|
||||
if (raw.startsWith("window.__DATA")) {
|
||||
val json = JSONObject(raw.substringAfter('=').substringBeforeLast(';'))
|
||||
val genres = json.getJSONObject("filters").getJSONArray("genres")
|
||||
val result = ArraySet<MangaTag>(genres.length())
|
||||
for (x in genres.JSONIterator()) {
|
||||
result += MangaTag(
|
||||
source = source,
|
||||
key = x.getInt("id").toString(),
|
||||
title = x.getString("name").toTitleCase(),
|
||||
)
|
||||
}
|
||||
return result
|
||||
}
|
||||
}
|
||||
throw ParseException("Script with genres not found", url)
|
||||
}
|
||||
|
||||
override val isAuthorized: Boolean
|
||||
get() {
|
||||
return context.cookieJar.getCookies(domain).any {
|
||||
it.name.startsWith("remember_web_")
|
||||
}
|
||||
}
|
||||
|
||||
override suspend fun getUsername(): String {
|
||||
val body = webClient.httpGet("https://$LIB_SOCIAL_LINK/messages").parseHtml().body()
|
||||
if (body.baseUri().endsWith("/login")) {
|
||||
throw AuthRequiredException(source)
|
||||
}
|
||||
return body.selectFirst(".profile-user__username")?.text() ?: body.parseFailed("Cannot find username")
|
||||
}
|
||||
|
||||
private suspend fun tryHeadImage(url: String): Boolean = runCatchingCancellable {
|
||||
withTimeoutOrNull(3_000) {
|
||||
webClient.httpHead(url).use { response ->
|
||||
response.isValidImage()
|
||||
}
|
||||
} ?: false
|
||||
}.getOrDefault(false)
|
||||
|
||||
protected open fun isNsfw(doc: Document): Boolean {
|
||||
val modal = doc.body().getElementById("title-caution")
|
||||
if (!modal?.getElementsContainingOwnText("18+").isNullOrEmpty()) {
|
||||
return true
|
||||
}
|
||||
val sidebar = doc.body().run {
|
||||
selectFirst(".media-sidebar") ?: selectFirst(".media-info")
|
||||
} ?: doc.parseFailed("Sidebar not found")
|
||||
return sidebar.getElementsContainingOwnText("18+").isNotEmpty()
|
||||
}
|
||||
|
||||
private fun getSortKey(sortOrder: SortOrder?) = when (sortOrder) {
|
||||
SortOrder.RATING -> "desc&sort=rate"
|
||||
SortOrder.ALPHABETICAL -> "asc&sort=name"
|
||||
SortOrder.POPULARITY -> "desc&sort=views"
|
||||
SortOrder.UPDATED -> "desc&sort=last_chapter_at"
|
||||
SortOrder.NEWEST -> "desc&sort=created_at"
|
||||
else -> "desc&sort=last_chapter_at"
|
||||
}
|
||||
|
||||
private suspend fun search(query: String): List<Manga> {
|
||||
val domain = domain
|
||||
val json = webClient.httpGet("https://$domain/search?type=manga&q=$query").parseJsonArray()
|
||||
return json.mapJSON { jo ->
|
||||
val slug = jo.getString("slug")
|
||||
val url = "/$slug"
|
||||
val cover = jo.getJSONObject("covers").getString("default").toAbsoluteUrl(domain)
|
||||
val title = jo.getString("rus_name").ifEmpty { jo.getString("name") }
|
||||
Manga(
|
||||
id = generateUid(url),
|
||||
url = url,
|
||||
publicUrl = "https://$domain/$slug",
|
||||
title = title,
|
||||
altTitle = jo.getString("name").takeUnless { it == title },
|
||||
author = null,
|
||||
tags = emptySet(),
|
||||
rating = jo.getString("rate_avg").toFloatOrNull()?.div(5f) ?: RATING_UNKNOWN,
|
||||
state = null,
|
||||
isNsfw = false,
|
||||
source = source,
|
||||
coverUrl = cover,
|
||||
largeCoverUrl = null,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
private inline fun <T> handle404(block: () -> T): T = try {
|
||||
block()
|
||||
} catch (e: NotFoundException) {
|
||||
if (isAuthorized) {
|
||||
throw e
|
||||
} else {
|
||||
throw AuthRequiredException(source)
|
||||
}
|
||||
}
|
||||
|
||||
private fun JSONObject.jsonValues(): JSONArray {
|
||||
val result = JSONArray()
|
||||
values().forEach { x ->
|
||||
result.put(x)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
private fun Response.isValidImage(): Boolean {
|
||||
return isSuccessful && mimeType?.startsWith("image/") == true && headersContentLength() >= 1024L
|
||||
}
|
||||
|
||||
@MangaSourceParser("MANGALIB", "MangaLib", "ru")
|
||||
class Impl(context: MangaLoaderContext) : MangaLibParser(context, MangaSource.MANGALIB)
|
||||
|
||||
companion object {
|
||||
|
||||
const val LIB_SOCIAL_LINK = "lib.social"
|
||||
}
|
||||
}
|
||||
) : LibSocialParser(
|
||||
context = context,
|
||||
source = MangaSource.MANGALIB,
|
||||
siteId = 1,
|
||||
siteDomain = "test-front.mangalib.me",
|
||||
)
|
||||
|
||||
@ -0,0 +1,13 @@
|
||||
package org.koitharu.kotatsu.parsers.site.ru.rulib
|
||||
|
||||
import org.koitharu.kotatsu.parsers.MangaLoaderContext
|
||||
import org.koitharu.kotatsu.parsers.MangaSourceParser
|
||||
import org.koitharu.kotatsu.parsers.model.MangaSource
|
||||
|
||||
@MangaSourceParser("YAOILIB", "SlashLib", "ru")
|
||||
internal class SlashLibParser(context: MangaLoaderContext) : LibSocialParser(
|
||||
context = context,
|
||||
source = MangaSource.YAOILIB,
|
||||
siteId = 2,
|
||||
siteDomain = "test-front.slashlib.me",
|
||||
)
|
||||
@ -1,13 +0,0 @@
|
||||
package org.koitharu.kotatsu.parsers.site.ru.rulib
|
||||
|
||||
import org.jsoup.nodes.Document
|
||||
import org.koitharu.kotatsu.parsers.MangaLoaderContext
|
||||
import org.koitharu.kotatsu.parsers.MangaSourceParser
|
||||
import org.koitharu.kotatsu.parsers.config.ConfigKey
|
||||
import org.koitharu.kotatsu.parsers.model.MangaSource
|
||||
|
||||
@MangaSourceParser("YAOILIB", "YaoiLib", "ru")
|
||||
internal class YaoiLibParser(context: MangaLoaderContext) : MangaLibParser(context, MangaSource.YAOILIB) {
|
||||
override val configKeyDomain = ConfigKey.Domain("yaoilib.me")
|
||||
override fun isNsfw(doc: Document) = true
|
||||
}
|
||||
Loading…
Reference in New Issue