[LibSocial] Rewrite parsers
parent
3e32a6280a
commit
952e9c39ac
@ -1,14 +1,14 @@
|
|||||||
package org.koitharu.kotatsu.parsers.site.ru.rulib
|
package org.koitharu.kotatsu.parsers.site.ru.rulib
|
||||||
|
|
||||||
import org.jsoup.nodes.Document
|
|
||||||
import org.koitharu.kotatsu.parsers.MangaLoaderContext
|
import org.koitharu.kotatsu.parsers.MangaLoaderContext
|
||||||
import org.koitharu.kotatsu.parsers.MangaSourceParser
|
import org.koitharu.kotatsu.parsers.MangaSourceParser
|
||||||
import org.koitharu.kotatsu.parsers.config.ConfigKey
|
|
||||||
import org.koitharu.kotatsu.parsers.model.ContentType
|
import org.koitharu.kotatsu.parsers.model.ContentType
|
||||||
import org.koitharu.kotatsu.parsers.model.MangaSource
|
import org.koitharu.kotatsu.parsers.model.MangaSource
|
||||||
|
|
||||||
@MangaSourceParser("HENTAILIB", "HentaiLib", "ru", type = ContentType.HENTAI)
|
@MangaSourceParser("HENTAILIB", "HentaiLib", "ru", type = ContentType.HENTAI)
|
||||||
internal class HentaiLibParser(context: MangaLoaderContext) : MangaLibParser(context, MangaSource.HENTAILIB) {
|
internal class HentaiLibParser(context: MangaLoaderContext) : LibSocialParser(
|
||||||
override val configKeyDomain = ConfigKey.Domain("hentailib.me")
|
context = context,
|
||||||
override fun isNsfw(doc: Document) = true
|
source = MangaSource.HENTAILIB,
|
||||||
}
|
siteId = 4,
|
||||||
|
siteDomain = "hentailib.me",
|
||||||
|
)
|
||||||
|
|||||||
@ -0,0 +1,307 @@
|
|||||||
|
package org.koitharu.kotatsu.parsers.site.ru.rulib
|
||||||
|
|
||||||
|
import androidx.collection.*
|
||||||
|
import kotlinx.coroutines.async
|
||||||
|
import kotlinx.coroutines.coroutineScope
|
||||||
|
import org.json.JSONArray
|
||||||
|
import org.json.JSONObject
|
||||||
|
import org.koitharu.kotatsu.parsers.MangaLoaderContext
|
||||||
|
import org.koitharu.kotatsu.parsers.PagedMangaParser
|
||||||
|
import org.koitharu.kotatsu.parsers.config.ConfigKey
|
||||||
|
import org.koitharu.kotatsu.parsers.model.*
|
||||||
|
import org.koitharu.kotatsu.parsers.util.*
|
||||||
|
import org.koitharu.kotatsu.parsers.util.json.*
|
||||||
|
import java.text.SimpleDateFormat
|
||||||
|
import java.util.*
|
||||||
|
|
||||||
|
internal abstract class LibSocialParser(
|
||||||
|
context: MangaLoaderContext,
|
||||||
|
source: MangaSource,
|
||||||
|
protected val siteDomain: String,
|
||||||
|
protected val siteId: Int,
|
||||||
|
) : PagedMangaParser(context, source, pageSize = 60) {
|
||||||
|
|
||||||
|
override val availableSortOrders: Set<SortOrder> = EnumSet.allOf(SortOrder::class.java)
|
||||||
|
|
||||||
|
final override val configKeyDomain = ConfigKey.Domain("lib.social")
|
||||||
|
override val availableStates: Set<MangaState> = EnumSet.allOf(MangaState::class.java)
|
||||||
|
override val isMultipleTagsSupported = true
|
||||||
|
override val isTagsExclusionSupported = true
|
||||||
|
override val isSearchSupported = true
|
||||||
|
|
||||||
|
private val statesMap = intObjectMapOf(
|
||||||
|
1, MangaState.ONGOING,
|
||||||
|
2, MangaState.FINISHED,
|
||||||
|
3, MangaState.UPCOMING,
|
||||||
|
4, MangaState.PAUSED,
|
||||||
|
5, MangaState.ABANDONED,
|
||||||
|
)
|
||||||
|
private val imageServers = SuspendLazy(::fetchServers)
|
||||||
|
|
||||||
|
override suspend fun getListPage(page: Int, filter: MangaListFilter?): List<Manga> {
|
||||||
|
val urlBuilder = urlBuilder("api")
|
||||||
|
.addPathSegment("api")
|
||||||
|
.addPathSegment("manga")
|
||||||
|
.addQueryParameter("site_id[]", siteId.toString())
|
||||||
|
.addQueryParameter("fields[]", "rate")
|
||||||
|
.addQueryParameter("fields[]", "rate_avg")
|
||||||
|
.addQueryParameter("page", page.toString())
|
||||||
|
when (filter) {
|
||||||
|
is MangaListFilter.Advanced -> {
|
||||||
|
for (state in filter.states) {
|
||||||
|
urlBuilder.addQueryParameter("status[]", statesMap.keyOf(state).toString())
|
||||||
|
}
|
||||||
|
for (tag in filter.tags) {
|
||||||
|
urlBuilder.addQueryParameter("${tag.typeKey()}[]", tag.key.drop(1))
|
||||||
|
}
|
||||||
|
for (tag in filter.tagsExclude) {
|
||||||
|
urlBuilder.addQueryParameter("${tag.typeKey()}_exclude[]", tag.key.drop(1))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
is MangaListFilter.Search -> {
|
||||||
|
urlBuilder.addQueryParameter("q", filter.query)
|
||||||
|
}
|
||||||
|
|
||||||
|
null -> Unit
|
||||||
|
}
|
||||||
|
val sortOrder = filter?.sortOrder ?: defaultSortOrder
|
||||||
|
urlBuilder.addQueryParameter(
|
||||||
|
"sort_by",
|
||||||
|
when (sortOrder) {
|
||||||
|
SortOrder.UPDATED -> "last_chapter_at"
|
||||||
|
SortOrder.POPULARITY -> "views"
|
||||||
|
SortOrder.RATING -> "rate_avg"
|
||||||
|
SortOrder.NEWEST -> "created_at"
|
||||||
|
SortOrder.ALPHABETICAL,
|
||||||
|
SortOrder.ALPHABETICAL_DESC,
|
||||||
|
-> "rus_name"
|
||||||
|
},
|
||||||
|
)
|
||||||
|
urlBuilder.addQueryParameter(
|
||||||
|
"sort_type",
|
||||||
|
when (sortOrder) {
|
||||||
|
SortOrder.UPDATED,
|
||||||
|
SortOrder.POPULARITY,
|
||||||
|
SortOrder.RATING,
|
||||||
|
SortOrder.NEWEST,
|
||||||
|
SortOrder.ALPHABETICAL_DESC,
|
||||||
|
-> "desc"
|
||||||
|
|
||||||
|
SortOrder.ALPHABETICAL -> "asc"
|
||||||
|
},
|
||||||
|
)
|
||||||
|
val json = webClient.httpGet(urlBuilder.build()).parseJson()
|
||||||
|
val data = json.getJSONArray("data")
|
||||||
|
return data.mapJSON(::parseManga)
|
||||||
|
}
|
||||||
|
|
||||||
|
override suspend fun getDetails(manga: Manga): Manga = coroutineScope {
|
||||||
|
val chapters = async { fetchChapters(manga) }
|
||||||
|
val url = urlBuilder("api")
|
||||||
|
.addPathSegment("api")
|
||||||
|
.addPathSegment("manga")
|
||||||
|
.addPathSegment(manga.url)
|
||||||
|
.addQueryParameter("fields[]", "summary")
|
||||||
|
.addQueryParameter("fields[]", "genres")
|
||||||
|
.addQueryParameter("fields[]", "tags")
|
||||||
|
.addQueryParameter("fields[]", "authors")
|
||||||
|
.build()
|
||||||
|
val json = webClient.httpGet(url).parseJson().getJSONObject("data")
|
||||||
|
val genres = json.getJSONArray("genres").mapJSON { jo ->
|
||||||
|
MangaTag(title = jo.getString("name"), key = "g" + jo.getInt("id"), source = source)
|
||||||
|
}
|
||||||
|
val tags = json.getJSONArray("genres").mapJSON { jo ->
|
||||||
|
MangaTag(title = jo.getString("name"), key = "t" + jo.getInt("id"), source = source)
|
||||||
|
}
|
||||||
|
manga.copy(
|
||||||
|
title = json.getStringOrNull("rus_name") ?: manga.title,
|
||||||
|
altTitle = json.getString("name"),
|
||||||
|
tags = tagsSetOf(tags, genres),
|
||||||
|
author = json.getJSONArray("authors").optJSONObject(0)?.getStringOrNull("name"),
|
||||||
|
description = json.getString("summary").nl2br(),
|
||||||
|
chapters = chapters.await(),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> = coroutineScope {
|
||||||
|
val pages = async {
|
||||||
|
webClient.httpGet(
|
||||||
|
concatUrl("https://api.$domain/api/manga/", chapter.url),
|
||||||
|
).parseJson().getJSONObject("data")
|
||||||
|
}
|
||||||
|
val servers = imageServers.get()
|
||||||
|
val json = pages.await()
|
||||||
|
val primaryServer =
|
||||||
|
checkNotNull(servers[SERVER_MAIN] ?: servers[SERVER_DOWNLOAD] ?: servers[SERVER_SECONDARY]) {
|
||||||
|
"No available images servers"
|
||||||
|
}
|
||||||
|
json.getJSONArray("pages").mapJSON { jo ->
|
||||||
|
val url = jo.getString("url")
|
||||||
|
MangaPage(
|
||||||
|
id = generateUid(jo.getLong("id")),
|
||||||
|
url = concatUrl(primaryServer, url),
|
||||||
|
preview = servers[SERVER_COMPRESS]?.let { concatUrl(it, url) },
|
||||||
|
source = source,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
override suspend fun getAvailableTags(): Set<MangaTag> = coroutineScope {
|
||||||
|
val tags = async { fetchTags("tags") }
|
||||||
|
val genres = async { fetchTags("genres") }
|
||||||
|
tagsSetOf(tags.await(), genres.await())
|
||||||
|
}
|
||||||
|
|
||||||
|
override suspend fun getRelatedManga(seed: Manga): List<Manga> {
|
||||||
|
val json = webClient.httpGet(
|
||||||
|
urlBuilder("api")
|
||||||
|
.addPathSegment("api")
|
||||||
|
.addPathSegment("manga")
|
||||||
|
.addPathSegment(seed.url)
|
||||||
|
.addPathSegment("similar")
|
||||||
|
.build(),
|
||||||
|
).parseJson().getJSONArray("data")
|
||||||
|
return json.mapJSON { jo ->
|
||||||
|
parseManga(jo.getJSONObject("media"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun parseManga(jo: JSONObject): Manga {
|
||||||
|
val cover = jo.getJSONObject("cover")
|
||||||
|
return Manga(
|
||||||
|
id = generateUid(jo.getLong("id")),
|
||||||
|
title = jo.getString("rus_name").ifEmpty { jo.getString("name") },
|
||||||
|
altTitle = jo.getString("name"),
|
||||||
|
url = jo.getString("slug_url"),
|
||||||
|
publicUrl = "https://$siteDomain/ru/manga/" + jo.getString("slug_url"),
|
||||||
|
rating = jo.optJSONObject("rating")
|
||||||
|
?.getFloatOrDefault("average", RATING_UNKNOWN * 10f)?.div(10f) ?: RATING_UNKNOWN,
|
||||||
|
isNsfw = jo.getJSONObject("ageRestriction").getIntOrDefault("id", 0) >= 3,
|
||||||
|
coverUrl = cover.getString("thumbnail"),
|
||||||
|
tags = setOf(),
|
||||||
|
state = statesMap[jo.optJSONObject("status")?.getIntOrDefault("id", -1) ?: -1],
|
||||||
|
author = null,
|
||||||
|
largeCoverUrl = cover.getString("default"),
|
||||||
|
source = source,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
private suspend fun fetchChapters(manga: Manga): List<MangaChapter> {
|
||||||
|
val url = urlBuilder("api")
|
||||||
|
.addPathSegment("api")
|
||||||
|
.addPathSegment("manga")
|
||||||
|
.addPathSegment(manga.url)
|
||||||
|
.addPathSegment("chapters")
|
||||||
|
.build()
|
||||||
|
val json = webClient.httpGet(url).parseJson().getJSONArray("data")
|
||||||
|
val builder = ChaptersListBuilder(json.length())
|
||||||
|
val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.US)
|
||||||
|
for (i in 0 until json.length()) {
|
||||||
|
val jo = json.getJSONObject(i)
|
||||||
|
val volume = jo.getIntOrDefault("volume", 0)
|
||||||
|
val number = jo.getFloatOrDefault("number", 0f)
|
||||||
|
val numberString = number.formatSimple()
|
||||||
|
val name = jo.getStringOrNull("name") ?: buildString {
|
||||||
|
if (volume > 0) append("Том ").append(volume).append(' ')
|
||||||
|
append("Глава ").append(numberString)
|
||||||
|
}
|
||||||
|
val branches = jo.getJSONArray("branches")
|
||||||
|
for (j in 0 until branches.length()) {
|
||||||
|
val bjo = branches.getJSONObject(j)
|
||||||
|
val id = bjo.getLong("id")
|
||||||
|
val team = bjo.getJSONArray("teams").optJSONObject(0)?.getStringOrNull("name")
|
||||||
|
builder += MangaChapter(
|
||||||
|
id = generateUid(id),
|
||||||
|
name = name,
|
||||||
|
number = number,
|
||||||
|
volume = volume,
|
||||||
|
url = "${manga.url}/chapter?number=$numberString&volume=$volume",
|
||||||
|
scanlator = team,
|
||||||
|
uploadDate = dateFormat.tryParse(bjo.getStringOrNull("created_at")),
|
||||||
|
branch = team,
|
||||||
|
source = source,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return builder.toList()
|
||||||
|
}
|
||||||
|
|
||||||
|
private suspend fun fetchTags(type: String): List<MangaTag> {
|
||||||
|
val data = webClient.httpGet(
|
||||||
|
urlBuilder("api").addPathSegment("api").addPathSegment(type).build(),
|
||||||
|
).parseJson().getJSONArray("data")
|
||||||
|
val prefix = type.first().toString()
|
||||||
|
return data.mapJSONNotNull { jo ->
|
||||||
|
val sites = jo.getJSONArray("site_ids").toIntSet()
|
||||||
|
if (siteId !in sites) {
|
||||||
|
return@mapJSONNotNull null
|
||||||
|
}
|
||||||
|
MangaTag(
|
||||||
|
title = jo.getString("name"),
|
||||||
|
key = prefix + jo.getInt("id"),
|
||||||
|
source = source,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private suspend fun fetchServers(): ScatterMap<String, String> {
|
||||||
|
val json = webClient.httpGet(
|
||||||
|
urlBuilder("api")
|
||||||
|
.addPathSegment("api")
|
||||||
|
.addPathSegment("constants")
|
||||||
|
.addQueryParameter("fields[]", "imageServers")
|
||||||
|
.build(),
|
||||||
|
).parseJson().getJSONObject("data").getJSONArray("imageServers")
|
||||||
|
val result = MutableScatterMap<String, String>()
|
||||||
|
for (i in 0 until json.length()) {
|
||||||
|
val jo = json.getJSONObject(i)
|
||||||
|
val sites = jo.getJSONArray("site_ids").toIntSet()
|
||||||
|
if (siteId !in sites) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
result[jo.getString("id")] = jo.getString("url")
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun <V> IntObjectMap<V>.keyOf(value: V): Int {
|
||||||
|
forEach { k, v ->
|
||||||
|
if (v == value) {
|
||||||
|
return k
|
||||||
|
}
|
||||||
|
}
|
||||||
|
throw NoSuchElementException("No key associated with value $value")
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun JSONArray.toIntSet(): IntSet {
|
||||||
|
val result = MutableIntSet(length())
|
||||||
|
for (i in 0 until length()) {
|
||||||
|
result.add(getInt(i))
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun MangaTag.typeKey() = when (key.firstOrNull()) {
|
||||||
|
'g' -> "genres"
|
||||||
|
't' -> "tags"
|
||||||
|
else -> throw IllegalArgumentException("Tag $key($title) is of unknown type")
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun tagsSetOf(tags: Collection<MangaTag>, genres: Collection<MangaTag>): Set<MangaTag> {
|
||||||
|
val result = ArraySet<MangaTag>(tags.size + genres.size)
|
||||||
|
val names = HashSet<String>(tags.size + genres.size)
|
||||||
|
genres.forEach { x -> if (names.add(x.title)) result.add(x) }
|
||||||
|
tags.forEach { x -> if (names.add(x.title)) result.add(x) }
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
protected companion object {
|
||||||
|
|
||||||
|
const val SERVER_MAIN = "main"
|
||||||
|
const val SERVER_SECONDARY = "secondary"
|
||||||
|
const val SERVER_COMPRESS = "compress"
|
||||||
|
const val SERVER_DOWNLOAD = "download"
|
||||||
|
const val SERVER_CROP = "crop"
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -1,350 +1,15 @@
|
|||||||
package org.koitharu.kotatsu.parsers.site.ru.rulib
|
package org.koitharu.kotatsu.parsers.site.ru.rulib
|
||||||
|
|
||||||
import androidx.collection.ArraySet
|
|
||||||
import kotlinx.coroutines.withTimeoutOrNull
|
|
||||||
import okhttp3.Response
|
|
||||||
import okhttp3.internal.headersContentLength
|
|
||||||
import org.json.JSONArray
|
|
||||||
import org.json.JSONObject
|
|
||||||
import org.jsoup.nodes.Document
|
|
||||||
import org.koitharu.kotatsu.parsers.MangaLoaderContext
|
import org.koitharu.kotatsu.parsers.MangaLoaderContext
|
||||||
import org.koitharu.kotatsu.parsers.MangaParserAuthProvider
|
|
||||||
import org.koitharu.kotatsu.parsers.MangaSourceParser
|
import org.koitharu.kotatsu.parsers.MangaSourceParser
|
||||||
import org.koitharu.kotatsu.parsers.PagedMangaParser
|
import org.koitharu.kotatsu.parsers.model.MangaSource
|
||||||
import org.koitharu.kotatsu.parsers.config.ConfigKey
|
|
||||||
import org.koitharu.kotatsu.parsers.exception.AuthRequiredException
|
|
||||||
import org.koitharu.kotatsu.parsers.exception.NotFoundException
|
|
||||||
import org.koitharu.kotatsu.parsers.exception.ParseException
|
|
||||||
import org.koitharu.kotatsu.parsers.model.*
|
|
||||||
import org.koitharu.kotatsu.parsers.util.*
|
|
||||||
import org.koitharu.kotatsu.parsers.util.json.*
|
|
||||||
import java.text.SimpleDateFormat
|
|
||||||
import java.util.*
|
|
||||||
|
|
||||||
internal open class MangaLibParser(
|
@MangaSourceParser("MANGALIB", "MangaLib", "ru")
|
||||||
|
internal class MangaLibParser(
|
||||||
context: MangaLoaderContext,
|
context: MangaLoaderContext,
|
||||||
source: MangaSource,
|
) : LibSocialParser(
|
||||||
) : PagedMangaParser(context, source, pageSize = 60), MangaParserAuthProvider {
|
context = context,
|
||||||
|
source = MangaSource.MANGALIB,
|
||||||
override val configKeyDomain = ConfigKey.Domain("mangalib.me")
|
siteId = 1,
|
||||||
|
siteDomain = "test-front.mangalib.me",
|
||||||
override val authUrl: String
|
)
|
||||||
get() = "https://$domain/login"
|
|
||||||
|
|
||||||
override val availableSortOrders: Set<SortOrder> = EnumSet.of(
|
|
||||||
SortOrder.RATING,
|
|
||||||
SortOrder.ALPHABETICAL,
|
|
||||||
SortOrder.POPULARITY,
|
|
||||||
SortOrder.UPDATED,
|
|
||||||
SortOrder.NEWEST,
|
|
||||||
)
|
|
||||||
|
|
||||||
override suspend fun getListPage(
|
|
||||||
page: Int,
|
|
||||||
query: String?,
|
|
||||||
tags: Set<MangaTag>?,
|
|
||||||
tagsExclude: Set<MangaTag>?,
|
|
||||||
sortOrder: SortOrder,
|
|
||||||
): List<Manga> {
|
|
||||||
if (!query.isNullOrEmpty()) {
|
|
||||||
return if (page == searchPaginator.firstPage) search(query) else emptyList()
|
|
||||||
}
|
|
||||||
val url = buildString {
|
|
||||||
append("https://")
|
|
||||||
append(domain)
|
|
||||||
append("/manga-list?dir=")
|
|
||||||
append(getSortKey(sortOrder))
|
|
||||||
append("&page=")
|
|
||||||
append(page)
|
|
||||||
tags?.forEach { tag ->
|
|
||||||
append("&genres[include][]=")
|
|
||||||
append(tag.key)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
val doc = webClient.httpGet(url).parseHtml()
|
|
||||||
val root = doc.body().getElementById("manga-list") ?: doc.parseFailed("Root not found")
|
|
||||||
val items = root.selectFirst("div.media-cards-grid")?.select("div.media-card-wrap") ?: return emptyList()
|
|
||||||
return items.mapNotNull { card ->
|
|
||||||
val a = card.selectFirst("a.media-card") ?: return@mapNotNull null
|
|
||||||
val href = a.attrAsRelativeUrl("href")
|
|
||||||
Manga(
|
|
||||||
id = generateUid(href),
|
|
||||||
title = card.selectFirst("h3")?.text().orEmpty(),
|
|
||||||
coverUrl = a.absUrl("data-src"),
|
|
||||||
altTitle = null,
|
|
||||||
author = null,
|
|
||||||
rating = RATING_UNKNOWN,
|
|
||||||
url = href,
|
|
||||||
publicUrl = href.toAbsoluteUrl(a.host ?: domain),
|
|
||||||
tags = emptySet(),
|
|
||||||
state = null,
|
|
||||||
isNsfw = false,
|
|
||||||
source = source,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
override suspend fun getDetails(manga: Manga): Manga {
|
|
||||||
val fullUrl = manga.url.toAbsoluteUrl(domain)
|
|
||||||
val doc = webClient.httpGet("$fullUrl?section=info").parseHtml()
|
|
||||||
val root = doc.body().getElementById("main-page") ?: throw ParseException("Root not found", fullUrl)
|
|
||||||
val title = root.selectFirst("div.media-header__wrap")?.children()
|
|
||||||
val info = root.selectFirst("div.media-content")
|
|
||||||
val chaptersDoc = webClient.httpGet("$fullUrl?section=chapters").parseHtml()
|
|
||||||
val scripts = chaptersDoc.select("script")
|
|
||||||
val dateFormat = SimpleDateFormat("yyy-MM-dd", Locale.US)
|
|
||||||
var chapters: ChaptersListBuilder? = null
|
|
||||||
scripts@ for (script in scripts) {
|
|
||||||
val raw = script.html().lines()
|
|
||||||
for (line in raw) {
|
|
||||||
if (line.startsWith("window.__DATA__")) {
|
|
||||||
val json = JSONObject(line.substringAfter('=').substringBeforeLast(';'))
|
|
||||||
val list = json.getJSONObject("chapters").getJSONArray("list")
|
|
||||||
val branches = json.getJSONObject("chapters").getJSONArray("branches").toJSONList()
|
|
||||||
.associate { x ->
|
|
||||||
x.getInt("id") to x.getJSONArray("teams").toJSONList().joinToString { it.getString("name") }
|
|
||||||
}
|
|
||||||
val id = json.optJSONObject("user")?.getLong("id")?.toString() ?: "not"
|
|
||||||
val total = list.length()
|
|
||||||
chapters = ChaptersListBuilder(total)
|
|
||||||
for (i in (0 until total).reversed()) {
|
|
||||||
val item = list.getJSONObject(i)
|
|
||||||
val chapterId = item.getLong("chapter_id")
|
|
||||||
val scanlator = item.getStringOrNull("username")
|
|
||||||
val volume = item.getInt("chapter_volume")
|
|
||||||
val number = item.getString("chapter_number")
|
|
||||||
val url = buildString {
|
|
||||||
append(manga.url)
|
|
||||||
append("/v")
|
|
||||||
append(volume)
|
|
||||||
append("/c")
|
|
||||||
append(number)
|
|
||||||
if (isAuthorized) {
|
|
||||||
append("?ui=")
|
|
||||||
append(id)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
val nameChapter = item.getStringOrNull("chapter_name")
|
|
||||||
val fullNameChapter = "Том $volume. Глава $number"
|
|
||||||
val branchId = item.getIntOrDefault("branch_id", 0)
|
|
||||||
chapters.add(
|
|
||||||
MangaChapter(
|
|
||||||
id = generateUid(chapterId),
|
|
||||||
url = url,
|
|
||||||
source = source,
|
|
||||||
number = number.toFloatOrNull() ?: 0f,
|
|
||||||
volume = volume,
|
|
||||||
uploadDate = dateFormat.tryParse(
|
|
||||||
item.getString("chapter_created_at").substringBefore(" "),
|
|
||||||
),
|
|
||||||
scanlator = scanlator,
|
|
||||||
branch = branches[branchId],
|
|
||||||
name = if (nameChapter.isNullOrBlank()) fullNameChapter else "$fullNameChapter - $nameChapter",
|
|
||||||
),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
break@scripts
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return manga.copy(
|
|
||||||
title = title?.getOrNull(0)?.text()?.takeUnless(String::isBlank) ?: manga.title,
|
|
||||||
altTitle = title?.getOrNull(1)?.text()?.substringBefore('/')?.trim(),
|
|
||||||
rating = root.selectFirst("div.media-stats-item__score")?.selectFirst("span")?.text()?.toFloatOrNull()
|
|
||||||
?.div(5f) ?: manga.rating,
|
|
||||||
author = info?.getElementsMatchingOwnText("Автор")?.firstOrNull()?.nextElementSibling()?.text()
|
|
||||||
?: manga.author,
|
|
||||||
tags = info?.selectFirst("div.media-tags")?.select("a.media-tag-item")?.mapNotNullToSet { a ->
|
|
||||||
val href = a.attr("href")
|
|
||||||
if (href.contains("genres")) {
|
|
||||||
MangaTag(
|
|
||||||
title = a.text().toTitleCase(),
|
|
||||||
key = href.substringAfterLast('='),
|
|
||||||
source = source,
|
|
||||||
)
|
|
||||||
} else {
|
|
||||||
null
|
|
||||||
}
|
|
||||||
} ?: manga.tags,
|
|
||||||
isNsfw = isNsfw(doc),
|
|
||||||
description = info?.selectFirst("div.media-description__text")?.html(),
|
|
||||||
chapters = chapters?.toList(),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
|
|
||||||
val fullUrl = chapter.url.toAbsoluteUrl(domain)
|
|
||||||
val doc = handle404 {
|
|
||||||
webClient.httpGet(fullUrl).parseHtml()
|
|
||||||
}
|
|
||||||
if (doc.location().substringBefore('?').endsWith("/register")) {
|
|
||||||
throw AuthRequiredException(source)
|
|
||||||
}
|
|
||||||
val scripts = doc.head().select("script")
|
|
||||||
val pg =
|
|
||||||
(doc.body().getElementById("pg")?.html() ?: doc.parseFailed("Element #pg not found")).substringAfter('=')
|
|
||||||
.substringBeforeLast(';')
|
|
||||||
val pages = JSONArray(pg)
|
|
||||||
for (script in scripts) {
|
|
||||||
val raw = script.html().trim()
|
|
||||||
if (raw.contains("window.__info")) {
|
|
||||||
val json = JSONObject(
|
|
||||||
raw.substringAfter("window.__info").substringAfter('=').substringBeforeLast(';'),
|
|
||||||
)
|
|
||||||
val servers = json.getJSONObject("servers")
|
|
||||||
val img = json.getJSONObject("img")
|
|
||||||
val defaultServer = servers.getStringOrNull(img.getString("server"))
|
|
||||||
val baseUrl = img.getString("url")
|
|
||||||
val pageJson = JSONObject()
|
|
||||||
pageJson.put("default", defaultServer)
|
|
||||||
pageJson.put("servers", servers.jsonValues())
|
|
||||||
return pages.mapJSON { x ->
|
|
||||||
val pageUrl = concatUrl(baseUrl, x.getString("u"))
|
|
||||||
pageJson.put("url", pageUrl)
|
|
||||||
MangaPage(
|
|
||||||
id = generateUid(pageUrl),
|
|
||||||
url = pageJson.toString(),
|
|
||||||
preview = null,
|
|
||||||
source = source,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
throw ParseException("Script with info not found", fullUrl)
|
|
||||||
}
|
|
||||||
|
|
||||||
override suspend fun getPageUrl(page: MangaPage): String {
|
|
||||||
val json = JSONObject(page.url)
|
|
||||||
val defaultServer = json.getString("default")
|
|
||||||
val servers = json.getJSONArray("servers")
|
|
||||||
val pageUrl = json.getString("url")
|
|
||||||
return (0 until servers.length()).firstNotNullOfOrNull { i ->
|
|
||||||
val server = servers.getString(i)
|
|
||||||
concatUrl(server, pageUrl).takeIf {
|
|
||||||
tryHeadImage(it)
|
|
||||||
}
|
|
||||||
} ?: concatUrl(defaultServer, pageUrl)
|
|
||||||
}
|
|
||||||
|
|
||||||
override suspend fun getAvailableTags(): Set<MangaTag> {
|
|
||||||
val url = "https://$domain/manga-list"
|
|
||||||
val doc = webClient.httpGet(url).parseHtml()
|
|
||||||
val scripts = doc.body().select("script")
|
|
||||||
for (script in scripts) {
|
|
||||||
val raw = script.html().trim()
|
|
||||||
if (raw.startsWith("window.__DATA")) {
|
|
||||||
val json = JSONObject(raw.substringAfter('=').substringBeforeLast(';'))
|
|
||||||
val genres = json.getJSONObject("filters").getJSONArray("genres")
|
|
||||||
val result = ArraySet<MangaTag>(genres.length())
|
|
||||||
for (x in genres.JSONIterator()) {
|
|
||||||
result += MangaTag(
|
|
||||||
source = source,
|
|
||||||
key = x.getInt("id").toString(),
|
|
||||||
title = x.getString("name").toTitleCase(),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
return result
|
|
||||||
}
|
|
||||||
}
|
|
||||||
throw ParseException("Script with genres not found", url)
|
|
||||||
}
|
|
||||||
|
|
||||||
override val isAuthorized: Boolean
|
|
||||||
get() {
|
|
||||||
return context.cookieJar.getCookies(domain).any {
|
|
||||||
it.name.startsWith("remember_web_")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
override suspend fun getUsername(): String {
|
|
||||||
val body = webClient.httpGet("https://$LIB_SOCIAL_LINK/messages").parseHtml().body()
|
|
||||||
if (body.baseUri().endsWith("/login")) {
|
|
||||||
throw AuthRequiredException(source)
|
|
||||||
}
|
|
||||||
return body.selectFirst(".profile-user__username")?.text() ?: body.parseFailed("Cannot find username")
|
|
||||||
}
|
|
||||||
|
|
||||||
private suspend fun tryHeadImage(url: String): Boolean = runCatchingCancellable {
|
|
||||||
withTimeoutOrNull(3_000) {
|
|
||||||
webClient.httpHead(url).use { response ->
|
|
||||||
response.isValidImage()
|
|
||||||
}
|
|
||||||
} ?: false
|
|
||||||
}.getOrDefault(false)
|
|
||||||
|
|
||||||
protected open fun isNsfw(doc: Document): Boolean {
|
|
||||||
val modal = doc.body().getElementById("title-caution")
|
|
||||||
if (!modal?.getElementsContainingOwnText("18+").isNullOrEmpty()) {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
val sidebar = doc.body().run {
|
|
||||||
selectFirst(".media-sidebar") ?: selectFirst(".media-info")
|
|
||||||
} ?: doc.parseFailed("Sidebar not found")
|
|
||||||
return sidebar.getElementsContainingOwnText("18+").isNotEmpty()
|
|
||||||
}
|
|
||||||
|
|
||||||
private fun getSortKey(sortOrder: SortOrder?) = when (sortOrder) {
|
|
||||||
SortOrder.RATING -> "desc&sort=rate"
|
|
||||||
SortOrder.ALPHABETICAL -> "asc&sort=name"
|
|
||||||
SortOrder.POPULARITY -> "desc&sort=views"
|
|
||||||
SortOrder.UPDATED -> "desc&sort=last_chapter_at"
|
|
||||||
SortOrder.NEWEST -> "desc&sort=created_at"
|
|
||||||
else -> "desc&sort=last_chapter_at"
|
|
||||||
}
|
|
||||||
|
|
||||||
private suspend fun search(query: String): List<Manga> {
|
|
||||||
val domain = domain
|
|
||||||
val json = webClient.httpGet("https://$domain/search?type=manga&q=$query").parseJsonArray()
|
|
||||||
return json.mapJSON { jo ->
|
|
||||||
val slug = jo.getString("slug")
|
|
||||||
val url = "/$slug"
|
|
||||||
val cover = jo.getJSONObject("covers").getString("default").toAbsoluteUrl(domain)
|
|
||||||
val title = jo.getString("rus_name").ifEmpty { jo.getString("name") }
|
|
||||||
Manga(
|
|
||||||
id = generateUid(url),
|
|
||||||
url = url,
|
|
||||||
publicUrl = "https://$domain/$slug",
|
|
||||||
title = title,
|
|
||||||
altTitle = jo.getString("name").takeUnless { it == title },
|
|
||||||
author = null,
|
|
||||||
tags = emptySet(),
|
|
||||||
rating = jo.getString("rate_avg").toFloatOrNull()?.div(5f) ?: RATING_UNKNOWN,
|
|
||||||
state = null,
|
|
||||||
isNsfw = false,
|
|
||||||
source = source,
|
|
||||||
coverUrl = cover,
|
|
||||||
largeCoverUrl = null,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private inline fun <T> handle404(block: () -> T): T = try {
|
|
||||||
block()
|
|
||||||
} catch (e: NotFoundException) {
|
|
||||||
if (isAuthorized) {
|
|
||||||
throw e
|
|
||||||
} else {
|
|
||||||
throw AuthRequiredException(source)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private fun JSONObject.jsonValues(): JSONArray {
|
|
||||||
val result = JSONArray()
|
|
||||||
values().forEach { x ->
|
|
||||||
result.put(x)
|
|
||||||
}
|
|
||||||
return result
|
|
||||||
}
|
|
||||||
|
|
||||||
private fun Response.isValidImage(): Boolean {
|
|
||||||
return isSuccessful && mimeType?.startsWith("image/") == true && headersContentLength() >= 1024L
|
|
||||||
}
|
|
||||||
|
|
||||||
@MangaSourceParser("MANGALIB", "MangaLib", "ru")
|
|
||||||
class Impl(context: MangaLoaderContext) : MangaLibParser(context, MangaSource.MANGALIB)
|
|
||||||
|
|
||||||
companion object {
|
|
||||||
|
|
||||||
const val LIB_SOCIAL_LINK = "lib.social"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@ -0,0 +1,13 @@
|
|||||||
|
package org.koitharu.kotatsu.parsers.site.ru.rulib
|
||||||
|
|
||||||
|
import org.koitharu.kotatsu.parsers.MangaLoaderContext
|
||||||
|
import org.koitharu.kotatsu.parsers.MangaSourceParser
|
||||||
|
import org.koitharu.kotatsu.parsers.model.MangaSource
|
||||||
|
|
||||||
|
@MangaSourceParser("YAOILIB", "SlashLib", "ru")
|
||||||
|
internal class SlashLibParser(context: MangaLoaderContext) : LibSocialParser(
|
||||||
|
context = context,
|
||||||
|
source = MangaSource.YAOILIB,
|
||||||
|
siteId = 2,
|
||||||
|
siteDomain = "test-front.slashlib.me",
|
||||||
|
)
|
||||||
@ -1,13 +0,0 @@
|
|||||||
package org.koitharu.kotatsu.parsers.site.ru.rulib
|
|
||||||
|
|
||||||
import org.jsoup.nodes.Document
|
|
||||||
import org.koitharu.kotatsu.parsers.MangaLoaderContext
|
|
||||||
import org.koitharu.kotatsu.parsers.MangaSourceParser
|
|
||||||
import org.koitharu.kotatsu.parsers.config.ConfigKey
|
|
||||||
import org.koitharu.kotatsu.parsers.model.MangaSource
|
|
||||||
|
|
||||||
@MangaSourceParser("YAOILIB", "YaoiLib", "ru")
|
|
||||||
internal class YaoiLibParser(context: MangaLoaderContext) : MangaLibParser(context, MangaSource.YAOILIB) {
|
|
||||||
override val configKeyDomain = ConfigKey.Domain("yaoilib.me")
|
|
||||||
override fun isNsfw(doc: Document) = true
|
|
||||||
}
|
|
||||||
Loading…
Reference in New Issue