KotatsuApp/kotatsu-parsers
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[Nicovideo] Refactor

Browse Source
This commit is contained in:
Koitharu
2022-06-15 12:12:45 +03:00
parent 282ca40400
commit e88fa57f17
7 changed files with 40 additions and 146 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
src/main/kotlin/org/koitharu/kotatsu/parsers/MangaParser.kt
View File

@@ -114,6 +114,11 @@ abstract class MangaParser @InternalParsersApi constructor(val source: MangaSour
return config[configKeyDomain]
}
fun getDomain(subdomain: String): String {
val domain = getDomain()
return subdomain + "." + domain.removePrefix("www.")
}
/**
* Create a unique id for [Manga]/[MangaChapter]/[MangaPage].
* @param url must be relative url, without a domain
@@ -150,34 +155,6 @@ abstract class MangaParser @InternalParsersApi constructor(val source: MangaSour
return h
}
/**
* Convert relative url to an absolute using [getDomain]
*/
@Deprecated(
message = "Use toAbsoluteUrl() instead",
replaceWith = ReplaceWith(
"toAbsoluteUrl(getDomain(), subdomain)",
"org.koitharu.kotatsu.parsers.util.toAbsoluteUrl",
),
)
protected fun String.withDomain(subdomain: String): String {
return toAbsoluteUrl(getDomain(), subdomain)
}
/**
* Convert relative url to an absolute using [getDomain]
*/
@Deprecated(
message = "Use toAbsoluteUrl() instead",
replaceWith = ReplaceWith(
"toAbsoluteUrl(getDomain())",
"org.koitharu.kotatsu.parsers.util.toAbsoluteUrl",
),
)
protected fun String.withDomain(): String {
return toAbsoluteUrl(getDomain())
}
@InternalParsersApi
@Suppress("NOTHING_TO_INLINE")
protected inline fun parseFailed(message: String? = null): Nothing {

6
src/main/kotlin/org/koitharu/kotatsu/parsers/site/AnibelParser.kt
View File

@@ -72,7 +72,7 @@ internal class AnibelParser(override val context: MangaLoaderContext) : MangaPar
id = generateUid(mediaId),
title = title.getString("be"),
coverUrl = jo.getString("poster").removePrefix("/cdn")
.toAbsoluteUrl(getDomain(), "cdn") + "?width=200&height=280",
.toAbsoluteUrl(getDomain("cdn")) + "?width=200&height=280",
altTitle = title.getString("alt").takeUnless(String::isEmpty),
author = null,
isNsfw = false,
@@ -112,7 +112,7 @@ internal class AnibelParser(override val context: MangaLoaderContext) : MangaPar
).getJSONObject("media")
val title = details.getJSONObject("title")
val poster = details.getString("poster").removePrefix("/cdn")
.toAbsoluteUrl(getDomain(), "cdn")
.toAbsoluteUrl(getDomain("cdn"))
val chapters = apiCall(
"""
chapters(mediaId: "${details.getString("mediaId")}") {
@@ -213,7 +213,7 @@ internal class AnibelParser(override val context: MangaLoaderContext) : MangaPar
id = generateUid(mediaId),
title = title.getString("be"),
coverUrl = jo.getString("poster").removePrefix("/cdn")
.toAbsoluteUrl(getDomain(), "cdn") + "?width=200&height=280",
.toAbsoluteUrl(getDomain("cdn")) + "?width=200&height=280",
altTitle = title.getString("en").takeUnless(String::isEmpty),
author = null,
isNsfw = false,

2
src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaTownParser.kt
View File

@@ -190,7 +190,7 @@ internal class MangaTownParser(override val context: MangaLoaderContext) : Manga
}
private suspend fun bypassLicensedChapters(manga: Manga): List<MangaChapter> {
val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain(), "m")).parseHtml()
val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain("m"))).parseHtml()
val list = doc.body().selectFirst("ul.detail-ch-list") ?: return emptyList()
val dateFormat = SimpleDateFormat("MMM dd,yyyy", Locale.US)
return list.select("li").asReversed().mapIndexedNotNull { i, li ->

59
src/main/kotlin/org/koitharu/kotatsu/parsers/site/NicovideoSeigaParser.kt
View File

@@ -1,9 +1,6 @@
package org.koitharu.kotatsu.parsers.site
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.MangaParserAuthProvider
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.*
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.exception.AuthRequiredException
import org.koitharu.kotatsu.parsers.model.*
@@ -13,22 +10,21 @@ import java.util.*
private const val STATUS_ONGOING = "連載"
private const val STATUS_FINISHED = "完結"
@MangaSourceParser("NICOVIDEOSEIGA", "Nicovideo Seiga", "ja")
class NicovideoSeigaParser(override val context: MangaLoaderContext) : MangaParser(MangaSource.NICOVIDEOSEIGA),
@MangaSourceParser("NICOVIDEO_SEIGA", "Nicovideo Seiga", "ja")
class NicovideoSeigaParser(override val context: MangaLoaderContext) :
MangaParser(MangaSource.NICOVIDEO_SEIGA),
MangaParserAuthProvider {
override val authUrl: String
get() = "https://account.nicovideo.jp/login?site=seiga"
get() = "https://${getDomain("account")}/login?site=seiga"
override val isAuthorized: Boolean
get() {
return context.cookieJar.getCookies(getDomain()).any {
it.name == "user_session"
}
get() = context.cookieJar.getCookies(getDomain("seiga")).any {
it.name == "user_session"
}
override suspend fun getUsername(): String {
val body = context.httpGet("https://app.nicovideo.jp/my/apps").parseHtml().body()
val body = context.httpGet("https://${getDomain("app")}/my/apps").parseHtml().body()
return body.selectFirst("#userinfo > div > div > strong")?.text() ?: throw AuthRequiredException(source)
}
@@ -37,21 +33,23 @@ class NicovideoSeigaParser(override val context: MangaLoaderContext) : MangaPars
SortOrder.POPULARITY,
)
override val configKeyDomain: ConfigKey.Domain = ConfigKey.Domain("seiga.nicovideo.jp", null)
override val configKeyDomain: ConfigKey.Domain = ConfigKey.Domain("nicovideo.jp", null)
@InternalParsersApi
override suspend fun getList(
offset: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder?,
sortOrder: SortOrder,
): List<Manga> {
val page = (offset / 20f).toIntUp().inc()
val domain = getDomain("seiga")
val url = when {
!query.isNullOrEmpty() -> return if (offset == 0) getSearchList(query, page) else emptyList()
tags.isNullOrEmpty() -> "https://${getDomain()}/manga/list?page=$page&sort=${getSortKey(sortOrder)}"
tags.size == 1 -> "https://${getDomain()}${tags.first().key}?page=$page&sort=${getSortKey(sortOrder)}"
tags.isNullOrEmpty() -> "https://$domain/manga/list?page=$page&sort=${getSortKey(sortOrder)}"
tags.size == 1 -> "https://$domain/manga/list?category=${tags.first().key}&page=$page&sort=${getSortKey(sortOrder)}"
tags.size > 1 -> throw IllegalArgumentException("This source supports only 1 category")
else -> "https://${getDomain()}/manga/list?page=$page&sort=${getSortKey(sortOrder)}"
else -> "https://$domain/manga/list?page=$page&sort=${getSortKey(sortOrder)}"
}
val doc = context.httpGet(url).parseHtml()
val comicList = doc.body().select("#comic_list > ul > li") ?: parseFailed("Container not found")
@@ -80,16 +78,18 @@ class NicovideoSeigaParser(override val context: MangaLoaderContext) : MangaPars
STATUS_FINISHED -> MangaState.FINISHED
else -> null
},
publicUrl = href.toAbsoluteUrl(item.host ?: getDomain()),
publicUrl = href.toAbsoluteUrl(item.host ?: getDomain("seiga")),
source = source,
)
}
}
override suspend fun getDetails(manga: Manga): Manga {
val doc = context.httpGet(manga.url.withDomain()).parseHtml()
val doc = context.httpGet(manga.url.toAbsoluteUrl(getDomain("seiga"))).parseHtml()
val contents = doc.body().selectFirst("#contents") ?: parseFailed("Cannot find root")
val statusText = contents.select("div.mg_work_detail > div > div:nth-child(2) > div.tip.content_status.status_series > span").text()
val statusText = contents
.select("div.mg_work_detail > div > div:nth-child(2) > div.tip.content_status.status_series > span")
.text()
return manga.copy(
description = contents.selectFirst("div.mg_work_detail > div > div.row > div.description_text")?.html(),
largeCoverUrl = contents.selectFirst("div.primaries > div.main_visual > a > img")?.attrAsAbsoluteUrlOrNull("src"),
@@ -116,7 +116,7 @@ class NicovideoSeigaParser(override val context: MangaLoaderContext) : MangaPars
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.withDomain()
val fullUrl = chapter.url.toAbsoluteUrl(getDomain("seiga"))
val doc = context.httpGet(fullUrl).parseHtml()
if (!doc.select("#login_manga").isEmpty())
throw AuthRequiredException(source)
@@ -134,7 +134,7 @@ class NicovideoSeigaParser(override val context: MangaLoaderContext) : MangaPars
}
override suspend fun getTags(): Set<MangaTag> {
val doc = context.httpGet("https://${getDomain()}/manga/list").parseHtml()
val doc = context.httpGet("https://${getDomain("seiga")}/manga/list").parseHtml()
val root = doc.body().select("#mg_category_list > ul > li") ?: parseFailed("Cannot find tags")
return root.mapToSet { li ->
val a = li.selectFirst("a") ?: parseFailed("a is null")
@@ -147,7 +147,7 @@ class NicovideoSeigaParser(override val context: MangaLoaderContext) : MangaPars
}
private suspend fun getSearchList(query: String, page: Int): List<Manga> {
val domain = getDomain()
val domain = getDomain("seiga")
val doc = context.httpGet("https://$domain/manga/search/?q=$query&page=$page&sort=score").parseHtml()
val root = doc.body().select(".search_result__item")
return root.mapNotNull { item ->
@@ -155,7 +155,7 @@ class NicovideoSeigaParser(override val context: MangaLoaderContext) : MangaPars
Manga(
id = generateUid(href),
url = href,
publicUrl = href.toAbsoluteUrl(item.host ?: getDomain()),
publicUrl = href.toAbsoluteUrl(item.host ?: domain),
title = item.selectFirst(".search_result__item__info > .search_result__item__info--title > a")
?.text()?.trim() ?: return@mapNotNull null,
altTitle = null,
@@ -171,10 +171,9 @@ class NicovideoSeigaParser(override val context: MangaLoaderContext) : MangaPars
}
}
private fun getSortKey(sortOrder: SortOrder?) =
when (sortOrder ?: sortOrders.minByOrNull { it.ordinal }) {
SortOrder.POPULARITY -> "manga_view"
SortOrder.UPDATED -> "manga_updated"
else -> "manga_view"
}
private fun getSortKey(sortOrder: SortOrder) = when (sortOrder) {
SortOrder.POPULARITY -> "manga_view"
SortOrder.UPDATED -> "manga_updated"
else -> "manga_view"
}
}

2
src/main/kotlin/org/koitharu/kotatsu/parsers/site/RemangaParser.kt
View File

@@ -171,7 +171,7 @@ internal class RemangaParser(
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val referer = "https://${getDomain()}/"
val content = context.httpGet(chapter.url.toAbsoluteUrl(getDomain(), "api"), getApiHeaders())
val content = context.httpGet(chapter.url.toAbsoluteUrl(getDomain("api")), getApiHeaders())
.handle401()
.parseJson()
.getJSONObject("content")

82
src/main/kotlin/org/koitharu/kotatsu/parsers/util/Parse.kt
View File

@@ -8,11 +8,7 @@ import okhttp3.internal.closeQuietly
import org.json.JSONArray
import org.json.JSONObject
import org.jsoup.Jsoup
import org.jsoup.internal.StringUtil
import org.jsoup.nodes.Document
import org.jsoup.nodes.Element
import org.jsoup.nodes.Node
import org.jsoup.select.Elements
import java.text.DateFormat
/**
@@ -50,49 +46,6 @@ fun Response.parseJsonArray(): JSONArray = try {
closeQuietly()
}
@Deprecated(
message = "",
level = DeprecationLevel.ERROR,
replaceWith = ReplaceWith("firstNotNullOfOrNull { it.ownText().takeIf(predicate) }"),
)
inline fun Elements.findOwnText(predicate: (String) -> Boolean): String? {
for (x in this) {
val ownText = x.ownText()
if (predicate(ownText)) {
return ownText
}
}
return null
}
@Deprecated(
message = "",
level = DeprecationLevel.ERROR,
replaceWith = ReplaceWith("firstNotNullOfOrNull { it.text().takeIf(predicate) }"),
)
inline fun Elements.findText(predicate: (String) -> Boolean): String? {
for (x in this) {
val text = x.text()
if (predicate(text)) {
return text
}
}
return null
}
@Deprecated(
message = "Use toAbsoluteUrl() instead",
level = DeprecationLevel.ERROR,
replaceWith = ReplaceWith("toAbsoluteUrl(node.host)"),
)
fun String.inContextOf(node: Node): String {
return if (this.isEmpty()) {
""
} else {
StringUtil.resolve(node.baseUri(), this)
}
}
/**
* Convert url to relative if it is on [domain]
* @return an url relative to the [domain] or absolute, if domain is mismatching
@@ -114,41 +67,6 @@ fun String.toAbsoluteUrl(domain: String): String = when {
else -> this
}
/**
* Convert url to absolute with specified domain and subdomain
* @return an absolute url with [subdomain].[domain] if this is relative
*/
fun String.toAbsoluteUrl(domain: String, subdomain: String): String {
if (!this.startsWith('/')) return this
return toAbsoluteUrl(subdomain + "." + domain.removePrefix("www."))
}
@Deprecated(
message = "",
level = DeprecationLevel.ERROR,
replaceWith = ReplaceWith("attrAsRelativeUrl(attributeKey)"),
)
fun Element.relUrl(attributeKey: String): String {
val attr = attr(attributeKey).trim()
if (attr.isEmpty()) {
return ""
}
if (attr.startsWith("/")) {
return attr
}
val baseUrl = REGEX_URL_BASE.find(baseUri())?.value ?: return attr
return attr.removePrefix(baseUrl.dropLast(1))
}
private val REGEX_URL_BASE = Regex("^[^/]{2,6}://[^/]+/", RegexOption.IGNORE_CASE)
@Deprecated(
message = "",
level = DeprecationLevel.ERROR,
replaceWith = ReplaceWith("styleValueOrNull(property)"),
)
fun Element.css(property: String): String? = styleValueOrNull(property)
fun DateFormat.tryParse(str: String?): Long = if (str.isNullOrEmpty()) {
0L
} else {

2
src/test/kotlin/org/koitharu/kotatsu/parsers/MangaParserTest.kt
View File

@@ -135,7 +135,7 @@ internal class MangaParserTest {
.scheme("https")
.toString()
val response = context.doRequest(url)
val realDomain = response.request.url.host
val realDomain = response.request.url.topPrivateDomain()
assertEquals(defaultDomain, realDomain)
}