Add documentation comments
This commit is contained in:
@@ -18,6 +18,11 @@ abstract class MangaLoaderContext {
|
||||
|
||||
abstract val cookieJar: CookieJar
|
||||
|
||||
/**
|
||||
* Do a GET http request to specific url
|
||||
* @param url
|
||||
* @param headers an additional headers for request, may be null
|
||||
*/
|
||||
suspend fun httpGet(url: String, headers: Headers? = null): Response {
|
||||
val request = Request.Builder()
|
||||
.get()
|
||||
@@ -28,6 +33,11 @@ abstract class MangaLoaderContext {
|
||||
return httpClient.newCall(request.build()).await().ensureSuccess()
|
||||
}
|
||||
|
||||
/**
|
||||
* Do a HEAD http request to specific url
|
||||
* @param url
|
||||
* @param headers an additional headers for request, may be null
|
||||
*/
|
||||
suspend fun httpHead(url: String, headers: Headers? = null): Response {
|
||||
val request = Request.Builder()
|
||||
.head()
|
||||
@@ -38,6 +48,12 @@ abstract class MangaLoaderContext {
|
||||
return httpClient.newCall(request.build()).await().ensureSuccess()
|
||||
}
|
||||
|
||||
/**
|
||||
* Do a POST http request to specific url with `multipart/form-data` payload
|
||||
* @param url
|
||||
* @param form payload as key=>value map
|
||||
* @param headers an additional headers for request, may be null
|
||||
*/
|
||||
suspend fun httpPost(
|
||||
url: String,
|
||||
form: Map<String, String>,
|
||||
@@ -56,6 +72,12 @@ abstract class MangaLoaderContext {
|
||||
return httpClient.newCall(request.build()).await().ensureSuccess()
|
||||
}
|
||||
|
||||
/**
|
||||
* Do a POST http request to specific url with `multipart/form-data` payload
|
||||
* @param url
|
||||
* @param payload payload as `key=value` string with `&` separator
|
||||
* @param headers an additional headers for request, may be null
|
||||
*/
|
||||
suspend fun httpPost(
|
||||
url: String,
|
||||
payload: String,
|
||||
@@ -79,6 +101,11 @@ abstract class MangaLoaderContext {
|
||||
return httpClient.newCall(request.build()).await().ensureSuccess()
|
||||
}
|
||||
|
||||
/**
|
||||
* Do a GraphQL request to specific url
|
||||
* @param endpoint an url
|
||||
* @param query GraphQL request payload
|
||||
*/
|
||||
suspend fun graphQLQuery(endpoint: String, query: String): JSONObject {
|
||||
val body = JSONObject()
|
||||
body.put("operationName", null as Any?)
|
||||
@@ -104,6 +131,11 @@ abstract class MangaLoaderContext {
|
||||
|
||||
open fun getPreferredLocales(): List<Locale> = listOf(Locale.getDefault())
|
||||
|
||||
/**
|
||||
* Execute JavaScript code and return result
|
||||
* @param script JavaScript source code
|
||||
* @return execution result as string, may be null
|
||||
*/
|
||||
abstract suspend fun evaluateJs(script: String): String?
|
||||
|
||||
abstract fun getConfig(source: MangaSource): MangaSourceConfig
|
||||
|
||||
@@ -1,20 +1,41 @@
|
||||
package org.koitharu.kotatsu.parsers
|
||||
|
||||
import androidx.annotation.CallSuper
|
||||
import org.koitharu.kotatsu.parsers.config.ConfigKey
|
||||
import org.koitharu.kotatsu.parsers.exception.ParseException
|
||||
import org.koitharu.kotatsu.parsers.model.*
|
||||
import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl
|
||||
import java.util.*
|
||||
|
||||
abstract class MangaParser(val source: MangaSource) {
|
||||
|
||||
protected abstract val context: MangaLoaderContext
|
||||
|
||||
/**
|
||||
* Supported [SortOrder] variants. Must not be empty.
|
||||
*
|
||||
* For better performance use [EnumSet] for more than one item.
|
||||
*/
|
||||
abstract val sortOrders: Set<SortOrder>
|
||||
|
||||
val config by lazy { context.getConfig(source) }
|
||||
|
||||
/**
|
||||
* Provide default domain and available alternatives, if any.
|
||||
*
|
||||
* Never hardcode domain in requests, use [getDomain] instead.
|
||||
*/
|
||||
protected abstract val configKeyDomain: ConfigKey.Domain
|
||||
|
||||
/**
|
||||
* Parse list of manga by specified criteria
|
||||
*
|
||||
* @param offset starting from 0 and used for pagination.
|
||||
* Note than passed value may not be divisible by internal page size, so you should adjust it manually.
|
||||
* @param query search query, may be null or empty if no search needed
|
||||
* @param tags genres for filtering, values from [getTags] and [Manga.tags]. May be null or empty
|
||||
* @param sortOrder one of [sortOrders] or null for default value
|
||||
*/
|
||||
abstract suspend fun getList(
|
||||
offset: Int,
|
||||
query: String? = null,
|
||||
@@ -22,16 +43,35 @@ abstract class MangaParser(val source: MangaSource) {
|
||||
sortOrder: SortOrder? = null,
|
||||
): List<Manga>
|
||||
|
||||
/**
|
||||
* Parse details for [Manga]: chapters list, description, large cover, etc.
|
||||
* Must return the same manga, may change any fields excepts id, url and source
|
||||
* @see Manga.copy
|
||||
*/
|
||||
abstract suspend fun getDetails(manga: Manga): Manga
|
||||
|
||||
/**
|
||||
* Parse pages list for specified chapter.
|
||||
* @see MangaPage for details
|
||||
*/
|
||||
abstract suspend fun getPages(chapter: MangaChapter): List<MangaPage>
|
||||
|
||||
/**
|
||||
* Fetch direct link to the page image.
|
||||
*/
|
||||
open suspend fun getPageUrl(page: MangaPage): String = page.url.withDomain()
|
||||
|
||||
/**
|
||||
* Fetch available tags (genres) for source
|
||||
*/
|
||||
abstract suspend fun getTags(): Set<MangaTag>
|
||||
|
||||
/**
|
||||
* Returns direct link to the website favicon
|
||||
*/
|
||||
open fun getFaviconUrl() = "https://${getDomain()}/favicon.ico"
|
||||
|
||||
@CallSuper
|
||||
open fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) {
|
||||
keys.add(configKeyDomain)
|
||||
}
|
||||
@@ -42,6 +82,13 @@ abstract class MangaParser(val source: MangaSource) {
|
||||
return config[configKeyDomain]
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a unique id for [Manga]/[MangaChapter]/[MangaPage].
|
||||
* @param url must be relative url, without a domain
|
||||
* @see [Manga.id]
|
||||
* @see [MangaChapter.id]
|
||||
* @see [MangaPage.id]
|
||||
*/
|
||||
protected fun generateUid(url: String): Long {
|
||||
var h = 1125899906842597L
|
||||
source.name.forEach { c ->
|
||||
@@ -53,6 +100,13 @@ abstract class MangaParser(val source: MangaSource) {
|
||||
return h
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a unique id for [Manga]/[MangaChapter]/[MangaPage].
|
||||
* @param id an internal identifier
|
||||
* @see [Manga.id]
|
||||
* @see [MangaChapter.id]
|
||||
* @see [MangaPage.id]
|
||||
*/
|
||||
protected fun generateUid(id: Long): Long {
|
||||
var h = 1125899906842597L
|
||||
source.name.forEach { c ->
|
||||
@@ -62,6 +116,9 @@ abstract class MangaParser(val source: MangaSource) {
|
||||
return h
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert relative url to an absolute using [getDomain]
|
||||
*/
|
||||
protected fun String.withDomain(subdomain: String? = null): String {
|
||||
var domain = getDomain()
|
||||
if (subdomain != null) {
|
||||
@@ -70,7 +127,8 @@ abstract class MangaParser(val source: MangaSource) {
|
||||
return toAbsoluteUrl(domain)
|
||||
}
|
||||
|
||||
protected fun parseFailed(message: String? = null): Nothing {
|
||||
throw ParseException(message)
|
||||
@Suppress("NOTHING_TO_INLINE")
|
||||
protected inline fun parseFailed(message: String? = null): Nothing {
|
||||
throw ParseException(message, null)
|
||||
}
|
||||
}
|
||||
@@ -1,10 +1,30 @@
|
||||
package org.koitharu.kotatsu.parsers
|
||||
|
||||
import org.koitharu.kotatsu.parsers.exception.AuthRequiredException
|
||||
import org.koitharu.kotatsu.parsers.exception.ParseException
|
||||
|
||||
/**
|
||||
* Implement this in your parser for authorization support
|
||||
*/
|
||||
interface MangaParserAuthProvider {
|
||||
|
||||
/**
|
||||
* Return link to the login page, which will be opened in browser.
|
||||
* Must be an absolute url
|
||||
*/
|
||||
val authUrl: String
|
||||
|
||||
/**
|
||||
* Quick check if user is logged in.
|
||||
* In most case you should check for cookies in [MangaLoaderContext.cookieJar].
|
||||
*/
|
||||
val isAuthorized: Boolean
|
||||
|
||||
/**
|
||||
* Fetch and return current user`s name or login.
|
||||
* Normally should not be called if [isAuthorized] returns false
|
||||
* @throws [AuthRequiredException] if user is not logged in or authorization is expired
|
||||
* @throws [ParseException] on parsing error
|
||||
*/
|
||||
suspend fun getUsername(): String
|
||||
}
|
||||
@@ -1,8 +1,20 @@
|
||||
package org.koitharu.kotatsu.parsers
|
||||
|
||||
/**
|
||||
* Annotate each [MangaParser] implementation with this annotation, used by codegen
|
||||
*/
|
||||
@Target(AnnotationTarget.CLASS)
|
||||
annotation class MangaSourceParser(
|
||||
/**
|
||||
* Name of manga source. Used as an Enum value, must be UPPER_CASE and unique.
|
||||
*/
|
||||
val name: String,
|
||||
/**
|
||||
* User-friendly title of manga source. In most case equals the website name.
|
||||
*/
|
||||
val title: String,
|
||||
/**
|
||||
* Language code (for example "en" or "ru") or blank if parser provide manga on different languages.
|
||||
*/
|
||||
val locale: String = "",
|
||||
)
|
||||
@@ -2,6 +2,9 @@ package org.koitharu.kotatsu.parsers.exception
|
||||
|
||||
import org.koitharu.kotatsu.parsers.model.MangaSource
|
||||
|
||||
/**
|
||||
* Authorization is required for access to the requested content
|
||||
*/
|
||||
class AuthRequiredException(
|
||||
val source: MangaSource,
|
||||
) : RuntimeException("Authorization required")
|
||||
@@ -1,6 +1,6 @@
|
||||
package org.koitharu.kotatsu.parsers.exception
|
||||
|
||||
class ParseException(
|
||||
message: String? = null,
|
||||
class ParseException @JvmOverloads constructor(
|
||||
message: String?,
|
||||
cause: Throwable? = null,
|
||||
) : RuntimeException(message, cause)
|
||||
@@ -1,3 +1,5 @@
|
||||
@file:JvmName("Constants")
|
||||
|
||||
package org.koitharu.kotatsu.parsers.model
|
||||
|
||||
internal const val RATING_UNKNOWN = -1f
|
||||
const val RATING_UNKNOWN = -1f
|
||||
@@ -1,23 +1,76 @@
|
||||
package org.koitharu.kotatsu.parsers.model
|
||||
|
||||
class Manga(
|
||||
/**
|
||||
* Unique identifier for manga
|
||||
*/
|
||||
val id: Long,
|
||||
/**
|
||||
* Manga title, human-readable
|
||||
*/
|
||||
val title: String,
|
||||
/**
|
||||
* Alternative title (for example on other language), may be null
|
||||
*/
|
||||
val altTitle: String?,
|
||||
val url: String, // relative url for internal use
|
||||
/**
|
||||
* Relative url to manga (**without** a domain) or any other uri.
|
||||
* Used principally in parsers
|
||||
*/
|
||||
val url: String,
|
||||
/**
|
||||
* Absolute url to manga, must be ready to open in browser
|
||||
*/
|
||||
val publicUrl: String,
|
||||
val rating: Float, // normalized value [0..1] or -1
|
||||
/**
|
||||
* Normalized manga rating, must be in range of 0..1 or [RATING_UNKNOWN] if rating s unknown
|
||||
* @see hasRating
|
||||
*/
|
||||
val rating: Float,
|
||||
/**
|
||||
* Indicates that manga may contain sensitive information (18+, NSFW)
|
||||
*/
|
||||
val isNsfw: Boolean,
|
||||
/**
|
||||
* Absolute link to the cover
|
||||
* @see largeCoverUrl
|
||||
*/
|
||||
val coverUrl: String,
|
||||
/**
|
||||
* Tags (genres) of the manga
|
||||
*/
|
||||
val tags: Set<MangaTag>,
|
||||
/**
|
||||
* Manga status (ongoing, finished) or null if unknown
|
||||
*/
|
||||
val state: MangaState?,
|
||||
/**
|
||||
* Author of the manga, may be null
|
||||
*/
|
||||
val author: String?,
|
||||
/**
|
||||
* Large cover url (absolute), null if is no large cover
|
||||
* @see coverUrl
|
||||
*/
|
||||
val largeCoverUrl: String? = null,
|
||||
val description: String? = null, // HTML
|
||||
/**
|
||||
* Manga description, may be html or null
|
||||
*/
|
||||
val description: String? = null,
|
||||
/**
|
||||
* List of chapters
|
||||
*/
|
||||
val chapters: List<MangaChapter>? = null,
|
||||
/**
|
||||
* Manga source
|
||||
*/
|
||||
val source: MangaSource,
|
||||
) {
|
||||
|
||||
/**
|
||||
* Return if manga has a specified rating
|
||||
* @see rating
|
||||
*/
|
||||
val hasRating: Boolean
|
||||
get() = rating in 0f..1f
|
||||
|
||||
|
||||
@@ -1,12 +1,35 @@
|
||||
package org.koitharu.kotatsu.parsers.model
|
||||
|
||||
class MangaChapter(
|
||||
/**
|
||||
* An unique id of chapter
|
||||
*/
|
||||
val id: Long,
|
||||
/**
|
||||
* User-readable name of chapter
|
||||
*/
|
||||
val name: String,
|
||||
/**
|
||||
* Chapter number starting from 1
|
||||
*/
|
||||
val number: Int,
|
||||
/**
|
||||
* Relative url to chapter (**without** a domain) or any other uri.
|
||||
* Used principally in parsers
|
||||
*/
|
||||
val url: String,
|
||||
/**
|
||||
* User-readable name of scanlator (releaser) or null if unknown
|
||||
*/
|
||||
val scanlator: String?,
|
||||
/**
|
||||
* Chapter upload date in milliseconds
|
||||
*/
|
||||
val uploadDate: Long,
|
||||
/**
|
||||
* User-readable name of branch.
|
||||
* A branch is a group of chapters that overlap (e.g. different languages)
|
||||
*/
|
||||
val branch: String?,
|
||||
val source: MangaSource,
|
||||
) : Comparable<MangaChapter> {
|
||||
@@ -44,6 +67,4 @@ class MangaChapter(
|
||||
result = 31 * result + source.hashCode()
|
||||
return result
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@@ -1,9 +1,27 @@
|
||||
package org.koitharu.kotatsu.parsers.model
|
||||
|
||||
import org.koitharu.kotatsu.parsers.MangaParser
|
||||
|
||||
class MangaPage(
|
||||
/**
|
||||
* Unique identifier for manga
|
||||
*/
|
||||
val id: Long,
|
||||
/**
|
||||
* Relative url to page (**without** a domain) or any other uri.
|
||||
* Used principally in parsers.
|
||||
* May contain link to image or html page.
|
||||
* @see MangaParser.getPageUrl
|
||||
*/
|
||||
val url: String,
|
||||
/**
|
||||
* Absolute link to the chapter or website home page.
|
||||
* Used in Referer header
|
||||
*/
|
||||
val referer: String,
|
||||
/**
|
||||
* Absolute url of the small page image if exists, null otherwise
|
||||
*/
|
||||
val preview: String?,
|
||||
val source: MangaSource,
|
||||
) {
|
||||
|
||||
@@ -1,7 +1,16 @@
|
||||
package org.koitharu.kotatsu.parsers.model
|
||||
|
||||
import org.koitharu.kotatsu.parsers.MangaParser
|
||||
|
||||
class MangaTag(
|
||||
/**
|
||||
* User-readable tag title, should be in Title case
|
||||
*/
|
||||
val title: String,
|
||||
/**
|
||||
* Identifier of a tag, must be unique among the source.
|
||||
* @see MangaParser.getList
|
||||
*/
|
||||
val key: String,
|
||||
val source: MangaSource,
|
||||
) {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
package org.koitharu.kotatsu.parsers.model
|
||||
|
||||
class WordSet(private vararg val words: String) {
|
||||
internal class WordSet(private vararg val words: String) {
|
||||
|
||||
fun anyWordIn(dateString: String): Boolean = words.any {
|
||||
dateString.contains(it, ignoreCase = true)
|
||||
|
||||
@@ -125,7 +125,7 @@ internal abstract class ChanParser(source: MangaSource) : MangaParser(source) {
|
||||
)
|
||||
}
|
||||
}
|
||||
throw ParseException("Pages list not found at ${chapter.url}")
|
||||
parseFailed("Pages list not found at ${chapter.url}")
|
||||
}
|
||||
|
||||
override suspend fun getTags(): Set<MangaTag> {
|
||||
|
||||
@@ -50,8 +50,7 @@ internal class DesuMeParser(override val context: MangaLoaderContext) : MangaPar
|
||||
append(query)
|
||||
}
|
||||
}
|
||||
val json = context.httpGet(url).parseJson().getJSONArray("response")
|
||||
?: throw ParseException("Invalid response")
|
||||
val json = context.httpGet(url).parseJson().getJSONArray("response") ?: parseFailed("Invalid response")
|
||||
val total = json.length()
|
||||
val list = ArrayList<Manga>(total)
|
||||
for (i in 0 until total) {
|
||||
|
||||
@@ -222,7 +222,7 @@ internal class ExHentaiParser(
|
||||
?: if (doc.getElementById("userlinksguest") != null) {
|
||||
throw AuthRequiredException(source)
|
||||
} else {
|
||||
throw ParseException()
|
||||
throw ParseException(null)
|
||||
}
|
||||
return username
|
||||
}
|
||||
|
||||
@@ -66,7 +66,7 @@ internal abstract class GroupleParser(source: MangaSource, userAgent: String) :
|
||||
else -> advancedSearch(domain, tags)
|
||||
}.parseHtml().body()
|
||||
val root = (doc.getElementById("mangaBox") ?: doc.getElementById("mangaResults"))
|
||||
?: throw ParseException("Cannot find root")
|
||||
?: parseFailed("Cannot find root")
|
||||
val tiles = root.selectFirst("div.tiles.row") ?: if (
|
||||
root.select(".alert").any { it.ownText() == NOTHING_FOUND }
|
||||
) {
|
||||
@@ -128,7 +128,7 @@ internal abstract class GroupleParser(source: MangaSource, userAgent: String) :
|
||||
override suspend fun getDetails(manga: Manga): Manga {
|
||||
val doc = context.httpGet(manga.url.withDomain(), headers).parseHtml()
|
||||
val root = doc.body().getElementById("mangaBox")?.selectFirst("div.leftContent")
|
||||
?: throw ParseException("Cannot find root")
|
||||
?: parseFailed("Cannot find root")
|
||||
val dateFormat = SimpleDateFormat("dd.MM.yy", Locale.US)
|
||||
val coverImg = root.selectFirst("div.subject-cover")?.selectFirst("img")
|
||||
return manga.copy(
|
||||
@@ -203,7 +203,7 @@ internal abstract class GroupleParser(source: MangaSource, userAgent: String) :
|
||||
)
|
||||
}
|
||||
}
|
||||
throw ParseException("Pages list not found at ${chapter.url}")
|
||||
parseFailed("Pages list not found at ${chapter.url}")
|
||||
}
|
||||
|
||||
override suspend fun getPageUrl(page: MangaPage): String {
|
||||
|
||||
@@ -4,7 +4,6 @@ import org.koitharu.kotatsu.parsers.MangaLoaderContext
|
||||
import org.koitharu.kotatsu.parsers.MangaParser
|
||||
import org.koitharu.kotatsu.parsers.MangaSourceParser
|
||||
import org.koitharu.kotatsu.parsers.config.ConfigKey
|
||||
import org.koitharu.kotatsu.parsers.exception.ParseException
|
||||
import org.koitharu.kotatsu.parsers.model.*
|
||||
import org.koitharu.kotatsu.parsers.util.*
|
||||
import java.text.SimpleDateFormat
|
||||
@@ -103,8 +102,7 @@ class MangaInUaParser(override val context: MangaLoaderContext) : MangaParser(Ma
|
||||
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
|
||||
val fullUrl = chapter.url.withDomain()
|
||||
val doc = context.httpGet(fullUrl).parseHtml()
|
||||
val root = doc.body().getElementById("comics")
|
||||
?: throw ParseException("Root not found")
|
||||
val root = doc.body().getElementById("comics") ?: parseFailed("Root not found")
|
||||
return root.select("ul.xfieldimagegallery").map { ul ->
|
||||
val img = ul.selectFirst("img") ?: parseFailed("Page image not found")
|
||||
val url = img.attrAsAbsoluteUrl("data-src")
|
||||
@@ -123,7 +121,7 @@ class MangaInUaParser(override val context: MangaLoaderContext) : MangaParser(Ma
|
||||
val doc = context.httpGet("https://$domain/mangas").parseHtml()
|
||||
val root = doc.body().getElementById("menu_1")?.selectFirst("div.menu__wrapper") ?: parseFailed("Cannot find root")
|
||||
return root.select("li").mapToSet { li ->
|
||||
val a = li.selectFirst("a") ?: throw ParseException("a is null")
|
||||
val a = li.selectFirst("a") ?: parseFailed("a is null")
|
||||
MangaTag(
|
||||
title = a.ownText(),
|
||||
key = a.attr("href").removeSuffix("/"),
|
||||
|
||||
@@ -15,8 +15,18 @@ val Element.host: String?
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Return an attribute value or null if it is missing or empty
|
||||
* @see [Element.attr] which returns empty string instead of null
|
||||
*/
|
||||
fun Element.attrOrNull(attributeKey: String) = attr(attributeKey).takeUnless { it.isEmpty() }
|
||||
|
||||
/**
|
||||
* Return an attribute value as relative url or null if it is missing or empty
|
||||
* @see attrAsRelativeUrl
|
||||
* @see attrAsAbsoluteUrlOrNull
|
||||
* @see attrAsAbsoluteUrl
|
||||
*/
|
||||
fun Element.attrAsRelativeUrlOrNull(attributeKey: String): String? {
|
||||
val attr = attr(attributeKey).trim()
|
||||
if (attr.isEmpty()) {
|
||||
@@ -29,12 +39,25 @@ fun Element.attrAsRelativeUrlOrNull(attributeKey: String): String? {
|
||||
return attr.substringAfter(host)
|
||||
}
|
||||
|
||||
/**
|
||||
* Return an attribute value as relative url or throw an exception if it is missing or empty
|
||||
* @throws IllegalArgumentException if attribute value is missing or empty
|
||||
* @see attrAsRelativeUrlOrNull
|
||||
* @see attrAsAbsoluteUrlOrNull
|
||||
* @see attrAsAbsoluteUrl
|
||||
*/
|
||||
fun Element.attrAsRelativeUrl(attributeKey: String): String {
|
||||
return requireNotNull(attrAsRelativeUrlOrNull(attributeKey)) {
|
||||
"Cannot get relative url for $attributeKey: \"${attr(attributeKey)}\""
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Return an attribute value as absolute url or null if it is missing or empty
|
||||
* @see attrAsAbsoluteUrl
|
||||
* @see attrAsRelativeUrl
|
||||
* @see attrAsRelativeUrlOrNull
|
||||
*/
|
||||
fun Element.attrAsAbsoluteUrlOrNull(attributeKey: String): String? {
|
||||
val attr = attr(attributeKey).trim()
|
||||
if (attr.isEmpty()) {
|
||||
@@ -43,12 +66,22 @@ fun Element.attrAsAbsoluteUrlOrNull(attributeKey: String): String? {
|
||||
return (baseUri().toHttpUrlOrNull()?.newBuilder(attr) ?: return null).toString()
|
||||
}
|
||||
|
||||
/**
|
||||
* Return an attribute value as absolute url or throw an exception if it is missing or empty
|
||||
* @throws IllegalArgumentException if attribute value is missing or empty
|
||||
* @see attrAsAbsoluteUrlOrNull
|
||||
* @see attrAsRelativeUrl
|
||||
* @see attrAsRelativeUrlOrNull
|
||||
*/
|
||||
fun Element.attrAsAbsoluteUrl(attributeKey: String): String {
|
||||
return requireNotNull(attrAsAbsoluteUrlOrNull(attributeKey)) {
|
||||
"Cannot get absolute url for $attributeKey: \"${attr(attributeKey)}\""
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Return css value from `style` attribute or null if it is missing
|
||||
*/
|
||||
fun Element.styleValueOrNull(property: String): String? {
|
||||
val regex = Regex("${Regex.escape(property)}\\s*:\\s*[^;]+")
|
||||
val css = attr("style").find(regex) ?: return null
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
package org.koitharu.kotatsu.parsers.util
|
||||
|
||||
import okhttp3.Response
|
||||
import okhttp3.ResponseBody
|
||||
import okhttp3.internal.closeQuietly
|
||||
import org.json.JSONArray
|
||||
import org.json.JSONObject
|
||||
@@ -14,37 +15,39 @@ import org.jsoup.nodes.Node
|
||||
import org.jsoup.select.Elements
|
||||
import java.text.DateFormat
|
||||
|
||||
fun Response.parseHtml(): Document {
|
||||
try {
|
||||
(body?.byteStream() ?: throw NullPointerException("Response body is null")).use { stream ->
|
||||
val charset = body!!.contentType()?.charset()?.name()
|
||||
return Jsoup.parse(
|
||||
stream,
|
||||
charset,
|
||||
request.url.toString(),
|
||||
)
|
||||
}
|
||||
} finally {
|
||||
closeQuietly()
|
||||
}
|
||||
/**
|
||||
* Parse [Response] body as html document using Jsoup
|
||||
* @see [parseJson]
|
||||
* @see [parseJsonArray]
|
||||
*/
|
||||
fun Response.parseHtml(): Document = try {
|
||||
val body = requireBody()
|
||||
val charset = body.contentType()?.charset()?.name()
|
||||
Jsoup.parse(body.byteStream(), charset, request.url.toString())
|
||||
} finally {
|
||||
closeQuietly()
|
||||
}
|
||||
|
||||
fun Response.parseJson(): JSONObject {
|
||||
try {
|
||||
val string = body?.string() ?: throw NullPointerException("Response body is null")
|
||||
return JSONObject(string)
|
||||
} finally {
|
||||
closeQuietly()
|
||||
}
|
||||
/**
|
||||
* Parse [Response] body as [JSONObject]
|
||||
* @see [parseJsonArray]
|
||||
* @see [parseHtml]
|
||||
*/
|
||||
fun Response.parseJson(): JSONObject = try {
|
||||
JSONObject(requireBody().string())
|
||||
} finally {
|
||||
closeQuietly()
|
||||
}
|
||||
|
||||
fun Response.parseJsonArray(): JSONArray {
|
||||
try {
|
||||
val string = body?.string() ?: throw NullPointerException("Response body is null")
|
||||
return JSONArray(string)
|
||||
} finally {
|
||||
closeQuietly()
|
||||
}
|
||||
/**
|
||||
* Parse [Response] body as [JSONArray]
|
||||
* @see [parseJson]
|
||||
* @see [parseHtml]
|
||||
*/
|
||||
fun Response.parseJsonArray(): JSONArray = try {
|
||||
JSONArray(requireBody().string())
|
||||
} finally {
|
||||
closeQuietly()
|
||||
}
|
||||
|
||||
@Deprecated(
|
||||
@@ -90,6 +93,10 @@ fun String.inContextOf(node: Node): String {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert url to relative if it is on [domain]
|
||||
* @return an url relative to the [domain] or absolute, if domain is mismatching
|
||||
*/
|
||||
fun String.toRelativeUrl(domain: String): String {
|
||||
if (isEmpty() || startsWith("/")) {
|
||||
return this
|
||||
@@ -97,6 +104,10 @@ fun String.toRelativeUrl(domain: String): String {
|
||||
return replace(Regex("^[^/]{2,6}://${Regex.escape(domain)}+/", RegexOption.IGNORE_CASE), "/")
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert url to absolute with specified [domain]
|
||||
* @return an absolute url with [domain] if this is relative
|
||||
*/
|
||||
fun String.toAbsoluteUrl(domain: String): String = when {
|
||||
this.startsWith("//") -> "https:$this"
|
||||
this.startsWith("/") -> "https://$domain$this"
|
||||
@@ -135,4 +146,6 @@ fun DateFormat.tryParse(str: String?): Long = if (str.isNullOrEmpty()) {
|
||||
runCatching {
|
||||
parse(str)?.time ?: 0L
|
||||
}.getOrDefault(0L)
|
||||
}
|
||||
}
|
||||
|
||||
private fun Response.requireBody(): ResponseBody = requireNotNull(body) { "Response body is null" }
|
||||
@@ -9,15 +9,6 @@ import java.security.MessageDigest
|
||||
import java.util.*
|
||||
import kotlin.math.min
|
||||
|
||||
fun String.longHashCode(): Long {
|
||||
var h = 1125899906842597L
|
||||
val len: Int = this.length
|
||||
for (i in 0 until len) {
|
||||
h = 31 * h + this[i].code
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
fun String.removeSurrounding(vararg chars: Char): String {
|
||||
if (isEmpty()) {
|
||||
return this
|
||||
|
||||
Reference in New Issue
Block a user