From 4a2f899a64187ff494c222927ba2a8dd98741b9c Mon Sep 17 00:00:00 2001 From: Koitharu Date: Fri, 20 May 2022 10:54:35 +0300 Subject: [PATCH] Add documentation comments --- .../kotatsu/parsers/MangaLoaderContext.kt | 32 +++++++++ .../koitharu/kotatsu/parsers/MangaParser.kt | 62 ++++++++++++++++- .../parsers/MangaParserAuthProvider.kt | 20 ++++++ .../kotatsu/parsers/MangaSourceParser.kt | 12 ++++ .../exception/AuthRequiredException.kt | 3 + .../parsers/exception/ParseException.kt | 4 +- .../kotatsu/parsers/model/Constants.kt | 4 +- .../koitharu/kotatsu/parsers/model/Manga.kt | 59 +++++++++++++++- .../kotatsu/parsers/model/MangaChapter.kt | 25 ++++++- .../kotatsu/parsers/model/MangaPage.kt | 18 +++++ .../kotatsu/parsers/model/MangaTag.kt | 9 +++ .../koitharu/kotatsu/parsers/model/WordSet.kt | 2 +- .../kotatsu/parsers/site/ChanParser.kt | 2 +- .../kotatsu/parsers/site/DesuMeParser.kt | 3 +- .../kotatsu/parsers/site/ExHentaiParser.kt | 2 +- .../kotatsu/parsers/site/GroupleParser.kt | 6 +- .../kotatsu/parsers/site/MangaInUaParser.kt | 6 +- .../koitharu/kotatsu/parsers/util/Jsoup.kt | 33 +++++++++ .../koitharu/kotatsu/parsers/util/Parse.kt | 69 +++++++++++-------- .../koitharu/kotatsu/parsers/util/String.kt | 9 --- 20 files changed, 321 insertions(+), 59 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaLoaderContext.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaLoaderContext.kt index ed7f84981..bdd704f10 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaLoaderContext.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaLoaderContext.kt @@ -18,6 +18,11 @@ abstract class MangaLoaderContext { abstract val cookieJar: CookieJar + /** + * Do a GET http request to specific url + * @param url + * @param headers an additional headers for request, may be null + */ suspend fun httpGet(url: String, headers: Headers? = null): Response { val request = Request.Builder() .get() @@ -28,6 +33,11 @@ abstract class MangaLoaderContext { return httpClient.newCall(request.build()).await().ensureSuccess() } + /** + * Do a HEAD http request to specific url + * @param url + * @param headers an additional headers for request, may be null + */ suspend fun httpHead(url: String, headers: Headers? = null): Response { val request = Request.Builder() .head() @@ -38,6 +48,12 @@ abstract class MangaLoaderContext { return httpClient.newCall(request.build()).await().ensureSuccess() } + /** + * Do a POST http request to specific url with `multipart/form-data` payload + * @param url + * @param form payload as key=>value map + * @param headers an additional headers for request, may be null + */ suspend fun httpPost( url: String, form: Map, @@ -56,6 +72,12 @@ abstract class MangaLoaderContext { return httpClient.newCall(request.build()).await().ensureSuccess() } + /** + * Do a POST http request to specific url with `multipart/form-data` payload + * @param url + * @param payload payload as `key=value` string with `&` separator + * @param headers an additional headers for request, may be null + */ suspend fun httpPost( url: String, payload: String, @@ -79,6 +101,11 @@ abstract class MangaLoaderContext { return httpClient.newCall(request.build()).await().ensureSuccess() } + /** + * Do a GraphQL request to specific url + * @param endpoint an url + * @param query GraphQL request payload + */ suspend fun graphQLQuery(endpoint: String, query: String): JSONObject { val body = JSONObject() body.put("operationName", null as Any?) @@ -104,6 +131,11 @@ abstract class MangaLoaderContext { open fun getPreferredLocales(): List = listOf(Locale.getDefault()) + /** + * Execute JavaScript code and return result + * @param script JavaScript source code + * @return execution result as string, may be null + */ abstract suspend fun evaluateJs(script: String): String? abstract fun getConfig(source: MangaSource): MangaSourceConfig diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaParser.kt index 7b7c889b6..c50aa3ea6 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaParser.kt @@ -1,20 +1,41 @@ package org.koitharu.kotatsu.parsers +import androidx.annotation.CallSuper import org.koitharu.kotatsu.parsers.config.ConfigKey import org.koitharu.kotatsu.parsers.exception.ParseException import org.koitharu.kotatsu.parsers.model.* import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl +import java.util.* abstract class MangaParser(val source: MangaSource) { protected abstract val context: MangaLoaderContext + /** + * Supported [SortOrder] variants. Must not be empty. + * + * For better performance use [EnumSet] for more than one item. + */ abstract val sortOrders: Set val config by lazy { context.getConfig(source) } + /** + * Provide default domain and available alternatives, if any. + * + * Never hardcode domain in requests, use [getDomain] instead. + */ protected abstract val configKeyDomain: ConfigKey.Domain + /** + * Parse list of manga by specified criteria + * + * @param offset starting from 0 and used for pagination. + * Note than passed value may not be divisible by internal page size, so you should adjust it manually. + * @param query search query, may be null or empty if no search needed + * @param tags genres for filtering, values from [getTags] and [Manga.tags]. May be null or empty + * @param sortOrder one of [sortOrders] or null for default value + */ abstract suspend fun getList( offset: Int, query: String? = null, @@ -22,16 +43,35 @@ abstract class MangaParser(val source: MangaSource) { sortOrder: SortOrder? = null, ): List + /** + * Parse details for [Manga]: chapters list, description, large cover, etc. + * Must return the same manga, may change any fields excepts id, url and source + * @see Manga.copy + */ abstract suspend fun getDetails(manga: Manga): Manga + /** + * Parse pages list for specified chapter. + * @see MangaPage for details + */ abstract suspend fun getPages(chapter: MangaChapter): List + /** + * Fetch direct link to the page image. + */ open suspend fun getPageUrl(page: MangaPage): String = page.url.withDomain() + /** + * Fetch available tags (genres) for source + */ abstract suspend fun getTags(): Set + /** + * Returns direct link to the website favicon + */ open fun getFaviconUrl() = "https://${getDomain()}/favicon.ico" + @CallSuper open fun onCreateConfig(keys: MutableCollection>) { keys.add(configKeyDomain) } @@ -42,6 +82,13 @@ abstract class MangaParser(val source: MangaSource) { return config[configKeyDomain] } + /** + * Create a unique id for [Manga]/[MangaChapter]/[MangaPage]. + * @param url must be relative url, without a domain + * @see [Manga.id] + * @see [MangaChapter.id] + * @see [MangaPage.id] + */ protected fun generateUid(url: String): Long { var h = 1125899906842597L source.name.forEach { c -> @@ -53,6 +100,13 @@ abstract class MangaParser(val source: MangaSource) { return h } + /** + * Create a unique id for [Manga]/[MangaChapter]/[MangaPage]. + * @param id an internal identifier + * @see [Manga.id] + * @see [MangaChapter.id] + * @see [MangaPage.id] + */ protected fun generateUid(id: Long): Long { var h = 1125899906842597L source.name.forEach { c -> @@ -62,6 +116,9 @@ abstract class MangaParser(val source: MangaSource) { return h } + /** + * Convert relative url to an absolute using [getDomain] + */ protected fun String.withDomain(subdomain: String? = null): String { var domain = getDomain() if (subdomain != null) { @@ -70,7 +127,8 @@ abstract class MangaParser(val source: MangaSource) { return toAbsoluteUrl(domain) } - protected fun parseFailed(message: String? = null): Nothing { - throw ParseException(message) + @Suppress("NOTHING_TO_INLINE") + protected inline fun parseFailed(message: String? = null): Nothing { + throw ParseException(message, null) } } \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaParserAuthProvider.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaParserAuthProvider.kt index 731e66f79..a2541072a 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaParserAuthProvider.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaParserAuthProvider.kt @@ -1,10 +1,30 @@ package org.koitharu.kotatsu.parsers +import org.koitharu.kotatsu.parsers.exception.AuthRequiredException +import org.koitharu.kotatsu.parsers.exception.ParseException + +/** + * Implement this in your parser for authorization support + */ interface MangaParserAuthProvider { + /** + * Return link to the login page, which will be opened in browser. + * Must be an absolute url + */ val authUrl: String + /** + * Quick check if user is logged in. + * In most case you should check for cookies in [MangaLoaderContext.cookieJar]. + */ val isAuthorized: Boolean + /** + * Fetch and return current user`s name or login. + * Normally should not be called if [isAuthorized] returns false + * @throws [AuthRequiredException] if user is not logged in or authorization is expired + * @throws [ParseException] on parsing error + */ suspend fun getUsername(): String } \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaSourceParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaSourceParser.kt index 7267256b4..db5782a34 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaSourceParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/MangaSourceParser.kt @@ -1,8 +1,20 @@ package org.koitharu.kotatsu.parsers +/** + * Annotate each [MangaParser] implementation with this annotation, used by codegen + */ @Target(AnnotationTarget.CLASS) annotation class MangaSourceParser( + /** + * Name of manga source. Used as an Enum value, must be UPPER_CASE and unique. + */ val name: String, + /** + * User-friendly title of manga source. In most case equals the website name. + */ val title: String, + /** + * Language code (for example "en" or "ru") or blank if parser provide manga on different languages. + */ val locale: String = "", ) \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/exception/AuthRequiredException.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/exception/AuthRequiredException.kt index ea7388b74..9a9f1a368 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/exception/AuthRequiredException.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/exception/AuthRequiredException.kt @@ -2,6 +2,9 @@ package org.koitharu.kotatsu.parsers.exception import org.koitharu.kotatsu.parsers.model.MangaSource +/** + * Authorization is required for access to the requested content + */ class AuthRequiredException( val source: MangaSource, ) : RuntimeException("Authorization required") \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/exception/ParseException.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/exception/ParseException.kt index c40eec038..5499fdfbc 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/exception/ParseException.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/exception/ParseException.kt @@ -1,6 +1,6 @@ package org.koitharu.kotatsu.parsers.exception -class ParseException( - message: String? = null, +class ParseException @JvmOverloads constructor( + message: String?, cause: Throwable? = null, ) : RuntimeException(message, cause) \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/model/Constants.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/model/Constants.kt index 1e790de18..b98ce963b 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/model/Constants.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/model/Constants.kt @@ -1,3 +1,5 @@ +@file:JvmName("Constants") + package org.koitharu.kotatsu.parsers.model -internal const val RATING_UNKNOWN = -1f \ No newline at end of file +const val RATING_UNKNOWN = -1f \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/model/Manga.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/model/Manga.kt index a7ee5bea5..49066162d 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/model/Manga.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/model/Manga.kt @@ -1,23 +1,76 @@ package org.koitharu.kotatsu.parsers.model class Manga( + /** + * Unique identifier for manga + */ val id: Long, + /** + * Manga title, human-readable + */ val title: String, + /** + * Alternative title (for example on other language), may be null + */ val altTitle: String?, - val url: String, // relative url for internal use + /** + * Relative url to manga (**without** a domain) or any other uri. + * Used principally in parsers + */ + val url: String, + /** + * Absolute url to manga, must be ready to open in browser + */ val publicUrl: String, - val rating: Float, // normalized value [0..1] or -1 + /** + * Normalized manga rating, must be in range of 0..1 or [RATING_UNKNOWN] if rating s unknown + * @see hasRating + */ + val rating: Float, + /** + * Indicates that manga may contain sensitive information (18+, NSFW) + */ val isNsfw: Boolean, + /** + * Absolute link to the cover + * @see largeCoverUrl + */ val coverUrl: String, + /** + * Tags (genres) of the manga + */ val tags: Set, + /** + * Manga status (ongoing, finished) or null if unknown + */ val state: MangaState?, + /** + * Author of the manga, may be null + */ val author: String?, + /** + * Large cover url (absolute), null if is no large cover + * @see coverUrl + */ val largeCoverUrl: String? = null, - val description: String? = null, // HTML + /** + * Manga description, may be html or null + */ + val description: String? = null, + /** + * List of chapters + */ val chapters: List? = null, + /** + * Manga source + */ val source: MangaSource, ) { + /** + * Return if manga has a specified rating + * @see rating + */ val hasRating: Boolean get() = rating in 0f..1f diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/model/MangaChapter.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/model/MangaChapter.kt index 98ef4cf93..52b01e6b1 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/model/MangaChapter.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/model/MangaChapter.kt @@ -1,12 +1,35 @@ package org.koitharu.kotatsu.parsers.model class MangaChapter( + /** + * An unique id of chapter + */ val id: Long, + /** + * User-readable name of chapter + */ val name: String, + /** + * Chapter number starting from 1 + */ val number: Int, + /** + * Relative url to chapter (**without** a domain) or any other uri. + * Used principally in parsers + */ val url: String, + /** + * User-readable name of scanlator (releaser) or null if unknown + */ val scanlator: String?, + /** + * Chapter upload date in milliseconds + */ val uploadDate: Long, + /** + * User-readable name of branch. + * A branch is a group of chapters that overlap (e.g. different languages) + */ val branch: String?, val source: MangaSource, ) : Comparable { @@ -44,6 +67,4 @@ class MangaChapter( result = 31 * result + source.hashCode() return result } - - } \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/model/MangaPage.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/model/MangaPage.kt index 77de77837..4cc66d298 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/model/MangaPage.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/model/MangaPage.kt @@ -1,9 +1,27 @@ package org.koitharu.kotatsu.parsers.model +import org.koitharu.kotatsu.parsers.MangaParser + class MangaPage( + /** + * Unique identifier for manga + */ val id: Long, + /** + * Relative url to page (**without** a domain) or any other uri. + * Used principally in parsers. + * May contain link to image or html page. + * @see MangaParser.getPageUrl + */ val url: String, + /** + * Absolute link to the chapter or website home page. + * Used in Referer header + */ val referer: String, + /** + * Absolute url of the small page image if exists, null otherwise + */ val preview: String?, val source: MangaSource, ) { diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/model/MangaTag.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/model/MangaTag.kt index f42252c91..0d9fa04b9 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/model/MangaTag.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/model/MangaTag.kt @@ -1,7 +1,16 @@ package org.koitharu.kotatsu.parsers.model +import org.koitharu.kotatsu.parsers.MangaParser + class MangaTag( + /** + * User-readable tag title, should be in Title case + */ val title: String, + /** + * Identifier of a tag, must be unique among the source. + * @see MangaParser.getList + */ val key: String, val source: MangaSource, ) { diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/model/WordSet.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/model/WordSet.kt index 2f4d3aaf8..3f2c0c688 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/model/WordSet.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/model/WordSet.kt @@ -1,6 +1,6 @@ package org.koitharu.kotatsu.parsers.model -class WordSet(private vararg val words: String) { +internal class WordSet(private vararg val words: String) { fun anyWordIn(dateString: String): Boolean = words.any { dateString.contains(it, ignoreCase = true) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ChanParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ChanParser.kt index 164f61c91..fc160db52 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ChanParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ChanParser.kt @@ -125,7 +125,7 @@ internal abstract class ChanParser(source: MangaSource) : MangaParser(source) { ) } } - throw ParseException("Pages list not found at ${chapter.url}") + parseFailed("Pages list not found at ${chapter.url}") } override suspend fun getTags(): Set { diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/DesuMeParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/DesuMeParser.kt index c1c19dde5..c38a80172 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/DesuMeParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/DesuMeParser.kt @@ -50,8 +50,7 @@ internal class DesuMeParser(override val context: MangaLoaderContext) : MangaPar append(query) } } - val json = context.httpGet(url).parseJson().getJSONArray("response") - ?: throw ParseException("Invalid response") + val json = context.httpGet(url).parseJson().getJSONArray("response") ?: parseFailed("Invalid response") val total = json.length() val list = ArrayList(total) for (i in 0 until total) { diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ExHentaiParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ExHentaiParser.kt index 17b45c6c8..5bce1ea4a 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ExHentaiParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ExHentaiParser.kt @@ -222,7 +222,7 @@ internal class ExHentaiParser( ?: if (doc.getElementById("userlinksguest") != null) { throw AuthRequiredException(source) } else { - throw ParseException() + throw ParseException(null) } return username } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/GroupleParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/GroupleParser.kt index 5ce4e34ef..3066e29f8 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/GroupleParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/GroupleParser.kt @@ -66,7 +66,7 @@ internal abstract class GroupleParser(source: MangaSource, userAgent: String) : else -> advancedSearch(domain, tags) }.parseHtml().body() val root = (doc.getElementById("mangaBox") ?: doc.getElementById("mangaResults")) - ?: throw ParseException("Cannot find root") + ?: parseFailed("Cannot find root") val tiles = root.selectFirst("div.tiles.row") ?: if ( root.select(".alert").any { it.ownText() == NOTHING_FOUND } ) { @@ -128,7 +128,7 @@ internal abstract class GroupleParser(source: MangaSource, userAgent: String) : override suspend fun getDetails(manga: Manga): Manga { val doc = context.httpGet(manga.url.withDomain(), headers).parseHtml() val root = doc.body().getElementById("mangaBox")?.selectFirst("div.leftContent") - ?: throw ParseException("Cannot find root") + ?: parseFailed("Cannot find root") val dateFormat = SimpleDateFormat("dd.MM.yy", Locale.US) val coverImg = root.selectFirst("div.subject-cover")?.selectFirst("img") return manga.copy( @@ -203,7 +203,7 @@ internal abstract class GroupleParser(source: MangaSource, userAgent: String) : ) } } - throw ParseException("Pages list not found at ${chapter.url}") + parseFailed("Pages list not found at ${chapter.url}") } override suspend fun getPageUrl(page: MangaPage): String { diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaInUaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaInUaParser.kt index f038962f3..c24ff7abb 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaInUaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaInUaParser.kt @@ -4,7 +4,6 @@ import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaParser import org.koitharu.kotatsu.parsers.MangaSourceParser import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.exception.ParseException import org.koitharu.kotatsu.parsers.model.* import org.koitharu.kotatsu.parsers.util.* import java.text.SimpleDateFormat @@ -103,8 +102,7 @@ class MangaInUaParser(override val context: MangaLoaderContext) : MangaParser(Ma override suspend fun getPages(chapter: MangaChapter): List { val fullUrl = chapter.url.withDomain() val doc = context.httpGet(fullUrl).parseHtml() - val root = doc.body().getElementById("comics") - ?: throw ParseException("Root not found") + val root = doc.body().getElementById("comics") ?: parseFailed("Root not found") return root.select("ul.xfieldimagegallery").map { ul -> val img = ul.selectFirst("img") ?: parseFailed("Page image not found") val url = img.attrAsAbsoluteUrl("data-src") @@ -123,7 +121,7 @@ class MangaInUaParser(override val context: MangaLoaderContext) : MangaParser(Ma val doc = context.httpGet("https://$domain/mangas").parseHtml() val root = doc.body().getElementById("menu_1")?.selectFirst("div.menu__wrapper") ?: parseFailed("Cannot find root") return root.select("li").mapToSet { li -> - val a = li.selectFirst("a") ?: throw ParseException("a is null") + val a = li.selectFirst("a") ?: parseFailed("a is null") MangaTag( title = a.ownText(), key = a.attr("href").removeSuffix("/"), diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Jsoup.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Jsoup.kt index a611d7639..d75c5bfd0 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Jsoup.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Jsoup.kt @@ -15,8 +15,18 @@ val Element.host: String? } } +/** + * Return an attribute value or null if it is missing or empty + * @see [Element.attr] which returns empty string instead of null + */ fun Element.attrOrNull(attributeKey: String) = attr(attributeKey).takeUnless { it.isEmpty() } +/** + * Return an attribute value as relative url or null if it is missing or empty + * @see attrAsRelativeUrl + * @see attrAsAbsoluteUrlOrNull + * @see attrAsAbsoluteUrl + */ fun Element.attrAsRelativeUrlOrNull(attributeKey: String): String? { val attr = attr(attributeKey).trim() if (attr.isEmpty()) { @@ -29,12 +39,25 @@ fun Element.attrAsRelativeUrlOrNull(attributeKey: String): String? { return attr.substringAfter(host) } +/** + * Return an attribute value as relative url or throw an exception if it is missing or empty + * @throws IllegalArgumentException if attribute value is missing or empty + * @see attrAsRelativeUrlOrNull + * @see attrAsAbsoluteUrlOrNull + * @see attrAsAbsoluteUrl + */ fun Element.attrAsRelativeUrl(attributeKey: String): String { return requireNotNull(attrAsRelativeUrlOrNull(attributeKey)) { "Cannot get relative url for $attributeKey: \"${attr(attributeKey)}\"" } } +/** + * Return an attribute value as absolute url or null if it is missing or empty + * @see attrAsAbsoluteUrl + * @see attrAsRelativeUrl + * @see attrAsRelativeUrlOrNull + */ fun Element.attrAsAbsoluteUrlOrNull(attributeKey: String): String? { val attr = attr(attributeKey).trim() if (attr.isEmpty()) { @@ -43,12 +66,22 @@ fun Element.attrAsAbsoluteUrlOrNull(attributeKey: String): String? { return (baseUri().toHttpUrlOrNull()?.newBuilder(attr) ?: return null).toString() } +/** + * Return an attribute value as absolute url or throw an exception if it is missing or empty + * @throws IllegalArgumentException if attribute value is missing or empty + * @see attrAsAbsoluteUrlOrNull + * @see attrAsRelativeUrl + * @see attrAsRelativeUrlOrNull + */ fun Element.attrAsAbsoluteUrl(attributeKey: String): String { return requireNotNull(attrAsAbsoluteUrlOrNull(attributeKey)) { "Cannot get absolute url for $attributeKey: \"${attr(attributeKey)}\"" } } +/** + * Return css value from `style` attribute or null if it is missing + */ fun Element.styleValueOrNull(property: String): String? { val regex = Regex("${Regex.escape(property)}\\s*:\\s*[^;]+") val css = attr("style").find(regex) ?: return null diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Parse.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Parse.kt index 61196bd77..0729b1bec 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Parse.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Parse.kt @@ -3,6 +3,7 @@ package org.koitharu.kotatsu.parsers.util import okhttp3.Response +import okhttp3.ResponseBody import okhttp3.internal.closeQuietly import org.json.JSONArray import org.json.JSONObject @@ -14,37 +15,39 @@ import org.jsoup.nodes.Node import org.jsoup.select.Elements import java.text.DateFormat -fun Response.parseHtml(): Document { - try { - (body?.byteStream() ?: throw NullPointerException("Response body is null")).use { stream -> - val charset = body!!.contentType()?.charset()?.name() - return Jsoup.parse( - stream, - charset, - request.url.toString(), - ) - } - } finally { - closeQuietly() - } +/** + * Parse [Response] body as html document using Jsoup + * @see [parseJson] + * @see [parseJsonArray] + */ +fun Response.parseHtml(): Document = try { + val body = requireBody() + val charset = body.contentType()?.charset()?.name() + Jsoup.parse(body.byteStream(), charset, request.url.toString()) +} finally { + closeQuietly() } -fun Response.parseJson(): JSONObject { - try { - val string = body?.string() ?: throw NullPointerException("Response body is null") - return JSONObject(string) - } finally { - closeQuietly() - } +/** + * Parse [Response] body as [JSONObject] + * @see [parseJsonArray] + * @see [parseHtml] + */ +fun Response.parseJson(): JSONObject = try { + JSONObject(requireBody().string()) +} finally { + closeQuietly() } -fun Response.parseJsonArray(): JSONArray { - try { - val string = body?.string() ?: throw NullPointerException("Response body is null") - return JSONArray(string) - } finally { - closeQuietly() - } +/** + * Parse [Response] body as [JSONArray] + * @see [parseJson] + * @see [parseHtml] + */ +fun Response.parseJsonArray(): JSONArray = try { + JSONArray(requireBody().string()) +} finally { + closeQuietly() } @Deprecated( @@ -90,6 +93,10 @@ fun String.inContextOf(node: Node): String { } } +/** + * Convert url to relative if it is on [domain] + * @return an url relative to the [domain] or absolute, if domain is mismatching + */ fun String.toRelativeUrl(domain: String): String { if (isEmpty() || startsWith("/")) { return this @@ -97,6 +104,10 @@ fun String.toRelativeUrl(domain: String): String { return replace(Regex("^[^/]{2,6}://${Regex.escape(domain)}+/", RegexOption.IGNORE_CASE), "/") } +/** + * Convert url to absolute with specified [domain] + * @return an absolute url with [domain] if this is relative + */ fun String.toAbsoluteUrl(domain: String): String = when { this.startsWith("//") -> "https:$this" this.startsWith("/") -> "https://$domain$this" @@ -135,4 +146,6 @@ fun DateFormat.tryParse(str: String?): Long = if (str.isNullOrEmpty()) { runCatching { parse(str)?.time ?: 0L }.getOrDefault(0L) -} \ No newline at end of file +} + +private fun Response.requireBody(): ResponseBody = requireNotNull(body) { "Response body is null" } \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/String.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/String.kt index d4e538e30..6de74f3fd 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/String.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/String.kt @@ -9,15 +9,6 @@ import java.security.MessageDigest import java.util.* import kotlin.math.min -fun String.longHashCode(): Long { - var h = 1125899906842597L - val len: Int = this.length - for (i in 0 until len) { - h = 31 * h + this[i].code - } - return h -} - fun String.removeSurrounding(vararg chars: Char): String { if (isEmpty()) { return this