Add documentation comments

pull/19/head
Koitharu 4 years ago
parent b6e2182069
commit 4a2f899a64
No known key found for this signature in database
GPG Key ID: 8E861F8CE6E7CE27

@ -18,6 +18,11 @@ abstract class MangaLoaderContext {
abstract val cookieJar: CookieJar
/**
* Do a GET http request to specific url
* @param url
* @param headers an additional headers for request, may be null
*/
suspend fun httpGet(url: String, headers: Headers? = null): Response {
val request = Request.Builder()
.get()
@ -28,6 +33,11 @@ abstract class MangaLoaderContext {
return httpClient.newCall(request.build()).await().ensureSuccess()
}
/**
* Do a HEAD http request to specific url
* @param url
* @param headers an additional headers for request, may be null
*/
suspend fun httpHead(url: String, headers: Headers? = null): Response {
val request = Request.Builder()
.head()
@ -38,6 +48,12 @@ abstract class MangaLoaderContext {
return httpClient.newCall(request.build()).await().ensureSuccess()
}
/**
* Do a POST http request to specific url with `multipart/form-data` payload
* @param url
* @param form payload as key=>value map
* @param headers an additional headers for request, may be null
*/
suspend fun httpPost(
url: String,
form: Map<String, String>,
@ -56,6 +72,12 @@ abstract class MangaLoaderContext {
return httpClient.newCall(request.build()).await().ensureSuccess()
}
/**
* Do a POST http request to specific url with `multipart/form-data` payload
* @param url
* @param payload payload as `key=value` string with `&` separator
* @param headers an additional headers for request, may be null
*/
suspend fun httpPost(
url: String,
payload: String,
@ -79,6 +101,11 @@ abstract class MangaLoaderContext {
return httpClient.newCall(request.build()).await().ensureSuccess()
}
/**
* Do a GraphQL request to specific url
* @param endpoint an url
* @param query GraphQL request payload
*/
suspend fun graphQLQuery(endpoint: String, query: String): JSONObject {
val body = JSONObject()
body.put("operationName", null as Any?)
@ -104,6 +131,11 @@ abstract class MangaLoaderContext {
open fun getPreferredLocales(): List<Locale> = listOf(Locale.getDefault())
/**
* Execute JavaScript code and return result
* @param script JavaScript source code
* @return execution result as string, may be null
*/
abstract suspend fun evaluateJs(script: String): String?
abstract fun getConfig(source: MangaSource): MangaSourceConfig

@ -1,20 +1,41 @@
package org.koitharu.kotatsu.parsers
import androidx.annotation.CallSuper
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.exception.ParseException
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl
import java.util.*
abstract class MangaParser(val source: MangaSource) {
protected abstract val context: MangaLoaderContext
/**
* Supported [SortOrder] variants. Must not be empty.
*
* For better performance use [EnumSet] for more than one item.
*/
abstract val sortOrders: Set<SortOrder>
val config by lazy { context.getConfig(source) }
/**
* Provide default domain and available alternatives, if any.
*
* Never hardcode domain in requests, use [getDomain] instead.
*/
protected abstract val configKeyDomain: ConfigKey.Domain
/**
* Parse list of manga by specified criteria
*
* @param offset starting from 0 and used for pagination.
* Note than passed value may not be divisible by internal page size, so you should adjust it manually.
* @param query search query, may be null or empty if no search needed
* @param tags genres for filtering, values from [getTags] and [Manga.tags]. May be null or empty
* @param sortOrder one of [sortOrders] or null for default value
*/
abstract suspend fun getList(
offset: Int,
query: String? = null,
@ -22,16 +43,35 @@ abstract class MangaParser(val source: MangaSource) {
sortOrder: SortOrder? = null,
): List<Manga>
/**
* Parse details for [Manga]: chapters list, description, large cover, etc.
* Must return the same manga, may change any fields excepts id, url and source
* @see Manga.copy
*/
abstract suspend fun getDetails(manga: Manga): Manga
/**
* Parse pages list for specified chapter.
* @see MangaPage for details
*/
abstract suspend fun getPages(chapter: MangaChapter): List<MangaPage>
/**
* Fetch direct link to the page image.
*/
open suspend fun getPageUrl(page: MangaPage): String = page.url.withDomain()
/**
* Fetch available tags (genres) for source
*/
abstract suspend fun getTags(): Set<MangaTag>
/**
* Returns direct link to the website favicon
*/
open fun getFaviconUrl() = "https://${getDomain()}/favicon.ico"
@CallSuper
open fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) {
keys.add(configKeyDomain)
}
@ -42,6 +82,13 @@ abstract class MangaParser(val source: MangaSource) {
return config[configKeyDomain]
}
/**
* Create a unique id for [Manga]/[MangaChapter]/[MangaPage].
* @param url must be relative url, without a domain
* @see [Manga.id]
* @see [MangaChapter.id]
* @see [MangaPage.id]
*/
protected fun generateUid(url: String): Long {
var h = 1125899906842597L
source.name.forEach { c ->
@ -53,6 +100,13 @@ abstract class MangaParser(val source: MangaSource) {
return h
}
/**
* Create a unique id for [Manga]/[MangaChapter]/[MangaPage].
* @param id an internal identifier
* @see [Manga.id]
* @see [MangaChapter.id]
* @see [MangaPage.id]
*/
protected fun generateUid(id: Long): Long {
var h = 1125899906842597L
source.name.forEach { c ->
@ -62,6 +116,9 @@ abstract class MangaParser(val source: MangaSource) {
return h
}
/**
* Convert relative url to an absolute using [getDomain]
*/
protected fun String.withDomain(subdomain: String? = null): String {
var domain = getDomain()
if (subdomain != null) {
@ -70,7 +127,8 @@ abstract class MangaParser(val source: MangaSource) {
return toAbsoluteUrl(domain)
}
protected fun parseFailed(message: String? = null): Nothing {
throw ParseException(message)
@Suppress("NOTHING_TO_INLINE")
protected inline fun parseFailed(message: String? = null): Nothing {
throw ParseException(message, null)
}
}

@ -1,10 +1,30 @@
package org.koitharu.kotatsu.parsers
import org.koitharu.kotatsu.parsers.exception.AuthRequiredException
import org.koitharu.kotatsu.parsers.exception.ParseException
/**
* Implement this in your parser for authorization support
*/
interface MangaParserAuthProvider {
/**
* Return link to the login page, which will be opened in browser.
* Must be an absolute url
*/
val authUrl: String
/**
* Quick check if user is logged in.
* In most case you should check for cookies in [MangaLoaderContext.cookieJar].
*/
val isAuthorized: Boolean
/**
* Fetch and return current user`s name or login.
* Normally should not be called if [isAuthorized] returns false
* @throws [AuthRequiredException] if user is not logged in or authorization is expired
* @throws [ParseException] on parsing error
*/
suspend fun getUsername(): String
}

@ -1,8 +1,20 @@
package org.koitharu.kotatsu.parsers
/**
* Annotate each [MangaParser] implementation with this annotation, used by codegen
*/
@Target(AnnotationTarget.CLASS)
annotation class MangaSourceParser(
/**
* Name of manga source. Used as an Enum value, must be UPPER_CASE and unique.
*/
val name: String,
/**
* User-friendly title of manga source. In most case equals the website name.
*/
val title: String,
/**
* Language code (for example "en" or "ru") or blank if parser provide manga on different languages.
*/
val locale: String = "",
)

@ -2,6 +2,9 @@ package org.koitharu.kotatsu.parsers.exception
import org.koitharu.kotatsu.parsers.model.MangaSource
/**
* Authorization is required for access to the requested content
*/
class AuthRequiredException(
val source: MangaSource,
) : RuntimeException("Authorization required")

@ -1,6 +1,6 @@
package org.koitharu.kotatsu.parsers.exception
class ParseException(
message: String? = null,
class ParseException @JvmOverloads constructor(
message: String?,
cause: Throwable? = null,
) : RuntimeException(message, cause)

@ -1,3 +1,5 @@
@file:JvmName("Constants")
package org.koitharu.kotatsu.parsers.model
internal const val RATING_UNKNOWN = -1f
const val RATING_UNKNOWN = -1f

@ -1,23 +1,76 @@
package org.koitharu.kotatsu.parsers.model
class Manga(
/**
* Unique identifier for manga
*/
val id: Long,
/**
* Manga title, human-readable
*/
val title: String,
/**
* Alternative title (for example on other language), may be null
*/
val altTitle: String?,
val url: String, // relative url for internal use
/**
* Relative url to manga (**without** a domain) or any other uri.
* Used principally in parsers
*/
val url: String,
/**
* Absolute url to manga, must be ready to open in browser
*/
val publicUrl: String,
val rating: Float, // normalized value [0..1] or -1
/**
* Normalized manga rating, must be in range of 0..1 or [RATING_UNKNOWN] if rating s unknown
* @see hasRating
*/
val rating: Float,
/**
* Indicates that manga may contain sensitive information (18+, NSFW)
*/
val isNsfw: Boolean,
/**
* Absolute link to the cover
* @see largeCoverUrl
*/
val coverUrl: String,
/**
* Tags (genres) of the manga
*/
val tags: Set<MangaTag>,
/**
* Manga status (ongoing, finished) or null if unknown
*/
val state: MangaState?,
/**
* Author of the manga, may be null
*/
val author: String?,
/**
* Large cover url (absolute), null if is no large cover
* @see coverUrl
*/
val largeCoverUrl: String? = null,
val description: String? = null, // HTML
/**
* Manga description, may be html or null
*/
val description: String? = null,
/**
* List of chapters
*/
val chapters: List<MangaChapter>? = null,
/**
* Manga source
*/
val source: MangaSource,
) {
/**
* Return if manga has a specified rating
* @see rating
*/
val hasRating: Boolean
get() = rating in 0f..1f

@ -1,12 +1,35 @@
package org.koitharu.kotatsu.parsers.model
class MangaChapter(
/**
* An unique id of chapter
*/
val id: Long,
/**
* User-readable name of chapter
*/
val name: String,
/**
* Chapter number starting from 1
*/
val number: Int,
/**
* Relative url to chapter (**without** a domain) or any other uri.
* Used principally in parsers
*/
val url: String,
/**
* User-readable name of scanlator (releaser) or null if unknown
*/
val scanlator: String?,
/**
* Chapter upload date in milliseconds
*/
val uploadDate: Long,
/**
* User-readable name of branch.
* A branch is a group of chapters that overlap (e.g. different languages)
*/
val branch: String?,
val source: MangaSource,
) : Comparable<MangaChapter> {
@ -44,6 +67,4 @@ class MangaChapter(
result = 31 * result + source.hashCode()
return result
}
}

@ -1,9 +1,27 @@
package org.koitharu.kotatsu.parsers.model
import org.koitharu.kotatsu.parsers.MangaParser
class MangaPage(
/**
* Unique identifier for manga
*/
val id: Long,
/**
* Relative url to page (**without** a domain) or any other uri.
* Used principally in parsers.
* May contain link to image or html page.
* @see MangaParser.getPageUrl
*/
val url: String,
/**
* Absolute link to the chapter or website home page.
* Used in Referer header
*/
val referer: String,
/**
* Absolute url of the small page image if exists, null otherwise
*/
val preview: String?,
val source: MangaSource,
) {

@ -1,7 +1,16 @@
package org.koitharu.kotatsu.parsers.model
import org.koitharu.kotatsu.parsers.MangaParser
class MangaTag(
/**
* User-readable tag title, should be in Title case
*/
val title: String,
/**
* Identifier of a tag, must be unique among the source.
* @see MangaParser.getList
*/
val key: String,
val source: MangaSource,
) {

@ -1,6 +1,6 @@
package org.koitharu.kotatsu.parsers.model
class WordSet(private vararg val words: String) {
internal class WordSet(private vararg val words: String) {
fun anyWordIn(dateString: String): Boolean = words.any {
dateString.contains(it, ignoreCase = true)

@ -125,7 +125,7 @@ internal abstract class ChanParser(source: MangaSource) : MangaParser(source) {
)
}
}
throw ParseException("Pages list not found at ${chapter.url}")
parseFailed("Pages list not found at ${chapter.url}")
}
override suspend fun getTags(): Set<MangaTag> {

@ -50,8 +50,7 @@ internal class DesuMeParser(override val context: MangaLoaderContext) : MangaPar
append(query)
}
}
val json = context.httpGet(url).parseJson().getJSONArray("response")
?: throw ParseException("Invalid response")
val json = context.httpGet(url).parseJson().getJSONArray("response") ?: parseFailed("Invalid response")
val total = json.length()
val list = ArrayList<Manga>(total)
for (i in 0 until total) {

@ -222,7 +222,7 @@ internal class ExHentaiParser(
?: if (doc.getElementById("userlinksguest") != null) {
throw AuthRequiredException(source)
} else {
throw ParseException()
throw ParseException(null)
}
return username
}

@ -66,7 +66,7 @@ internal abstract class GroupleParser(source: MangaSource, userAgent: String) :
else -> advancedSearch(domain, tags)
}.parseHtml().body()
val root = (doc.getElementById("mangaBox") ?: doc.getElementById("mangaResults"))
?: throw ParseException("Cannot find root")
?: parseFailed("Cannot find root")
val tiles = root.selectFirst("div.tiles.row") ?: if (
root.select(".alert").any { it.ownText() == NOTHING_FOUND }
) {
@ -128,7 +128,7 @@ internal abstract class GroupleParser(source: MangaSource, userAgent: String) :
override suspend fun getDetails(manga: Manga): Manga {
val doc = context.httpGet(manga.url.withDomain(), headers).parseHtml()
val root = doc.body().getElementById("mangaBox")?.selectFirst("div.leftContent")
?: throw ParseException("Cannot find root")
?: parseFailed("Cannot find root")
val dateFormat = SimpleDateFormat("dd.MM.yy", Locale.US)
val coverImg = root.selectFirst("div.subject-cover")?.selectFirst("img")
return manga.copy(
@ -203,7 +203,7 @@ internal abstract class GroupleParser(source: MangaSource, userAgent: String) :
)
}
}
throw ParseException("Pages list not found at ${chapter.url}")
parseFailed("Pages list not found at ${chapter.url}")
}
override suspend fun getPageUrl(page: MangaPage): String {

@ -4,7 +4,6 @@ import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.exception.ParseException
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import java.text.SimpleDateFormat
@ -103,8 +102,7 @@ class MangaInUaParser(override val context: MangaLoaderContext) : MangaParser(Ma
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val fullUrl = chapter.url.withDomain()
val doc = context.httpGet(fullUrl).parseHtml()
val root = doc.body().getElementById("comics")
?: throw ParseException("Root not found")
val root = doc.body().getElementById("comics") ?: parseFailed("Root not found")
return root.select("ul.xfieldimagegallery").map { ul ->
val img = ul.selectFirst("img") ?: parseFailed("Page image not found")
val url = img.attrAsAbsoluteUrl("data-src")
@ -123,7 +121,7 @@ class MangaInUaParser(override val context: MangaLoaderContext) : MangaParser(Ma
val doc = context.httpGet("https://$domain/mangas").parseHtml()
val root = doc.body().getElementById("menu_1")?.selectFirst("div.menu__wrapper") ?: parseFailed("Cannot find root")
return root.select("li").mapToSet { li ->
val a = li.selectFirst("a") ?: throw ParseException("a is null")
val a = li.selectFirst("a") ?: parseFailed("a is null")
MangaTag(
title = a.ownText(),
key = a.attr("href").removeSuffix("/"),

@ -15,8 +15,18 @@ val Element.host: String?
}
}
/**
* Return an attribute value or null if it is missing or empty
* @see [Element.attr] which returns empty string instead of null
*/
fun Element.attrOrNull(attributeKey: String) = attr(attributeKey).takeUnless { it.isEmpty() }
/**
* Return an attribute value as relative url or null if it is missing or empty
* @see attrAsRelativeUrl
* @see attrAsAbsoluteUrlOrNull
* @see attrAsAbsoluteUrl
*/
fun Element.attrAsRelativeUrlOrNull(attributeKey: String): String? {
val attr = attr(attributeKey).trim()
if (attr.isEmpty()) {
@ -29,12 +39,25 @@ fun Element.attrAsRelativeUrlOrNull(attributeKey: String): String? {
return attr.substringAfter(host)
}
/**
* Return an attribute value as relative url or throw an exception if it is missing or empty
* @throws IllegalArgumentException if attribute value is missing or empty
* @see attrAsRelativeUrlOrNull
* @see attrAsAbsoluteUrlOrNull
* @see attrAsAbsoluteUrl
*/
fun Element.attrAsRelativeUrl(attributeKey: String): String {
return requireNotNull(attrAsRelativeUrlOrNull(attributeKey)) {
"Cannot get relative url for $attributeKey: \"${attr(attributeKey)}\""
}
}
/**
* Return an attribute value as absolute url or null if it is missing or empty
* @see attrAsAbsoluteUrl
* @see attrAsRelativeUrl
* @see attrAsRelativeUrlOrNull
*/
fun Element.attrAsAbsoluteUrlOrNull(attributeKey: String): String? {
val attr = attr(attributeKey).trim()
if (attr.isEmpty()) {
@ -43,12 +66,22 @@ fun Element.attrAsAbsoluteUrlOrNull(attributeKey: String): String? {
return (baseUri().toHttpUrlOrNull()?.newBuilder(attr) ?: return null).toString()
}
/**
* Return an attribute value as absolute url or throw an exception if it is missing or empty
* @throws IllegalArgumentException if attribute value is missing or empty
* @see attrAsAbsoluteUrlOrNull
* @see attrAsRelativeUrl
* @see attrAsRelativeUrlOrNull
*/
fun Element.attrAsAbsoluteUrl(attributeKey: String): String {
return requireNotNull(attrAsAbsoluteUrlOrNull(attributeKey)) {
"Cannot get absolute url for $attributeKey: \"${attr(attributeKey)}\""
}
}
/**
* Return css value from `style` attribute or null if it is missing
*/
fun Element.styleValueOrNull(property: String): String? {
val regex = Regex("${Regex.escape(property)}\\s*:\\s*[^;]+")
val css = attr("style").find(regex) ?: return null

@ -3,6 +3,7 @@
package org.koitharu.kotatsu.parsers.util
import okhttp3.Response
import okhttp3.ResponseBody
import okhttp3.internal.closeQuietly
import org.json.JSONArray
import org.json.JSONObject
@ -14,37 +15,39 @@ import org.jsoup.nodes.Node
import org.jsoup.select.Elements
import java.text.DateFormat
fun Response.parseHtml(): Document {
try {
(body?.byteStream() ?: throw NullPointerException("Response body is null")).use { stream ->
val charset = body!!.contentType()?.charset()?.name()
return Jsoup.parse(
stream,
charset,
request.url.toString(),
)
}
} finally {
closeQuietly()
}
/**
* Parse [Response] body as html document using Jsoup
* @see [parseJson]
* @see [parseJsonArray]
*/
fun Response.parseHtml(): Document = try {
val body = requireBody()
val charset = body.contentType()?.charset()?.name()
Jsoup.parse(body.byteStream(), charset, request.url.toString())
} finally {
closeQuietly()
}
fun Response.parseJson(): JSONObject {
try {
val string = body?.string() ?: throw NullPointerException("Response body is null")
return JSONObject(string)
} finally {
closeQuietly()
}
/**
* Parse [Response] body as [JSONObject]
* @see [parseJsonArray]
* @see [parseHtml]
*/
fun Response.parseJson(): JSONObject = try {
JSONObject(requireBody().string())
} finally {
closeQuietly()
}
fun Response.parseJsonArray(): JSONArray {
try {
val string = body?.string() ?: throw NullPointerException("Response body is null")
return JSONArray(string)
} finally {
closeQuietly()
}
/**
* Parse [Response] body as [JSONArray]
* @see [parseJson]
* @see [parseHtml]
*/
fun Response.parseJsonArray(): JSONArray = try {
JSONArray(requireBody().string())
} finally {
closeQuietly()
}
@Deprecated(
@ -90,6 +93,10 @@ fun String.inContextOf(node: Node): String {
}
}
/**
* Convert url to relative if it is on [domain]
* @return an url relative to the [domain] or absolute, if domain is mismatching
*/
fun String.toRelativeUrl(domain: String): String {
if (isEmpty() || startsWith("/")) {
return this
@ -97,6 +104,10 @@ fun String.toRelativeUrl(domain: String): String {
return replace(Regex("^[^/]{2,6}://${Regex.escape(domain)}+/", RegexOption.IGNORE_CASE), "/")
}
/**
* Convert url to absolute with specified [domain]
* @return an absolute url with [domain] if this is relative
*/
fun String.toAbsoluteUrl(domain: String): String = when {
this.startsWith("//") -> "https:$this"
this.startsWith("/") -> "https://$domain$this"
@ -135,4 +146,6 @@ fun DateFormat.tryParse(str: String?): Long = if (str.isNullOrEmpty()) {
runCatching {
parse(str)?.time ?: 0L
}.getOrDefault(0L)
}
}
private fun Response.requireBody(): ResponseBody = requireNotNull(body) { "Response body is null" }

@ -9,15 +9,6 @@ import java.security.MessageDigest
import java.util.*
import kotlin.math.min
fun String.longHashCode(): Long {
var h = 1125899906842597L
val len: Int = this.length
for (i in 0 until len) {
h = 31 * h + this[i].code
}
return h
}
fun String.removeSurrounding(vararg chars: Char): String {
if (isEmpty()) {
return this

Loading…
Cancel
Save