From 93f5f70d7969c3a26a5e29b8674d8bfcd7ad8380 Mon Sep 17 00:00:00 2001 From: Koitharu Date: Sun, 19 Mar 2023 09:28:38 +0200 Subject: [PATCH] Remove referrer field from page model --- .../kotatsu/parsers/site/AnibelParser.kt | 343 ++++++----- .../kotatsu/parsers/site/BatoToParser.kt | 546 +++++++++--------- .../kotatsu/parsers/site/BlogTruyenParser.kt | 466 ++++++++------- .../kotatsu/parsers/site/CloneMangaParser.kt | 1 - .../kotatsu/parsers/site/ComickFunParser.kt | 357 ++++++------ .../kotatsu/parsers/site/DesuMeParser.kt | 1 - .../kotatsu/parsers/site/ExHentaiParser.kt | 529 +++++++++-------- .../kotatsu/parsers/site/MangaDexParser.kt | 464 ++++++++------- .../kotatsu/parsers/site/MangaInUaParser.kt | 257 ++++----- .../kotatsu/parsers/site/MangaTownParser.kt | 379 ++++++------ .../kotatsu/parsers/site/Manhwa18Parser.kt | 1 - .../kotatsu/parsers/site/NHentaiParser.kt | 317 +++++----- .../kotatsu/parsers/site/NetTruyenParser.kt | 409 +++++++------ .../parsers/site/NicovideoSeigaParser.kt | 315 +++++----- .../kotatsu/parsers/site/NineMangaParser.kt | 431 +++++++------- .../kotatsu/parsers/site/NudeMoonParser.kt | 413 +++++++------ .../kotatsu/parsers/site/RemangaParser.kt | 473 ++++++++------- .../parsers/site/TruyentranhLHParser.kt | 1 - .../kotatsu/parsers/site/UnionMangasParser.kt | 1 - .../parsers/site/grouple/GroupleParser.kt | 1 - .../parsers/site/madara/Madara5Parser.kt | 5 +- .../parsers/site/madara/MadaraParser.kt | 1 - .../kotatsu/parsers/site/madara/NeatManga.kt | 1 - .../site/mangareader/MangaReaderParser.kt | 1 - .../parsers/site/multichan/ChanParser.kt | 311 +++++----- .../parsers/site/rulib/MangaLibParser.kt | 525 +++++++++-------- 26 files changed, 3264 insertions(+), 3285 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/AnibelParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/AnibelParser.kt index dddfc47d..a50fc5b5 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/AnibelParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/AnibelParser.kt @@ -20,32 +20,32 @@ import java.util.* @MangaSourceParser("ANIBEL", "Anibel", "be") internal class AnibelParser(context: MangaLoaderContext) : MangaParser(context, MangaSource.ANIBEL) { - override val configKeyDomain = ConfigKey.Domain("anibel.net", null) + override val configKeyDomain = ConfigKey.Domain("anibel.net", null) - override val sortOrders: Set = EnumSet.of( - SortOrder.NEWEST, - ) + override val sortOrders: Set = EnumSet.of( + SortOrder.NEWEST, + ) - override suspend fun getList( - offset: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - if (!query.isNullOrEmpty()) { - return if (offset == 0) { - search(query) - } else { - emptyList() - } - } - val filters = tags?.takeUnless { it.isEmpty() }?.joinToString( - separator = ",", - prefix = "genres: [", - postfix = "]", - ) { "\"${it.key}\"" }.orEmpty() - val array = apiCall( - """ + override suspend fun getList( + offset: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + if (!query.isNullOrEmpty()) { + return if (offset == 0) { + search(query) + } else { + emptyList() + } + } + val filters = tags?.takeUnless { it.isEmpty() }?.joinToString( + separator = ",", + prefix = "genres: [", + postfix = "]", + ) { "\"${it.key}\"" }.orEmpty() + val array = apiCall( + """ getMediaList(offset: $offset, limit: 20, mediaType: manga, filters: {$filters}) { docs { mediaId @@ -62,37 +62,37 @@ internal class AnibelParser(context: MangaLoaderContext) : MangaParser(context, } } """.trimIndent(), - ).getJSONObject("getMediaList").getJSONArray("docs") - return array.mapJSON { jo -> - val mediaId = jo.getString("mediaId") - val title = jo.getJSONObject("title") - val href = "${jo.getString("mediaType")}/${jo.getString("slug")}" - Manga( - id = generateUid(mediaId), - title = title.getString("be"), - coverUrl = jo.getString("poster").removePrefix("/cdn") - .toAbsoluteUrl(getDomain("cdn")) + "?width=200&height=280", - altTitle = title.getString("alt").takeUnless(String::isEmpty), - author = null, - isNsfw = false, - rating = jo.getDouble("rating").toFloat() / 10f, - url = href, - publicUrl = "https://${domain}/$href", - tags = jo.getJSONArray("genres").mapToTags(), - state = when (jo.getString("status")) { - "ongoing" -> MangaState.ONGOING - "finished" -> MangaState.FINISHED - else -> null - }, - source = source, - ) - } - } + ).getJSONObject("getMediaList").getJSONArray("docs") + return array.mapJSON { jo -> + val mediaId = jo.getString("mediaId") + val title = jo.getJSONObject("title") + val href = "${jo.getString("mediaType")}/${jo.getString("slug")}" + Manga( + id = generateUid(mediaId), + title = title.getString("be"), + coverUrl = jo.getString("poster").removePrefix("/cdn") + .toAbsoluteUrl(getDomain("cdn")) + "?width=200&height=280", + altTitle = title.getString("alt").takeUnless(String::isEmpty), + author = null, + isNsfw = false, + rating = jo.getDouble("rating").toFloat() / 10f, + url = href, + publicUrl = "https://${domain}/$href", + tags = jo.getJSONArray("genres").mapToTags(), + state = when (jo.getString("status")) { + "ongoing" -> MangaState.ONGOING + "finished" -> MangaState.FINISHED + else -> null + }, + source = source, + ) + } + } - override suspend fun getDetails(manga: Manga): Manga { - val (type, slug) = manga.url.split('/') - val details = apiCall( - """ + override suspend fun getDetails(manga: Manga): Manga { + val (type, slug) = manga.url.split('/') + val details = apiCall( + """ media(mediaType: $type, slug: "$slug") { mediaId title { @@ -108,52 +108,52 @@ internal class AnibelParser(context: MangaLoaderContext) : MangaParser(context, genres } """.trimIndent(), - ).getJSONObject("media") - val title = details.getJSONObject("title") - val poster = details.getString("poster").removePrefix("/cdn") - .toAbsoluteUrl(getDomain("cdn")) - val chapters = apiCall( - """ + ).getJSONObject("media") + val title = details.getJSONObject("title") + val poster = details.getString("poster").removePrefix("/cdn") + .toAbsoluteUrl(getDomain("cdn")) + val chapters = apiCall( + """ chapters(mediaId: "${details.getString("mediaId")}") { id chapter released } """.trimIndent(), - ).getJSONArray("chapters") - return manga.copy( - title = title.getString("be"), - altTitle = title.getString("alt"), - coverUrl = "$poster?width=200&height=280", - largeCoverUrl = poster, - description = details.getJSONObject("description").getString("be"), - rating = details.getDouble("rating").toFloat() / 10f, - tags = details.getJSONArray("genres").mapToTags(), - state = when (details.getString("status")) { - "ongoing" -> MangaState.ONGOING - "finished" -> MangaState.FINISHED - else -> null - }, - chapters = chapters.mapJSON { jo -> - val number = jo.getInt("chapter") - MangaChapter( - id = generateUid(jo.getString("id")), - name = "Глава $number", - number = number, - url = "${manga.url}/read/$number", - scanlator = null, - uploadDate = jo.getLong("released"), - branch = null, - source = source, - ) - }, - ) - } + ).getJSONArray("chapters") + return manga.copy( + title = title.getString("be"), + altTitle = title.getString("alt"), + coverUrl = "$poster?width=200&height=280", + largeCoverUrl = poster, + description = details.getJSONObject("description").getString("be"), + rating = details.getDouble("rating").toFloat() / 10f, + tags = details.getJSONArray("genres").mapToTags(), + state = when (details.getString("status")) { + "ongoing" -> MangaState.ONGOING + "finished" -> MangaState.FINISHED + else -> null + }, + chapters = chapters.mapJSON { jo -> + val number = jo.getInt("chapter") + MangaChapter( + id = generateUid(jo.getString("id")), + name = "Глава $number", + number = number, + url = "${manga.url}/read/$number", + scanlator = null, + uploadDate = jo.getLong("released"), + branch = null, + source = source, + ) + }, + ) + } - override suspend fun getPages(chapter: MangaChapter): List { - val (_, slug, _, number) = chapter.url.split('/') - val chapterJson = apiCall( - """ + override suspend fun getPages(chapter: MangaChapter): List { + val (_, slug, _, number) = chapter.url.split('/') + val chapterJson = apiCall( + """ chapter(slug: "$slug", chapter: $number) { id images { @@ -162,35 +162,34 @@ internal class AnibelParser(context: MangaLoaderContext) : MangaParser(context, } } """.trimIndent(), - ).getJSONObject("chapter") - val pages = chapterJson.getJSONArray("images") - val chapterUrl = "https://${domain}/${chapter.url}" - return pages.mapJSONIndexed { i, jo -> - MangaPage( - id = generateUid("${chapter.url}/$i"), - url = jo.getString("large"), - referer = chapterUrl, - preview = jo.getString("thumbnail"), - source = source, - ) - } - } + ).getJSONObject("chapter") + val pages = chapterJson.getJSONArray("images") + val chapterUrl = "https://${domain}/${chapter.url}" + return pages.mapJSONIndexed { i, jo -> + MangaPage( + id = generateUid("${chapter.url}/$i"), + url = jo.getString("large"), + preview = jo.getString("thumbnail"), + source = source, + ) + } + } - override suspend fun getTags(): Set { - val json = apiCall( - """ + override suspend fun getTags(): Set { + val json = apiCall( + """ getFilters(mediaType: manga) { genres } """.trimIndent(), - ) - val array = json.getJSONObject("getFilters").getJSONArray("genres") - return array.mapToTags() - } + ) + val array = json.getJSONObject("getFilters").getJSONArray("genres") + return array.mapToTags() + } - private suspend fun search(query: String): List { - val json = apiCall( - """ + private suspend fun search(query: String): List { + val json = apiCall( + """ search(query: "$query", limit: 40) { id title { @@ -202,65 +201,65 @@ internal class AnibelParser(context: MangaLoaderContext) : MangaParser(context, type } """.trimIndent(), - ) - val array = json.getJSONArray("search") - return array.mapJSON { jo -> - val mediaId = jo.getString("id") - val title = jo.getJSONObject("title") - val href = "${jo.getString("type").lowercase()}/${jo.getString("url")}" - Manga( - id = generateUid(mediaId), - title = title.getString("be"), - coverUrl = jo.getString("poster").removePrefix("/cdn") - .toAbsoluteUrl(getDomain("cdn")) + "?width=200&height=280", - altTitle = title.getString("en").takeUnless(String::isEmpty), - author = null, - isNsfw = false, - rating = RATING_UNKNOWN, - url = href, - publicUrl = "https://${domain}/$href", - tags = emptySet(), - state = null, - source = source, - ) - } - } + ) + val array = json.getJSONArray("search") + return array.mapJSON { jo -> + val mediaId = jo.getString("id") + val title = jo.getJSONObject("title") + val href = "${jo.getString("type").lowercase()}/${jo.getString("url")}" + Manga( + id = generateUid(mediaId), + title = title.getString("be"), + coverUrl = jo.getString("poster").removePrefix("/cdn") + .toAbsoluteUrl(getDomain("cdn")) + "?width=200&height=280", + altTitle = title.getString("en").takeUnless(String::isEmpty), + author = null, + isNsfw = false, + rating = RATING_UNKNOWN, + url = href, + publicUrl = "https://${domain}/$href", + tags = emptySet(), + state = null, + source = source, + ) + } + } - private suspend fun apiCall(request: String): JSONObject { - return webClient.graphQLQuery("https://api.${domain}/graphql", request) - .getJSONObject("data") - } + private suspend fun apiCall(request: String): JSONObject { + return webClient.graphQLQuery("https://api.${domain}/graphql", request) + .getJSONObject("data") + } - private fun JSONArray.mapToTags(): Set { + private fun JSONArray.mapToTags(): Set { - fun toTitle(slug: String): String { - val builder = StringBuilder(slug) - var capitalize = true - for ((i, c) in builder.withIndex()) { - when { - c == '-' -> { - builder.setCharAt(i, ' ') - } + fun toTitle(slug: String): String { + val builder = StringBuilder(slug) + var capitalize = true + for ((i, c) in builder.withIndex()) { + when { + c == '-' -> { + builder.setCharAt(i, ' ') + } - capitalize -> { - builder.setCharAt(i, c.uppercaseChar()) - capitalize = false - } - } - } - return builder.toString() - } + capitalize -> { + builder.setCharAt(i, c.uppercaseChar()) + capitalize = false + } + } + } + return builder.toString() + } - val result = ArraySet(length()) - stringIterator().forEach { - result.add( - MangaTag( - title = toTitle(it), - key = it, - source = source, - ), - ) - } - return result - } + val result = ArraySet(length()) + stringIterator().forEach { + result.add( + MangaTag( + title = toTitle(it), + key = it, + source = source, + ), + ) + } + return result + } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BatoToParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BatoToParser.kt index 0f5e3603..127c455f 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BatoToParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BatoToParser.kt @@ -20,290 +20,298 @@ import javax.crypto.spec.SecretKeySpec @MangaSourceParser("BATOTO", "Bato.To") internal class BatoToParser(context: MangaLoaderContext) : PagedMangaParser( - context = context, - source = MangaSource.BATOTO, - pageSize = 60, - searchPageSize = 20, + context = context, + source = MangaSource.BATOTO, + pageSize = 60, + searchPageSize = 20, ) { - override val sortOrders: Set = EnumSet.of( - SortOrder.NEWEST, - SortOrder.UPDATED, - SortOrder.POPULARITY, - SortOrder.ALPHABETICAL, - ) + override val sortOrders: Set = EnumSet.of( + SortOrder.NEWEST, + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.ALPHABETICAL, + ) - override val configKeyDomain = ConfigKey.Domain( - "bato.to", - arrayOf("bato.to", "mto.to", "hto.to", "mangatoto.com", "battwo.com", "batotwo.com", "comiko.net", "batotoo.com"), - ) + override val configKeyDomain = ConfigKey.Domain( + "bato.to", + arrayOf( + "bato.to", + "mto.to", + "hto.to", + "mangatoto.com", + "battwo.com", + "batotwo.com", + "comiko.net", + "batotoo.com", + ), + ) - override suspend fun getListPage( - page: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - if (!query.isNullOrEmpty()) { - return search(page, query) - } - @Suppress("NON_EXHAUSTIVE_WHEN_STATEMENT") - val url = buildString { - append("https://") - append(domain) - append("/browse?sort=") - when (sortOrder) { - SortOrder.UPDATED, - -> append("update.za") + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + if (!query.isNullOrEmpty()) { + return search(page, query) + } + @Suppress("NON_EXHAUSTIVE_WHEN_STATEMENT") + val url = buildString { + append("https://") + append(domain) + append("/browse?sort=") + when (sortOrder) { + SortOrder.UPDATED, + -> append("update.za") - SortOrder.POPULARITY -> append("views_a.za") - SortOrder.NEWEST -> append("create.za") - SortOrder.ALPHABETICAL -> append("title.az") - } - if (!tags.isNullOrEmpty()) { - append("&genres=") - appendAll(tags, ",") { it.key } - } - append("&page=") - append(page) - } - return parseList(url, page) - } + SortOrder.POPULARITY -> append("views_a.za") + SortOrder.NEWEST -> append("create.za") + SortOrder.ALPHABETICAL -> append("title.az") + } + if (!tags.isNullOrEmpty()) { + append("&genres=") + appendAll(tags, ",") { it.key } + } + append("&page=") + append(page) + } + return parseList(url, page) + } - override suspend fun getDetails(manga: Manga): Manga { - val root = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - .requireElementById("mainer") - val details = root.selectFirstOrThrow(".detail-set") - val attrs = details.selectFirst(".attr-main")?.select(".attr-item")?.associate { - it.child(0).text().trim() to it.child(1) - }.orEmpty() - return manga.copy( - title = root.selectFirst("h3.item-title")?.text() ?: manga.title, - isNsfw = !root.selectFirst("alert")?.getElementsContainingOwnText("NSFW").isNullOrEmpty(), - largeCoverUrl = details.selectFirst("img[src]")?.absUrl("src"), - description = details.getElementById("limit-height-body-summary") - ?.selectFirst(".limit-html") - ?.html(), - tags = manga.tags + attrs["Genres:"]?.parseTags().orEmpty(), - state = when (attrs["Release status:"]?.text()) { - "Ongoing" -> MangaState.ONGOING - "Completed" -> MangaState.FINISHED - else -> manga.state - }, - author = attrs["Authors:"]?.text()?.trim() ?: manga.author, - chapters = root.selectFirst(".episode-list") - ?.selectFirst(".main") - ?.children() - ?.reversed() - ?.mapChapters { i, div -> - div.parseChapter(i) - }.orEmpty(), - ) - } + override suspend fun getDetails(manga: Manga): Manga { + val root = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + .requireElementById("mainer") + val details = root.selectFirstOrThrow(".detail-set") + val attrs = details.selectFirst(".attr-main")?.select(".attr-item")?.associate { + it.child(0).text().trim() to it.child(1) + }.orEmpty() + return manga.copy( + title = root.selectFirst("h3.item-title")?.text() ?: manga.title, + isNsfw = !root.selectFirst("alert")?.getElementsContainingOwnText("NSFW").isNullOrEmpty(), + largeCoverUrl = details.selectFirst("img[src]")?.absUrl("src"), + description = details.getElementById("limit-height-body-summary") + ?.selectFirst(".limit-html") + ?.html(), + tags = manga.tags + attrs["Genres:"]?.parseTags().orEmpty(), + state = when (attrs["Release status:"]?.text()) { + "Ongoing" -> MangaState.ONGOING + "Completed" -> MangaState.FINISHED + else -> manga.state + }, + author = attrs["Authors:"]?.text()?.trim() ?: manga.author, + chapters = root.selectFirst(".episode-list") + ?.selectFirst(".main") + ?.children() + ?.reversed() + ?.mapChapters { i, div -> + div.parseChapter(i) + }.orEmpty(), + ) + } - override suspend fun getPages(chapter: MangaChapter): List { - val fullUrl = chapter.url.toAbsoluteUrl(domain) - val scripts = webClient.httpGet(fullUrl).parseHtml().select("script") - for (script in scripts) { - val scriptSrc = script.html() - val p = scriptSrc.indexOf("const imgHttpLis =") - if (p == -1) continue - val start = scriptSrc.indexOf('[', p) - val end = scriptSrc.indexOf(';', start) - if (start == -1 || end == -1) { - continue - } - val images = JSONArray(scriptSrc.substring(start, end)) - val batoPass = scriptSrc.substringBetweenFirst("batoPass =", ";")?.trim(' ', '"', '\n') - ?: script.parseFailed("Cannot find batoPass") - val batoWord = scriptSrc.substringBetweenFirst("batoWord =", ";")?.trim(' ', '"', '\n') - ?: script.parseFailed("Cannot find batoWord") - val password = context.evaluateJs(batoPass)?.removeSurrounding('"') - ?: script.parseFailed("Cannot evaluate batoPass") - val args = JSONArray(decryptAES(batoWord, password)) - val result = ArrayList(images.length()) - repeat(images.length()) { i -> - val url = images.getString(i) - result += MangaPage( - id = generateUid(url), - url = url + "?" + args.getString(i), - referer = fullUrl, - preview = null, - source = source, - ) - } - return result - } - throw ParseException("Cannot find images list", fullUrl) - } + override suspend fun getPages(chapter: MangaChapter): List { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val scripts = webClient.httpGet(fullUrl).parseHtml().select("script") + for (script in scripts) { + val scriptSrc = script.html() + val p = scriptSrc.indexOf("const imgHttpLis =") + if (p == -1) continue + val start = scriptSrc.indexOf('[', p) + val end = scriptSrc.indexOf(';', start) + if (start == -1 || end == -1) { + continue + } + val images = JSONArray(scriptSrc.substring(start, end)) + val batoPass = scriptSrc.substringBetweenFirst("batoPass =", ";")?.trim(' ', '"', '\n') + ?: script.parseFailed("Cannot find batoPass") + val batoWord = scriptSrc.substringBetweenFirst("batoWord =", ";")?.trim(' ', '"', '\n') + ?: script.parseFailed("Cannot find batoWord") + val password = context.evaluateJs(batoPass)?.removeSurrounding('"') + ?: script.parseFailed("Cannot evaluate batoPass") + val args = JSONArray(decryptAES(batoWord, password)) + val result = ArrayList(images.length()) + repeat(images.length()) { i -> + val url = images.getString(i) + result += MangaPage( + id = generateUid(url), + url = url + "?" + args.getString(i), + preview = null, + source = source, + ) + } + return result + } + throw ParseException("Cannot find images list", fullUrl) + } - override suspend fun getTags(): Set { - val scripts = webClient.httpGet( - "https://${domain}/browse", - ).parseHtml().selectOrThrow("script") - for (script in scripts) { - val genres = script.html().substringBetweenFirst("const _genres =", ";") ?: continue - val jo = JSONObject(genres) - val result = ArraySet(jo.length()) - jo.keys().forEach { key -> - val item = jo.getJSONObject(key) - result += MangaTag( - title = item.getString("text").toTitleCase(), - key = item.getString("file"), - source = source, - ) - } - return result - } - throw ParseException("Cannot find gernes list", scripts[0].baseUri()) - } + override suspend fun getTags(): Set { + val scripts = webClient.httpGet( + "https://${domain}/browse", + ).parseHtml().selectOrThrow("script") + for (script in scripts) { + val genres = script.html().substringBetweenFirst("const _genres =", ";") ?: continue + val jo = JSONObject(genres) + val result = ArraySet(jo.length()) + jo.keys().forEach { key -> + val item = jo.getJSONObject(key) + result += MangaTag( + title = item.getString("text").toTitleCase(), + key = item.getString("file"), + source = source, + ) + } + return result + } + throw ParseException("Cannot find gernes list", scripts[0].baseUri()) + } - private suspend fun search(page: Int, query: String): List { - val url = buildString { - append("https://") - append(domain) - append("/search?word=") - append(query.replace(' ', '+')) - append("&page=") - append(page) - } - return parseList(url, page) - } + private suspend fun search(page: Int, query: String): List { + val url = buildString { + append("https://") + append(domain) + append("/search?word=") + append(query.replace(' ', '+')) + append("&page=") + append(page) + } + return parseList(url, page) + } - private fun getActivePage(body: Element): Int = body.select("nav ul.pagination > li.page-item.active") - .lastOrNull() - ?.text() - ?.toIntOrNull() ?: body.parseFailed("Cannot determine current page") + private fun getActivePage(body: Element): Int = body.select("nav ul.pagination > li.page-item.active") + .lastOrNull() + ?.text() + ?.toIntOrNull() ?: body.parseFailed("Cannot determine current page") - private suspend fun parseList(url: String, page: Int): List { - val body = webClient.httpGet(url).parseHtml().body() - if (body.selectFirst(".browse-no-matches") != null) { - return emptyList() - } - val activePage = getActivePage(body) - if (activePage != page) { - return emptyList() - } - val root = body.requireElementById("series-list") - return root.children().map { div -> - val a = div.selectFirstOrThrow("a") - val href = a.attrAsRelativeUrl("href") - val title = div.selectFirstOrThrow(".item-title").text() - Manga( - id = generateUid(href), - title = title, - altTitle = div.selectFirst(".item-alias")?.text()?.takeUnless { it == title }, - url = href, - publicUrl = a.absUrl("href"), - rating = RATING_UNKNOWN, - isNsfw = false, - coverUrl = div.selectFirst("img[src]")?.absUrl("src").orEmpty(), - largeCoverUrl = null, - description = null, - tags = div.selectFirst(".item-genre")?.parseTags().orEmpty(), - state = null, - author = null, - source = source, - ) - } - } + private suspend fun parseList(url: String, page: Int): List { + val body = webClient.httpGet(url).parseHtml().body() + if (body.selectFirst(".browse-no-matches") != null) { + return emptyList() + } + val activePage = getActivePage(body) + if (activePage != page) { + return emptyList() + } + val root = body.requireElementById("series-list") + return root.children().map { div -> + val a = div.selectFirstOrThrow("a") + val href = a.attrAsRelativeUrl("href") + val title = div.selectFirstOrThrow(".item-title").text() + Manga( + id = generateUid(href), + title = title, + altTitle = div.selectFirst(".item-alias")?.text()?.takeUnless { it == title }, + url = href, + publicUrl = a.absUrl("href"), + rating = RATING_UNKNOWN, + isNsfw = false, + coverUrl = div.selectFirst("img[src]")?.absUrl("src").orEmpty(), + largeCoverUrl = null, + description = null, + tags = div.selectFirst(".item-genre")?.parseTags().orEmpty(), + state = null, + author = null, + source = source, + ) + } + } - private fun Element.parseTags() = children().mapToSet { span -> - val text = span.ownText() - MangaTag( - title = text.toTitleCase(), - key = text.lowercase(Locale.ENGLISH).replace(' ', '_'), - source = source, - ) - } + private fun Element.parseTags() = children().mapToSet { span -> + val text = span.ownText() + MangaTag( + title = text.toTitleCase(), + key = text.lowercase(Locale.ENGLISH).replace(' ', '_'), + source = source, + ) + } - private fun Element.parseChapter(index: Int): MangaChapter? { - val a = selectFirst("a.chapt") ?: return null - val extra = selectFirst(".extra") - val href = a.attrAsRelativeUrl("href") - return MangaChapter( - id = generateUid(href), - name = a.text(), - number = index + 1, - url = href, - scanlator = extra?.getElementsByAttributeValueContaining("href", "/group/")?.text(), - uploadDate = runCatching { - parseChapterDate(extra?.select("i")?.lastOrNull()?.ownText()) - }.getOrDefault(0), - branch = null, - source = source, - ) - } + private fun Element.parseChapter(index: Int): MangaChapter? { + val a = selectFirst("a.chapt") ?: return null + val extra = selectFirst(".extra") + val href = a.attrAsRelativeUrl("href") + return MangaChapter( + id = generateUid(href), + name = a.text(), + number = index + 1, + url = href, + scanlator = extra?.getElementsByAttributeValueContaining("href", "/group/")?.text(), + uploadDate = runCatching { + parseChapterDate(extra?.select("i")?.lastOrNull()?.ownText()) + }.getOrDefault(0), + branch = null, + source = source, + ) + } - private fun parseChapterDate(date: String?): Long { - if (date.isNullOrEmpty()) { - return 0 - } - val value = date.substringBefore(' ').toInt() - val field = when { - "sec" in date -> Calendar.SECOND - "min" in date -> Calendar.MINUTE - "hour" in date -> Calendar.HOUR - "day" in date -> Calendar.DAY_OF_MONTH - "week" in date -> Calendar.WEEK_OF_YEAR - "month" in date -> Calendar.MONTH - "year" in date -> Calendar.YEAR - else -> return 0 - } - val calendar = Calendar.getInstance() - calendar.add(field, -value) - return calendar.timeInMillis - } + private fun parseChapterDate(date: String?): Long { + if (date.isNullOrEmpty()) { + return 0 + } + val value = date.substringBefore(' ').toInt() + val field = when { + "sec" in date -> Calendar.SECOND + "min" in date -> Calendar.MINUTE + "hour" in date -> Calendar.HOUR + "day" in date -> Calendar.DAY_OF_MONTH + "week" in date -> Calendar.WEEK_OF_YEAR + "month" in date -> Calendar.MONTH + "year" in date -> Calendar.YEAR + else -> return 0 + } + val calendar = Calendar.getInstance() + calendar.add(field, -value) + return calendar.timeInMillis + } - private fun decryptAES(encrypted: String, password: String): String { - val cipherData = context.decodeBase64(encrypted) - val saltData = cipherData.copyOfRange(8, 16) - val (key, iv) = generateKeyAndIV( - keyLength = 32, - ivLength = 16, - iterations = 1, - salt = saltData, - password = password.toByteArray(StandardCharsets.UTF_8), - md = MessageDigest.getInstance("MD5"), - ) - val encryptedData = cipherData.copyOfRange(16, cipherData.size) - val cipher = Cipher.getInstance("AES/CBC/PKCS5Padding") - cipher.init(Cipher.DECRYPT_MODE, key, iv) - return cipher.doFinal(encryptedData).toString(Charsets.UTF_8) - } + private fun decryptAES(encrypted: String, password: String): String { + val cipherData = context.decodeBase64(encrypted) + val saltData = cipherData.copyOfRange(8, 16) + val (key, iv) = generateKeyAndIV( + keyLength = 32, + ivLength = 16, + iterations = 1, + salt = saltData, + password = password.toByteArray(StandardCharsets.UTF_8), + md = MessageDigest.getInstance("MD5"), + ) + val encryptedData = cipherData.copyOfRange(16, cipherData.size) + val cipher = Cipher.getInstance("AES/CBC/PKCS5Padding") + cipher.init(Cipher.DECRYPT_MODE, key, iv) + return cipher.doFinal(encryptedData).toString(Charsets.UTF_8) + } - @Suppress("SameParameterValue") - private fun generateKeyAndIV( - keyLength: Int, - ivLength: Int, - iterations: Int, - salt: ByteArray, - password: ByteArray, - md: MessageDigest, - ): Pair { - val digestLength = md.digestLength - val requiredLength = (keyLength + ivLength + digestLength - 1) / digestLength * digestLength - val generatedData = ByteArray(requiredLength) - var generatedLength = 0 - md.reset() - while (generatedLength < keyLength + ivLength) { - if (generatedLength > 0) { - md.update(generatedData, generatedLength - digestLength, digestLength) - } - md.update(password) - md.update(salt, 0, 8) - md.digest(generatedData, generatedLength, digestLength) - repeat(iterations - 1) { - md.update(generatedData, generatedLength, digestLength) - md.digest(generatedData, generatedLength, digestLength) - } - generatedLength += digestLength - } + @Suppress("SameParameterValue") + private fun generateKeyAndIV( + keyLength: Int, + ivLength: Int, + iterations: Int, + salt: ByteArray, + password: ByteArray, + md: MessageDigest, + ): Pair { + val digestLength = md.digestLength + val requiredLength = (keyLength + ivLength + digestLength - 1) / digestLength * digestLength + val generatedData = ByteArray(requiredLength) + var generatedLength = 0 + md.reset() + while (generatedLength < keyLength + ivLength) { + if (generatedLength > 0) { + md.update(generatedData, generatedLength - digestLength, digestLength) + } + md.update(password) + md.update(salt, 0, 8) + md.digest(generatedData, generatedLength, digestLength) + repeat(iterations - 1) { + md.update(generatedData, generatedLength, digestLength) + md.digest(generatedData, generatedLength, digestLength) + } + generatedLength += digestLength + } - return SecretKeySpec(generatedData.copyOfRange(0, keyLength), "AES") to IvParameterSpec( - if (ivLength > 0) { - generatedData.copyOfRange(keyLength, keyLength + ivLength) - } else byteArrayOf(), - ) - } + return SecretKeySpec(generatedData.copyOfRange(0, keyLength), "AES") to IvParameterSpec( + if (ivLength > 0) { + generatedData.copyOfRange(keyLength, keyLength + ivLength) + } else byteArrayOf(), + ) + } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BlogTruyenParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BlogTruyenParser.kt index 05b7b22d..27ce2241 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BlogTruyenParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/BlogTruyenParser.kt @@ -17,238 +17,236 @@ import java.util.* @MangaSourceParser("BLOGTRUYEN", "BlogTruyen", "vi") class BlogTruyenParser(context: MangaLoaderContext) : - PagedMangaParser(context, MangaSource.BLOGTRUYEN, pageSize = 20) { - - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("blogtruyen.vn", null) - - override val sortOrders: Set - get() = EnumSet.of(SortOrder.UPDATED) - - private val mutex = Mutex() - private val dateFormat = SimpleDateFormat("dd/MM/yyyy HH:mm", Locale.US) - private var cacheTags: ArrayMap? = null - - override suspend fun getDetails(manga: Manga): Manga { - val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - val descriptionElement = doc.selectFirstOrThrow("div.description") - val statusText = descriptionElement - .selectFirst("p:contains(Trạng thái) > span.color-red") - ?.text() - - val state = when (statusText) { - "Đang tiến hành" -> MangaState.ONGOING - "Đã hoàn thành" -> MangaState.FINISHED - else -> null - } - - val rating = doc.selectFirst("span.total-vote")?.attr("ng-init")?.let { text -> - val like = text.substringAfter("TotalLike=") - .substringBefore(';') - .toIntOrNull() ?: return@let RATING_UNKNOWN - val dislike = text.substringAfter("TotalDisLike=") - .toIntOrNull() ?: return@let RATING_UNKNOWN - - when { - like == 0 && dislike == 0 -> RATING_UNKNOWN - else -> like.toFloat() / (like + dislike) - } - } - - val tagMap = getOrCreateTagMap() - val tags = descriptionElement.select("p > span.category").mapNotNullToSet { - val tagName = it.selectFirst("a")?.text()?.trim() ?: return@mapNotNullToSet null - tagMap[tagName] - } - - return manga.copy( - tags = tags, - author = descriptionElement.selectFirst("p:contains(Tác giả) > a")?.text(), - description = doc.selectFirst(".detail .content")?.html(), - chapters = parseChapterList(doc), - largeCoverUrl = doc.selectLast("div.thumbnail > img")?.imageUrl(), - state = state, - rating = rating ?: RATING_UNKNOWN, - isNsfw = doc.getElementById("warningCategory") != null, - ) - } - - private fun parseChapterList(doc: Document): List { - val chapterList = doc.select("#list-chapters > p") - return chapterList.asReversed().mapChapters { index, element -> - val titleElement = element.selectFirst("span.title > a") ?: return@mapChapters null - val name = titleElement.text() - val relativeUrl = titleElement.attrAsRelativeUrl("href") - val id = relativeUrl.substringAfter('/').substringBefore('/') - val uploadDate = dateFormat.tryParse(element.select("span.publishedDate").text()) - MangaChapter( - id = generateUid(id), - name = name, - number = index + 1, - url = relativeUrl, - scanlator = null, - uploadDate = uploadDate, - branch = null, - source = source, - ) - } - } - - override suspend fun getListPage( - page: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - return when { - !query.isNullOrEmpty() -> { - val searchUrl = "https://${domain}/timkiem/nangcao/1/0/-1/-1?txt=${query.urlEncoded()}&p=$page" - val searchContent = webClient.httpGet(searchUrl).parseHtml() - .selectFirst("section.list-manga-bycate > div.list") - parseMangaList(searchContent) - } - - !tags.isNullOrEmpty() -> { - val tag = tags.oneOrThrowIfMany()!! - val categoryAjax = - "https://${domain}/ajax/Category/AjaxLoadMangaByCategory?id=${tag.key}&orderBy=5&p=$page" - val listContent = webClient.httpGet(categoryAjax).parseHtml().selectFirst("div.list") - parseMangaList(listContent) - } - - else -> getNormalList(page) - } - } - - private suspend fun getNormalList(page: Int): List { - val pageLink = "https://${domain}/page-$page" - val doc = webClient.httpGet(pageLink).parseHtml() - val listElements = doc.selectFirstOrThrow("section.list-mainpage.listview") - .select("div.bg-white.storyitem") - - return listElements.mapNotNull { - val linkTag = it.selectFirst("div.fl-l > a") ?: return@mapNotNull null - val relativeUrl = linkTag.attrAsRelativeUrl("href") - val tagMap = getOrCreateTagMap() - val tags = it.select("footer > div.category > a").mapNotNullToSet { a -> - tagMap[a.text()] - } - - Manga( - id = generateUid(relativeUrl), - title = linkTag.attr("title"), - altTitle = null, - description = it.selectFirst("p.al-j.break.line-height-15")?.text(), - url = relativeUrl, - publicUrl = relativeUrl.toAbsoluteUrl(domain), - coverUrl = linkTag.selectLast("img")?.imageUrl().orEmpty(), - source = source, - tags = tags, - isNsfw = false, - rating = RATING_UNKNOWN, - author = null, - state = null, - ) - } - } - - private fun parseMangaList(listElement: Element?): List { - listElement ?: return emptyList() - - return listElement.select("span.tiptip[data-tiptip]").mapNotNull { - val mangaInfo = listElement.getElementById(it.attr("data-tiptip")) ?: return@mapNotNull null - val a = it.selectFirst("a") ?: return@mapNotNull null - val relativeUrl = a.attrAsRelativeUrl("href") - Manga( - id = generateUid(relativeUrl), - title = a.text(), - altTitle = null, - description = mangaInfo.select("div.al-j.fs-12").text(), - url = relativeUrl, - publicUrl = relativeUrl.toAbsoluteUrl(domain), - coverUrl = mangaInfo.selectFirst("div > img.img")?.imageUrl().orEmpty(), - isNsfw = false, - rating = RATING_UNKNOWN, - tags = emptySet(), - author = null, - state = null, - source = source, - ) - } - } - - override suspend fun getPages(chapter: MangaChapter): List { - fun generateImageId(index: Int) = generateUid("${chapter.url}/$index") - - val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() - val pages = ArrayList() - val referer = chapter.url.toAbsoluteUrl(domain) - doc.select("#content > img").forEach { img -> - pages.add( - MangaPage( - id = generateImageId(pages.size), - url = img.imageUrl(), - referer = referer, - preview = null, - source = source, - ), - ) - } - - // Some chapters use js script to render images - val script = doc.selectLast("#content > script") - if (script != null && script.data().contains("listImageCaption")) { - val imagesStr = script.data().substringBefore(';').substringAfterLast('=').trim() - val imageArr = JSONArray(imagesStr) - for (i in 0 until imageArr.length()) { - val imageUrl = imageArr.getJSONObject(i).getString("url") - pages.add( - MangaPage( - id = generateImageId(pages.size), - url = imageUrl, - referer = referer, - preview = null, - source = source, - ), - ) - } - } - - return pages - } - - override suspend fun getTags(): Set { - val map = getOrCreateTagMap() - val tags = HashSet(map.size) - for (entry in map) { - tags.add(entry.value) - } - - return tags - } - - - private suspend fun getOrCreateTagMap(): ArrayMap = mutex.withLock { - cacheTags?.let { return@withLock it } - val doc = webClient.httpGet("/timkiem/nangcao".toAbsoluteUrl(domain)).parseHtml() - val tagItems = doc.select("li[data-id]") - val tagMap = ArrayMap(tagItems.size) - for (tag in tagItems) { - val title = tag.text().trim() - tagMap[tag.text().trim()] = MangaTag( - title = title, - key = tag.attr("data-id"), - source = source, - ) - } - - cacheTags = tagMap - tagMap - } - - private fun Element.imageUrl(): String { - return attrAsAbsoluteUrlOrNull("src") - ?: attrAsAbsoluteUrlOrNull("data-cfsrc") - ?: "" - } + PagedMangaParser(context, MangaSource.BLOGTRUYEN, pageSize = 20) { + + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("blogtruyen.vn", null) + + override val sortOrders: Set + get() = EnumSet.of(SortOrder.UPDATED) + + private val mutex = Mutex() + private val dateFormat = SimpleDateFormat("dd/MM/yyyy HH:mm", Locale.US) + private var cacheTags: ArrayMap? = null + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val descriptionElement = doc.selectFirstOrThrow("div.description") + val statusText = descriptionElement + .selectFirst("p:contains(Trạng thái) > span.color-red") + ?.text() + + val state = when (statusText) { + "Đang tiến hành" -> MangaState.ONGOING + "Đã hoàn thành" -> MangaState.FINISHED + else -> null + } + + val rating = doc.selectFirst("span.total-vote")?.attr("ng-init")?.let { text -> + val like = text.substringAfter("TotalLike=") + .substringBefore(';') + .toIntOrNull() ?: return@let RATING_UNKNOWN + val dislike = text.substringAfter("TotalDisLike=") + .toIntOrNull() ?: return@let RATING_UNKNOWN + + when { + like == 0 && dislike == 0 -> RATING_UNKNOWN + else -> like.toFloat() / (like + dislike) + } + } + + val tagMap = getOrCreateTagMap() + val tags = descriptionElement.select("p > span.category").mapNotNullToSet { + val tagName = it.selectFirst("a")?.text()?.trim() ?: return@mapNotNullToSet null + tagMap[tagName] + } + + return manga.copy( + tags = tags, + author = descriptionElement.selectFirst("p:contains(Tác giả) > a")?.text(), + description = doc.selectFirst(".detail .content")?.html(), + chapters = parseChapterList(doc), + largeCoverUrl = doc.selectLast("div.thumbnail > img")?.imageUrl(), + state = state, + rating = rating ?: RATING_UNKNOWN, + isNsfw = doc.getElementById("warningCategory") != null, + ) + } + + private fun parseChapterList(doc: Document): List { + val chapterList = doc.select("#list-chapters > p") + return chapterList.asReversed().mapChapters { index, element -> + val titleElement = element.selectFirst("span.title > a") ?: return@mapChapters null + val name = titleElement.text() + val relativeUrl = titleElement.attrAsRelativeUrl("href") + val id = relativeUrl.substringAfter('/').substringBefore('/') + val uploadDate = dateFormat.tryParse(element.select("span.publishedDate").text()) + MangaChapter( + id = generateUid(id), + name = name, + number = index + 1, + url = relativeUrl, + scanlator = null, + uploadDate = uploadDate, + branch = null, + source = source, + ) + } + } + + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + return when { + !query.isNullOrEmpty() -> { + val searchUrl = "https://${domain}/timkiem/nangcao/1/0/-1/-1?txt=${query.urlEncoded()}&p=$page" + val searchContent = webClient.httpGet(searchUrl).parseHtml() + .selectFirst("section.list-manga-bycate > div.list") + parseMangaList(searchContent) + } + + !tags.isNullOrEmpty() -> { + val tag = tags.oneOrThrowIfMany()!! + val categoryAjax = + "https://${domain}/ajax/Category/AjaxLoadMangaByCategory?id=${tag.key}&orderBy=5&p=$page" + val listContent = webClient.httpGet(categoryAjax).parseHtml().selectFirst("div.list") + parseMangaList(listContent) + } + + else -> getNormalList(page) + } + } + + private suspend fun getNormalList(page: Int): List { + val pageLink = "https://${domain}/page-$page" + val doc = webClient.httpGet(pageLink).parseHtml() + val listElements = doc.selectFirstOrThrow("section.list-mainpage.listview") + .select("div.bg-white.storyitem") + + return listElements.mapNotNull { + val linkTag = it.selectFirst("div.fl-l > a") ?: return@mapNotNull null + val relativeUrl = linkTag.attrAsRelativeUrl("href") + val tagMap = getOrCreateTagMap() + val tags = it.select("footer > div.category > a").mapNotNullToSet { a -> + tagMap[a.text()] + } + + Manga( + id = generateUid(relativeUrl), + title = linkTag.attr("title"), + altTitle = null, + description = it.selectFirst("p.al-j.break.line-height-15")?.text(), + url = relativeUrl, + publicUrl = relativeUrl.toAbsoluteUrl(domain), + coverUrl = linkTag.selectLast("img")?.imageUrl().orEmpty(), + source = source, + tags = tags, + isNsfw = false, + rating = RATING_UNKNOWN, + author = null, + state = null, + ) + } + } + + private fun parseMangaList(listElement: Element?): List { + listElement ?: return emptyList() + + return listElement.select("span.tiptip[data-tiptip]").mapNotNull { + val mangaInfo = listElement.getElementById(it.attr("data-tiptip")) ?: return@mapNotNull null + val a = it.selectFirst("a") ?: return@mapNotNull null + val relativeUrl = a.attrAsRelativeUrl("href") + Manga( + id = generateUid(relativeUrl), + title = a.text(), + altTitle = null, + description = mangaInfo.select("div.al-j.fs-12").text(), + url = relativeUrl, + publicUrl = relativeUrl.toAbsoluteUrl(domain), + coverUrl = mangaInfo.selectFirst("div > img.img")?.imageUrl().orEmpty(), + isNsfw = false, + rating = RATING_UNKNOWN, + tags = emptySet(), + author = null, + state = null, + source = source, + ) + } + } + + override suspend fun getPages(chapter: MangaChapter): List { + fun generateImageId(index: Int) = generateUid("${chapter.url}/$index") + + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + val pages = ArrayList() + val referer = chapter.url.toAbsoluteUrl(domain) + doc.select("#content > img").forEach { img -> + pages.add( + MangaPage( + id = generateImageId(pages.size), + url = img.imageUrl(), + preview = null, + source = source, + ), + ) + } + + // Some chapters use js script to render images + val script = doc.selectLast("#content > script") + if (script != null && script.data().contains("listImageCaption")) { + val imagesStr = script.data().substringBefore(';').substringAfterLast('=').trim() + val imageArr = JSONArray(imagesStr) + for (i in 0 until imageArr.length()) { + val imageUrl = imageArr.getJSONObject(i).getString("url") + pages.add( + MangaPage( + id = generateImageId(pages.size), + url = imageUrl, + preview = null, + source = source, + ), + ) + } + } + + return pages + } + + override suspend fun getTags(): Set { + val map = getOrCreateTagMap() + val tags = HashSet(map.size) + for (entry in map) { + tags.add(entry.value) + } + + return tags + } + + + private suspend fun getOrCreateTagMap(): ArrayMap = mutex.withLock { + cacheTags?.let { return@withLock it } + val doc = webClient.httpGet("/timkiem/nangcao".toAbsoluteUrl(domain)).parseHtml() + val tagItems = doc.select("li[data-id]") + val tagMap = ArrayMap(tagItems.size) + for (tag in tagItems) { + val title = tag.text().trim() + tagMap[tag.text().trim()] = MangaTag( + title = title, + key = tag.attr("data-id"), + source = source, + ) + } + + cacheTags = tagMap + tagMap + } + + private fun Element.imageUrl(): String { + return attrAsAbsoluteUrlOrNull("src") + ?: attrAsAbsoluteUrlOrNull("data-cfsrc") + ?: "" + } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/CloneMangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/CloneMangaParser.kt index 56fd66e0..aa42dfd0 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/CloneMangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/CloneMangaParser.kt @@ -90,7 +90,6 @@ internal class CloneMangaParser(context: MangaLoaderContext) : PagedMangaParser( MangaPage( id = generateUid(imgUrl), url = imgUrl, - referer = imgUrl, preview = null, source = source, ), diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ComickFunParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ComickFunParser.kt index fdb94512..ad3b87b2 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ComickFunParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ComickFunParser.kt @@ -24,193 +24,192 @@ private const val CHAPTERS_LIMIT = 99999 @MangaSourceParser("COMICK_FUN", "ComicK") internal class ComickFunParser(context: MangaLoaderContext) : MangaParser(context, MangaSource.COMICK_FUN) { - override val configKeyDomain = ConfigKey.Domain("comick.app", null) + override val configKeyDomain = ConfigKey.Domain("comick.app", null) - override val sortOrders: Set = EnumSet.of( - SortOrder.POPULARITY, - SortOrder.UPDATED, - SortOrder.RATING, - ) + override val sortOrders: Set = EnumSet.of( + SortOrder.POPULARITY, + SortOrder.UPDATED, + SortOrder.RATING, + ) - @Volatile - private var cachedTags: SparseArrayCompat? = null + @Volatile + private var cachedTags: SparseArrayCompat? = null - override suspend fun getList( - offset: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - val domain = domain - val url = buildString { - append("https://api.") - append(domain) - append("/search?tachiyomi=true") - if (!query.isNullOrEmpty()) { - if (offset > 0) { - return emptyList() - } - append("&q=") - append(query.urlEncoded()) - } else { - append("&limit=") - append(PAGE_SIZE) - append("&page=") - append((offset / PAGE_SIZE) + 1) - if (!tags.isNullOrEmpty()) { - append("&genres=") - appendAll(tags, "&genres=", MangaTag::key) - } - append("&sort=") // view, uploaded, rating, follow, user_follow_count - append( - when (sortOrder) { - SortOrder.POPULARITY -> "view" - SortOrder.RATING -> "rating" - else -> "uploaded" - }, - ) - } - } - val ja = webClient.httpGet(url).parseJsonArray() - val tagsMap = cachedTags ?: loadTags() - return ja.mapJSON { jo -> - val slug = jo.getString("slug") - Manga( - id = generateUid(slug), - title = jo.getString("title"), - altTitle = null, - url = slug, - publicUrl = "https://$domain/comic/$slug", - rating = jo.getDoubleOrDefault("rating", -10.0).toFloat() / 10f, - isNsfw = false, - coverUrl = jo.getString("cover_url"), - largeCoverUrl = null, - description = jo.getStringOrNull("desc"), - tags = jo.selectGenres("genres", tagsMap), - state = runCatching { - if (jo.getBoolean("translation_completed")) { - MangaState.FINISHED - } else { - MangaState.ONGOING - } - }.getOrNull(), - author = null, - source = source, - ) - } - } + override suspend fun getList( + offset: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + val domain = domain + val url = buildString { + append("https://api.") + append(domain) + append("/search?tachiyomi=true") + if (!query.isNullOrEmpty()) { + if (offset > 0) { + return emptyList() + } + append("&q=") + append(query.urlEncoded()) + } else { + append("&limit=") + append(PAGE_SIZE) + append("&page=") + append((offset / PAGE_SIZE) + 1) + if (!tags.isNullOrEmpty()) { + append("&genres=") + appendAll(tags, "&genres=", MangaTag::key) + } + append("&sort=") // view, uploaded, rating, follow, user_follow_count + append( + when (sortOrder) { + SortOrder.POPULARITY -> "view" + SortOrder.RATING -> "rating" + else -> "uploaded" + }, + ) + } + } + val ja = webClient.httpGet(url).parseJsonArray() + val tagsMap = cachedTags ?: loadTags() + return ja.mapJSON { jo -> + val slug = jo.getString("slug") + Manga( + id = generateUid(slug), + title = jo.getString("title"), + altTitle = null, + url = slug, + publicUrl = "https://$domain/comic/$slug", + rating = jo.getDoubleOrDefault("rating", -10.0).toFloat() / 10f, + isNsfw = false, + coverUrl = jo.getString("cover_url"), + largeCoverUrl = null, + description = jo.getStringOrNull("desc"), + tags = jo.selectGenres("genres", tagsMap), + state = runCatching { + if (jo.getBoolean("translation_completed")) { + MangaState.FINISHED + } else { + MangaState.ONGOING + } + }.getOrNull(), + author = null, + source = source, + ) + } + } - override suspend fun getDetails(manga: Manga): Manga { - val domain = domain - val url = "https://api.$domain/comic/${manga.url}?tachiyomi=true" - val jo = webClient.httpGet(url).parseJson() - val comic = jo.getJSONObject("comic") - return manga.copy( - title = comic.getString("title"), - altTitle = null, // TODO - isNsfw = jo.getBoolean("matureContent") || comic.getBoolean("hentai"), - description = comic.getStringOrNull("parsed") ?: comic.getString("desc"), - tags = manga.tags + jo.getJSONArray("genres").mapJSONToSet { - MangaTag( - title = it.getString("name"), - key = it.getString("slug"), - source = source, - ) - }, - author = jo.getJSONArray("artists").optJSONObject(0)?.getString("name"), - chapters = getChapters(comic.getLong("id")), - ) - } + override suspend fun getDetails(manga: Manga): Manga { + val domain = domain + val url = "https://api.$domain/comic/${manga.url}?tachiyomi=true" + val jo = webClient.httpGet(url).parseJson() + val comic = jo.getJSONObject("comic") + return manga.copy( + title = comic.getString("title"), + altTitle = null, // TODO + isNsfw = jo.getBoolean("matureContent") || comic.getBoolean("hentai"), + description = comic.getStringOrNull("parsed") ?: comic.getString("desc"), + tags = manga.tags + jo.getJSONArray("genres").mapJSONToSet { + MangaTag( + title = it.getString("name"), + key = it.getString("slug"), + source = source, + ) + }, + author = jo.getJSONArray("artists").optJSONObject(0)?.getString("name"), + chapters = getChapters(comic.getLong("id")), + ) + } - override suspend fun getPages(chapter: MangaChapter): List { - val jo = webClient.httpGet( - "https://api.${domain}/chapter/${chapter.url}?tachiyomi=true", - ).parseJson().getJSONObject("chapter") - val referer = "https://${domain}/" - return jo.getJSONArray("images").mapJSON { - val url = it.getString("url") - MangaPage( - id = generateUid(url), - url = url, - referer = referer, - preview = null, - source = source, - ) - } - } + override suspend fun getPages(chapter: MangaChapter): List { + val jo = webClient.httpGet( + "https://api.${domain}/chapter/${chapter.url}?tachiyomi=true", + ).parseJson().getJSONObject("chapter") + val referer = "https://${domain}/" + return jo.getJSONArray("images").mapJSON { + val url = it.getString("url") + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } - override suspend fun getTags(): Set { - val sparseArray = cachedTags ?: loadTags() - val set = ArraySet(sparseArray.size()) - for (i in 0 until sparseArray.size()) { - set.add(sparseArray.valueAt(i)) - } - return set - } + override suspend fun getTags(): Set { + val sparseArray = cachedTags ?: loadTags() + val set = ArraySet(sparseArray.size()) + for (i in 0 until sparseArray.size()) { + set.add(sparseArray.valueAt(i)) + } + return set + } - private suspend fun loadTags(): SparseArrayCompat { - val ja = webClient.httpGet("https://api.${domain}/genre").parseJsonArray() - val tags = SparseArrayCompat(ja.length()) - for (jo in ja.JSONIterator()) { - tags.append( - jo.getInt("id"), - MangaTag( - title = jo.getString("name"), - key = jo.getString("slug"), - source = source, - ), - ) - } - cachedTags = tags - return tags - } + private suspend fun loadTags(): SparseArrayCompat { + val ja = webClient.httpGet("https://api.${domain}/genre").parseJsonArray() + val tags = SparseArrayCompat(ja.length()) + for (jo in ja.JSONIterator()) { + tags.append( + jo.getInt("id"), + MangaTag( + title = jo.getString("name"), + key = jo.getString("slug"), + source = source, + ), + ) + } + cachedTags = tags + return tags + } - private suspend fun getChapters(id: Long): List { - val ja = webClient.httpGet( - url = "https://api.${domain}/comic/$id/chapter?tachiyomi=true&limit=$CHAPTERS_LIMIT", - ).parseJson().getJSONArray("chapters") - val dateFormat = SimpleDateFormat("yyyy-MM-dd") - val counters = HashMap() - return ja.mapReversed { jo -> - val locale = Locale.forLanguageTag(jo.getString("lang")) - var number = counters[locale] ?: 0 - number++ - counters[locale] = number - MangaChapter( - id = generateUid(jo.getLong("id")), - name = buildString { - jo.getStringOrNull("vol")?.let { append("Vol ").append(it).append(' ') } - jo.getStringOrNull("chap")?.let { append("Chap ").append(it) } - jo.getStringOrNull("title")?.let { append(": ").append(it) } - }, - number = number, - url = jo.getString("hid"), - scanlator = jo.optJSONArray("group_name")?.optString(0), - uploadDate = dateFormat.tryParse(jo.getString("created_at").substringBefore('T')), - branch = locale.getDisplayName(locale).toTitleCase(locale), - source = source, - ) - } - } + private suspend fun getChapters(id: Long): List { + val ja = webClient.httpGet( + url = "https://api.${domain}/comic/$id/chapter?tachiyomi=true&limit=$CHAPTERS_LIMIT", + ).parseJson().getJSONArray("chapters") + val dateFormat = SimpleDateFormat("yyyy-MM-dd") + val counters = HashMap() + return ja.mapReversed { jo -> + val locale = Locale.forLanguageTag(jo.getString("lang")) + var number = counters[locale] ?: 0 + number++ + counters[locale] = number + MangaChapter( + id = generateUid(jo.getLong("id")), + name = buildString { + jo.getStringOrNull("vol")?.let { append("Vol ").append(it).append(' ') } + jo.getStringOrNull("chap")?.let { append("Chap ").append(it) } + jo.getStringOrNull("title")?.let { append(": ").append(it) } + }, + number = number, + url = jo.getString("hid"), + scanlator = jo.optJSONArray("group_name")?.optString(0), + uploadDate = dateFormat.tryParse(jo.getString("created_at").substringBefore('T')), + branch = locale.getDisplayName(locale).toTitleCase(locale), + source = source, + ) + } + } - private inline fun JSONArray.mapReversed(block: (JSONObject) -> R): List { - val len = length() - val destination = ArrayList(len) - for (i in (0 until len).reversed()) { - val jo = getJSONObject(i) - destination.add(block(jo)) - } - return destination - } + private inline fun JSONArray.mapReversed(block: (JSONObject) -> R): List { + val len = length() + val destination = ArrayList(len) + for (i in (0 until len).reversed()) { + val jo = getJSONObject(i) + destination.add(block(jo)) + } + return destination + } - private fun JSONObject.selectGenres(name: String, tags: SparseArrayCompat): Set { - val array = optJSONArray(name) ?: return emptySet() - val res = ArraySet(array.length()) - for (i in 0 until array.length()) { - val id = array.getInt(i) - val tag = tags.get(id) ?: continue - res.add(tag) - } - return res - } + private fun JSONObject.selectGenres(name: String, tags: SparseArrayCompat): Set { + val array = optJSONArray(name) ?: return emptySet() + val res = ArraySet(array.length()) + for (i in 0 until array.length()) { + val id = array.getInt(i) + val tag = tags.get(id) ?: continue + res.add(tag) + } + return res + } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/DesuMeParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/DesuMeParser.kt index 3004dcc4..d974ffed 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/DesuMeParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/DesuMeParser.kt @@ -135,7 +135,6 @@ internal class DesuMeParser(context: MangaLoaderContext) : PagedMangaParser(cont return json.getJSONObject("pages").getJSONArray("list").mapJSON { jo -> MangaPage( id = generateUid(jo.getLong("id")), - referer = fullUrl, preview = null, source = chapter.source, url = jo.getString("img"), diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ExHentaiParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ExHentaiParser.kt index 86be7938..0f0664bd 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ExHentaiParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ExHentaiParser.kt @@ -20,289 +20,288 @@ private const val DOMAIN_AUTHORIZED = "exhentai.org" @MangaSourceParser("EXHENTAI", "ExHentai") internal class ExHentaiParser( - context: MangaLoaderContext, + context: MangaLoaderContext, ) : PagedMangaParser(context, MangaSource.EXHENTAI, pageSize = 25), MangaParserAuthProvider { - override val sortOrders: Set = Collections.singleton( - SortOrder.NEWEST, - ) + override val sortOrders: Set = Collections.singleton( + SortOrder.NEWEST, + ) - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain(if (isAuthorized) DOMAIN_AUTHORIZED else DOMAIN_UNAUTHORIZED, null) + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain(if (isAuthorized) DOMAIN_AUTHORIZED else DOMAIN_UNAUTHORIZED, null) - override val authUrl: String - get() = "https://${domain}/bounce_login.php" + override val authUrl: String + get() = "https://${domain}/bounce_login.php" - private val ratingPattern = Regex("-?[0-9]+px") - private val authCookies = arrayOf("ipb_member_id", "ipb_pass_hash") - private var updateDm = false - private val nextPages = SparseArrayCompat() - private val suspiciousContentKey = ConfigKey.ShowSuspiciousContent(true) + private val ratingPattern = Regex("-?[0-9]+px") + private val authCookies = arrayOf("ipb_member_id", "ipb_pass_hash") + private var updateDm = false + private val nextPages = SparseArrayCompat() + private val suspiciousContentKey = ConfigKey.ShowSuspiciousContent(true) - override val isAuthorized: Boolean - get() { - val authorized = isAuthorized(DOMAIN_UNAUTHORIZED) - if (authorized) { - if (!isAuthorized(DOMAIN_AUTHORIZED)) { - context.cookieJar.copyCookies( - DOMAIN_UNAUTHORIZED, - DOMAIN_AUTHORIZED, - authCookies, - ) - context.cookieJar.insertCookies(DOMAIN_AUTHORIZED, "yay=louder") - } - return true - } - return false - } + override val isAuthorized: Boolean + get() { + val authorized = isAuthorized(DOMAIN_UNAUTHORIZED) + if (authorized) { + if (!isAuthorized(DOMAIN_AUTHORIZED)) { + context.cookieJar.copyCookies( + DOMAIN_UNAUTHORIZED, + DOMAIN_AUTHORIZED, + authCookies, + ) + context.cookieJar.insertCookies(DOMAIN_AUTHORIZED, "yay=louder") + } + return true + } + return false + } - init { - context.cookieJar.insertCookies(DOMAIN_AUTHORIZED, "nw=1", "sl=dm_2") - context.cookieJar.insertCookies(DOMAIN_UNAUTHORIZED, "nw=1", "sl=dm_2") - paginator.firstPage = 0 - } + init { + context.cookieJar.insertCookies(DOMAIN_AUTHORIZED, "nw=1", "sl=dm_2") + context.cookieJar.insertCookies(DOMAIN_UNAUTHORIZED, "nw=1", "sl=dm_2") + paginator.firstPage = 0 + } - override suspend fun getListPage( - page: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - var search = query?.urlEncoded().orEmpty() - val next = nextPages.get(page, 0L) - if (page > 0 && next == 0L) { - assert(false) { "Page timestamp not found" } - return emptyList() - } - val url = buildString { - append("https://") - append(domain) - append("/?next=") - append(next) - if (!tags.isNullOrEmpty()) { - var fCats = 0 - for (tag in tags) { - tag.key.toIntOrNull()?.let { fCats = fCats or it } ?: run { - search += tag.key + " " - } - } - if (fCats != 0) { - append("&f_cats=") - append(1023 - fCats) - } - } - if (search.isNotEmpty()) { - append("&f_search=") - append(search.trim().replace(' ', '+')) - } - // by unknown reason cookie "sl=dm_2" is ignored, so, we should request it again - if (updateDm) { - append("&inline_set=dm_e") - } - append("&advsearch=1") - if (config[suspiciousContentKey]) { - append("&f_sh=on") - } - } - val body = webClient.httpGet(url).parseHtml().body() - val root = body.selectFirst("table.itg") - ?.selectFirst("tbody") - ?: if (updateDm) { - body.parseFailed("Cannot find root") - } else { - updateDm = true - return getListPage(page, query, tags, sortOrder) - } - updateDm = false - nextPages[page + 1] = getNextTimestamp(body) - return root.children().mapNotNull { tr -> - if (tr.childrenSize() != 2) return@mapNotNull null - val (td1, td2) = tr.children() - val glink = td2.selectFirstOrThrow("div.glink") - val a = glink.parents().select("a").first() ?: glink.parseFailed("link not found") - val href = a.attrAsRelativeUrl("href") - val tagsDiv = glink.nextElementSibling() ?: glink.parseFailed("tags div not found") - val mainTag = td2.selectFirst("div.cn")?.let { div -> - MangaTag( - title = div.text().toTitleCase(), - key = tagIdByClass(div.classNames()) ?: return@let null, - source = source, - ) - } - Manga( - id = generateUid(href), - title = glink.text().cleanupTitle(), - altTitle = null, - url = href, - publicUrl = a.absUrl("href"), - rating = td2.selectFirst("div.ir")?.parseRating() ?: RATING_UNKNOWN, - isNsfw = true, - coverUrl = td1.selectFirst("img")?.absUrl("src").orEmpty(), - tags = setOfNotNull(mainTag), - state = null, - author = tagsDiv.getElementsContainingOwnText("artist:").first() - ?.nextElementSibling()?.text(), - source = source, - ) - } - } + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + var search = query?.urlEncoded().orEmpty() + val next = nextPages.get(page, 0L) + if (page > 0 && next == 0L) { + assert(false) { "Page timestamp not found" } + return emptyList() + } + val url = buildString { + append("https://") + append(domain) + append("/?next=") + append(next) + if (!tags.isNullOrEmpty()) { + var fCats = 0 + for (tag in tags) { + tag.key.toIntOrNull()?.let { fCats = fCats or it } ?: run { + search += tag.key + " " + } + } + if (fCats != 0) { + append("&f_cats=") + append(1023 - fCats) + } + } + if (search.isNotEmpty()) { + append("&f_search=") + append(search.trim().replace(' ', '+')) + } + // by unknown reason cookie "sl=dm_2" is ignored, so, we should request it again + if (updateDm) { + append("&inline_set=dm_e") + } + append("&advsearch=1") + if (config[suspiciousContentKey]) { + append("&f_sh=on") + } + } + val body = webClient.httpGet(url).parseHtml().body() + val root = body.selectFirst("table.itg") + ?.selectFirst("tbody") + ?: if (updateDm) { + body.parseFailed("Cannot find root") + } else { + updateDm = true + return getListPage(page, query, tags, sortOrder) + } + updateDm = false + nextPages[page + 1] = getNextTimestamp(body) + return root.children().mapNotNull { tr -> + if (tr.childrenSize() != 2) return@mapNotNull null + val (td1, td2) = tr.children() + val glink = td2.selectFirstOrThrow("div.glink") + val a = glink.parents().select("a").first() ?: glink.parseFailed("link not found") + val href = a.attrAsRelativeUrl("href") + val tagsDiv = glink.nextElementSibling() ?: glink.parseFailed("tags div not found") + val mainTag = td2.selectFirst("div.cn")?.let { div -> + MangaTag( + title = div.text().toTitleCase(), + key = tagIdByClass(div.classNames()) ?: return@let null, + source = source, + ) + } + Manga( + id = generateUid(href), + title = glink.text().cleanupTitle(), + altTitle = null, + url = href, + publicUrl = a.absUrl("href"), + rating = td2.selectFirst("div.ir")?.parseRating() ?: RATING_UNKNOWN, + isNsfw = true, + coverUrl = td1.selectFirst("img")?.absUrl("src").orEmpty(), + tags = setOfNotNull(mainTag), + state = null, + author = tagsDiv.getElementsContainingOwnText("artist:").first() + ?.nextElementSibling()?.text(), + source = source, + ) + } + } - override suspend fun getDetails(manga: Manga): Manga { - val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - val root = doc.body().selectFirstOrThrow("div.gm") - val cover = root.getElementById("gd1")?.children()?.first() - val title = root.getElementById("gd2") - val taglist = root.getElementById("taglist") - val tabs = doc.body().selectFirst("table.ptt")?.selectFirst("tr") - return manga.copy( - title = title?.getElementById("gn")?.text()?.cleanupTitle() ?: manga.title, - altTitle = title?.getElementById("gj")?.text()?.cleanupTitle() ?: manga.altTitle, - publicUrl = doc.baseUri().ifEmpty { manga.publicUrl }, - rating = root.getElementById("rating_label")?.text() - ?.substringAfterLast(' ') - ?.toFloatOrNull() - ?.div(5f) ?: manga.rating, - largeCoverUrl = cover?.styleValueOrNull("background")?.cssUrl(), - description = taglist?.select("tr")?.joinToString("
") { tr -> - val (tc, td) = tr.children() - val subtags = td.select("a").joinToString { it.html() } - "${tc.html()} $subtags" - }, - chapters = tabs?.select("a")?.findLast { a -> - a.text().toIntOrNull() != null - }?.let { a -> - val count = a.text().toInt() - val chapters = ChaptersListBuilder(count) - for (i in 1..count) { - val url = "${manga.url}?p=${i - 1}" - chapters += MangaChapter( - id = generateUid(url), - name = "${manga.title} #$i", - number = i, - url = url, - uploadDate = 0L, - source = source, - scanlator = null, - branch = null, - ) - } - chapters.toList() - }, - ) - } + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val root = doc.body().selectFirstOrThrow("div.gm") + val cover = root.getElementById("gd1")?.children()?.first() + val title = root.getElementById("gd2") + val taglist = root.getElementById("taglist") + val tabs = doc.body().selectFirst("table.ptt")?.selectFirst("tr") + return manga.copy( + title = title?.getElementById("gn")?.text()?.cleanupTitle() ?: manga.title, + altTitle = title?.getElementById("gj")?.text()?.cleanupTitle() ?: manga.altTitle, + publicUrl = doc.baseUri().ifEmpty { manga.publicUrl }, + rating = root.getElementById("rating_label")?.text() + ?.substringAfterLast(' ') + ?.toFloatOrNull() + ?.div(5f) ?: manga.rating, + largeCoverUrl = cover?.styleValueOrNull("background")?.cssUrl(), + description = taglist?.select("tr")?.joinToString("
") { tr -> + val (tc, td) = tr.children() + val subtags = td.select("a").joinToString { it.html() } + "${tc.html()} $subtags" + }, + chapters = tabs?.select("a")?.findLast { a -> + a.text().toIntOrNull() != null + }?.let { a -> + val count = a.text().toInt() + val chapters = ChaptersListBuilder(count) + for (i in 1..count) { + val url = "${manga.url}?p=${i - 1}" + chapters += MangaChapter( + id = generateUid(url), + name = "${manga.title} #$i", + number = i, + url = url, + uploadDate = 0L, + source = source, + scanlator = null, + branch = null, + ) + } + chapters.toList() + }, + ) + } - override suspend fun getPages(chapter: MangaChapter): List { - val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() - val root = doc.body().requireElementById("gdt") - return root.select("a").map { a -> - val url = a.attrAsRelativeUrl("href") - MangaPage( - id = generateUid(url), - url = url, - referer = a.absUrl("href"), - preview = null, - source = source, - ) - } - } + override suspend fun getPages(chapter: MangaChapter): List { + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + val root = doc.body().requireElementById("gdt") + return root.select("a").map { a -> + val url = a.attrAsRelativeUrl("href") + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } - override suspend fun getPageUrl(page: MangaPage): String { - val doc = webClient.httpGet(page.url.toAbsoluteUrl(domain)).parseHtml() - return doc.body().requireElementById("img").attrAsAbsoluteUrl("src") - } + override suspend fun getPageUrl(page: MangaPage): String { + val doc = webClient.httpGet(page.url.toAbsoluteUrl(domain)).parseHtml() + return doc.body().requireElementById("img").attrAsAbsoluteUrl("src") + } - override suspend fun getTags(): Set { - val doc = webClient.httpGet("https://${domain}").parseHtml() - val root = doc.body().requireElementById("searchbox").selectFirstOrThrow("table") - return root.select("div.cs").mapNotNullToSet { div -> - val id = div.id().substringAfterLast('_').toIntOrNull() - ?: return@mapNotNullToSet null - MangaTag( - title = div.text().toTitleCase(), - key = id.toString(), - source = source, - ) - } - } + override suspend fun getTags(): Set { + val doc = webClient.httpGet("https://${domain}").parseHtml() + val root = doc.body().requireElementById("searchbox").selectFirstOrThrow("table") + return root.select("div.cs").mapNotNullToSet { div -> + val id = div.id().substringAfterLast('_').toIntOrNull() + ?: return@mapNotNullToSet null + MangaTag( + title = div.text().toTitleCase(), + key = id.toString(), + source = source, + ) + } + } - override suspend fun getUsername(): String { - val doc = webClient.httpGet("https://forums.$DOMAIN_UNAUTHORIZED/").parseHtml().body() - val username = doc.getElementById("userlinks") - ?.getElementsByAttributeValueContaining("href", "showuser=") - ?.firstOrNull() - ?.ownText() - ?: if (doc.getElementById("userlinksguest") != null) { - throw AuthRequiredException(source) - } else { - doc.parseFailed() - } - return username - } + override suspend fun getUsername(): String { + val doc = webClient.httpGet("https://forums.$DOMAIN_UNAUTHORIZED/").parseHtml().body() + val username = doc.getElementById("userlinks") + ?.getElementsByAttributeValueContaining("href", "showuser=") + ?.firstOrNull() + ?.ownText() + ?: if (doc.getElementById("userlinksguest") != null) { + throw AuthRequiredException(source) + } else { + doc.parseFailed() + } + return username + } - override fun onCreateConfig(keys: MutableCollection>) { - super.onCreateConfig(keys) - keys.add(suspiciousContentKey) - } + override fun onCreateConfig(keys: MutableCollection>) { + super.onCreateConfig(keys) + keys.add(suspiciousContentKey) + } - private fun isAuthorized(domain: String): Boolean { - val cookies = context.cookieJar.getCookies(domain).mapToSet { x -> x.name } - return authCookies.all { it in cookies } - } + private fun isAuthorized(domain: String): Boolean { + val cookies = context.cookieJar.getCookies(domain).mapToSet { x -> x.name } + return authCookies.all { it in cookies } + } - private fun Element.parseRating(): Float { - return runCatching { - val style = requireNotNull(attr("style")) - val (v1, v2) = ratingPattern.find(style)!!.destructured - var p1 = v1.dropLast(2).toInt() - val p2 = v2.dropLast(2).toInt() - if (p2 != -1) { - p1 += 8 - } - (80 - p1) / 80f - }.getOrDefault(RATING_UNKNOWN) - } + private fun Element.parseRating(): Float { + return runCatching { + val style = requireNotNull(attr("style")) + val (v1, v2) = ratingPattern.find(style)!!.destructured + var p1 = v1.dropLast(2).toInt() + val p2 = v2.dropLast(2).toInt() + if (p2 != -1) { + p1 += 8 + } + (80 - p1) / 80f + }.getOrDefault(RATING_UNKNOWN) + } - private fun String.cleanupTitle(): String { - val result = StringBuilder(length) - var skip = false - for (c in this) { - when { - c == '[' -> skip = true - c == ']' -> skip = false - c.isWhitespace() && result.isEmpty() -> continue - !skip -> result.append(c) - } - } - while (result.lastOrNull()?.isWhitespace() == true) { - result.deleteCharAt(result.lastIndex) - } - return result.toString() - } + private fun String.cleanupTitle(): String { + val result = StringBuilder(length) + var skip = false + for (c in this) { + when { + c == '[' -> skip = true + c == ']' -> skip = false + c.isWhitespace() && result.isEmpty() -> continue + !skip -> result.append(c) + } + } + while (result.lastOrNull()?.isWhitespace() == true) { + result.deleteCharAt(result.lastIndex) + } + return result.toString() + } - private fun String.cssUrl(): String? { - val fromIndex = indexOf("url(") - if (fromIndex == -1) { - return null - } - val toIndex = indexOf(')', startIndex = fromIndex) - return if (toIndex == -1) { - null - } else { - substring(fromIndex + 4, toIndex).trim() - } - } + private fun String.cssUrl(): String? { + val fromIndex = indexOf("url(") + if (fromIndex == -1) { + return null + } + val toIndex = indexOf(')', startIndex = fromIndex) + return if (toIndex == -1) { + null + } else { + substring(fromIndex + 4, toIndex).trim() + } + } - private fun tagIdByClass(classNames: Collection): String? { - val className = classNames.find { x -> x.startsWith("ct") } ?: return null - val num = className.drop(2).toIntOrNull(16) ?: return null - return 2.0.pow(num).toInt().toString() - } + private fun tagIdByClass(classNames: Collection): String? { + val className = classNames.find { x -> x.startsWith("ct") } ?: return null + val num = className.drop(2).toIntOrNull(16) ?: return null + return 2.0.pow(num).toInt().toString() + } - private fun getNextTimestamp(root: Element): Long { - return root.getElementById("unext") - ?.attrAsAbsoluteUrlOrNull("href") - ?.toHttpUrlOrNull() - ?.queryParameter("next") - ?.toLongOrNull() ?: 1 - } + private fun getNextTimestamp(root: Element): Long { + return root.getElementById("unext") + ?.attrAsAbsoluteUrlOrNull("href") + ?.toHttpUrlOrNull() + ?.queryParameter("next") + ?.toLongOrNull() ?: 1 + } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaDexParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaDexParser.kt index ed236b15..96b795ef 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaDexParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaDexParser.kt @@ -22,254 +22,252 @@ private const val CHAPTERS_MAX_PAGE_SIZE = 500 private const val CHAPTERS_PARALLELISM = 3 private const val CHAPTERS_MAX_COUNT = 10_000 // strange api behavior, looks like a bug private const val CONTENT_RATING = - "contentRating[]=safe&contentRating[]=suggestive&contentRating[]=erotica&contentRating[]=pornographic" + "contentRating[]=safe&contentRating[]=suggestive&contentRating[]=erotica&contentRating[]=pornographic" private const val LOCALE_FALLBACK = "en" @MangaSourceParser("MANGADEX", "MangaDex") internal class MangaDexParser(context: MangaLoaderContext) : MangaParser(context, MangaSource.MANGADEX) { - override val configKeyDomain = ConfigKey.Domain("mangadex.org", null) + override val configKeyDomain = ConfigKey.Domain("mangadex.org", null) - override val sortOrders: EnumSet = EnumSet.of( - SortOrder.UPDATED, - SortOrder.ALPHABETICAL, - SortOrder.NEWEST, - SortOrder.POPULARITY, - ) + override val sortOrders: EnumSet = EnumSet.of( + SortOrder.UPDATED, + SortOrder.ALPHABETICAL, + SortOrder.NEWEST, + SortOrder.POPULARITY, + ) - override suspend fun getList( - offset: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - val domain = domain - val url = buildString { - append("https://api.") - append(domain) - append("/manga?limit=") - append(PAGE_SIZE) - append("&offset=") - append(offset) - append("&includes[]=cover_art&includes[]=author&includes[]=artist&") - tags?.forEach { tag -> - append("includedTags[]=") - append(tag.key) - append('&') - } - if (!query.isNullOrEmpty()) { - append("title=") - append(query.urlEncoded()) - append('&') - } - append(CONTENT_RATING) - append("&order") - append( - when (sortOrder) { - SortOrder.UPDATED, - -> "[latestUploadedChapter]=desc" + override suspend fun getList( + offset: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + val domain = domain + val url = buildString { + append("https://api.") + append(domain) + append("/manga?limit=") + append(PAGE_SIZE) + append("&offset=") + append(offset) + append("&includes[]=cover_art&includes[]=author&includes[]=artist&") + tags?.forEach { tag -> + append("includedTags[]=") + append(tag.key) + append('&') + } + if (!query.isNullOrEmpty()) { + append("title=") + append(query.urlEncoded()) + append('&') + } + append(CONTENT_RATING) + append("&order") + append( + when (sortOrder) { + SortOrder.UPDATED, + -> "[latestUploadedChapter]=desc" - SortOrder.ALPHABETICAL -> "[title]=asc" - SortOrder.NEWEST -> "[createdAt]=desc" - SortOrder.POPULARITY -> "[followedCount]=desc" - else -> "[followedCount]=desc" - }, - ) - } - val json = webClient.httpGet(url).parseJson().getJSONArray("data") - return json.mapJSON { jo -> - val id = jo.getString("id") - val attrs = jo.getJSONObject("attributes") - val relations = jo.getJSONArray("relationships").associateByKey("type") - val cover = relations["cover_art"] - ?.getJSONObject("attributes") - ?.getString("fileName") - ?.let { - "https://uploads.$domain/covers/$id/$it" - } - Manga( - id = generateUid(id), - title = requireNotNull(attrs.getJSONObject("title").selectByLocale()) { - "Title should not be null" - }, - altTitle = attrs.optJSONObject("altTitles")?.selectByLocale(), - url = id, - publicUrl = "https://$domain/title/$id", - rating = RATING_UNKNOWN, - isNsfw = attrs.getStringOrNull("contentRating") == "erotica", - coverUrl = cover?.plus(".256.jpg").orEmpty(), - largeCoverUrl = cover, - description = attrs.optJSONObject("description")?.selectByLocale(), - tags = attrs.getJSONArray("tags").mapJSONToSet { tag -> - MangaTag( - title = tag.getJSONObject("attributes") - .getJSONObject("name") - .firstStringValue() - .toTitleCase(), - key = tag.getString("id"), - source = source, - ) - }, - state = when (jo.getStringOrNull("status")) { - "ongoing" -> MangaState.ONGOING - "completed" -> MangaState.FINISHED - else -> null - }, - author = (relations["author"] ?: relations["artist"]) - ?.getJSONObject("attributes") - ?.getStringOrNull("name"), - source = source, - ) - } - } + SortOrder.ALPHABETICAL -> "[title]=asc" + SortOrder.NEWEST -> "[createdAt]=desc" + SortOrder.POPULARITY -> "[followedCount]=desc" + else -> "[followedCount]=desc" + }, + ) + } + val json = webClient.httpGet(url).parseJson().getJSONArray("data") + return json.mapJSON { jo -> + val id = jo.getString("id") + val attrs = jo.getJSONObject("attributes") + val relations = jo.getJSONArray("relationships").associateByKey("type") + val cover = relations["cover_art"] + ?.getJSONObject("attributes") + ?.getString("fileName") + ?.let { + "https://uploads.$domain/covers/$id/$it" + } + Manga( + id = generateUid(id), + title = requireNotNull(attrs.getJSONObject("title").selectByLocale()) { + "Title should not be null" + }, + altTitle = attrs.optJSONObject("altTitles")?.selectByLocale(), + url = id, + publicUrl = "https://$domain/title/$id", + rating = RATING_UNKNOWN, + isNsfw = attrs.getStringOrNull("contentRating") == "erotica", + coverUrl = cover?.plus(".256.jpg").orEmpty(), + largeCoverUrl = cover, + description = attrs.optJSONObject("description")?.selectByLocale(), + tags = attrs.getJSONArray("tags").mapJSONToSet { tag -> + MangaTag( + title = tag.getJSONObject("attributes") + .getJSONObject("name") + .firstStringValue() + .toTitleCase(), + key = tag.getString("id"), + source = source, + ) + }, + state = when (jo.getStringOrNull("status")) { + "ongoing" -> MangaState.ONGOING + "completed" -> MangaState.FINISHED + else -> null + }, + author = (relations["author"] ?: relations["artist"]) + ?.getJSONObject("attributes") + ?.getStringOrNull("name"), + source = source, + ) + } + } - override suspend fun getDetails(manga: Manga): Manga = coroutineScope { - val domain = domain - val mangaId = manga.url.removePrefix("/") - val attrsDeferred = async { - webClient.httpGet( - "https://api.$domain/manga/${mangaId}?includes[]=artist&includes[]=author&includes[]=cover_art", - ).parseJson().getJSONObject("data").getJSONObject("attributes") - } - val feedDeferred = async { loadChapters(mangaId) } - val mangaAttrs = attrsDeferred.await() - val feed = feedDeferred.await() - // 2022-01-02T00:27:11+00:00 - val dateFormat = SimpleDateFormat( - "yyyy-MM-dd'T'HH:mm:ss'+00:00'", - Locale.ROOT, - ) - manga.copy( - description = mangaAttrs.optJSONObject("description")?.selectByLocale() - ?: manga.description, - chapters = feed.mapChapters { _, jo -> - val id = jo.getString("id") - val attrs = jo.getJSONObject("attributes") - if (!attrs.isNull("externalUrl")) { - return@mapChapters null - } - val locale = attrs.getStringOrNull("translatedLanguage")?.let { Locale.forLanguageTag(it) } - val relations = jo.getJSONArray("relationships").associateByKey("type") - val number = attrs.getIntOrDefault("chapter", 0) - MangaChapter( - id = generateUid(id), - name = attrs.getStringOrNull("title")?.takeUnless(String::isEmpty) - ?: "Chapter #$number", - number = number, - url = id, - scanlator = relations["scanlation_group"]?.getStringOrNull("name"), - uploadDate = dateFormat.tryParse(attrs.getString("publishAt")), - branch = locale?.getDisplayName(locale)?.toTitleCase(locale), - source = source, - ) - }, - ) - } + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { + val domain = domain + val mangaId = manga.url.removePrefix("/") + val attrsDeferred = async { + webClient.httpGet( + "https://api.$domain/manga/${mangaId}?includes[]=artist&includes[]=author&includes[]=cover_art", + ).parseJson().getJSONObject("data").getJSONObject("attributes") + } + val feedDeferred = async { loadChapters(mangaId) } + val mangaAttrs = attrsDeferred.await() + val feed = feedDeferred.await() + // 2022-01-02T00:27:11+00:00 + val dateFormat = SimpleDateFormat( + "yyyy-MM-dd'T'HH:mm:ss'+00:00'", + Locale.ROOT, + ) + manga.copy( + description = mangaAttrs.optJSONObject("description")?.selectByLocale() + ?: manga.description, + chapters = feed.mapChapters { _, jo -> + val id = jo.getString("id") + val attrs = jo.getJSONObject("attributes") + if (!attrs.isNull("externalUrl")) { + return@mapChapters null + } + val locale = attrs.getStringOrNull("translatedLanguage")?.let { Locale.forLanguageTag(it) } + val relations = jo.getJSONArray("relationships").associateByKey("type") + val number = attrs.getIntOrDefault("chapter", 0) + MangaChapter( + id = generateUid(id), + name = attrs.getStringOrNull("title")?.takeUnless(String::isEmpty) + ?: "Chapter #$number", + number = number, + url = id, + scanlator = relations["scanlation_group"]?.getStringOrNull("name"), + uploadDate = dateFormat.tryParse(attrs.getString("publishAt")), + branch = locale?.getDisplayName(locale)?.toTitleCase(locale), + source = source, + ) + }, + ) + } - override suspend fun getPages(chapter: MangaChapter): List { - val domain = domain - val chapterJson = webClient.httpGet("https://api.$domain/at-home/server/${chapter.url}?forcePort443=false") - .parseJson() - .getJSONObject("chapter") - val pages = chapterJson.getJSONArray("data") - val prefix = "https://uploads.$domain/data/${chapterJson.getString("hash")}/" - val referer = "https://$domain/" - return List(pages.length()) { i -> - val url = prefix + pages.getString(i) - MangaPage( - id = generateUid(url), - url = url, - referer = referer, - preview = null, // TODO prefix + dataSaver.getString(i), - source = source, - ) - } - } + override suspend fun getPages(chapter: MangaChapter): List { + val domain = domain + val chapterJson = webClient.httpGet("https://api.$domain/at-home/server/${chapter.url}?forcePort443=false") + .parseJson() + .getJSONObject("chapter") + val pages = chapterJson.getJSONArray("data") + val prefix = "https://uploads.$domain/data/${chapterJson.getString("hash")}/" + return List(pages.length()) { i -> + val url = prefix + pages.getString(i) + MangaPage( + id = generateUid(url), + url = url, + preview = null, // TODO prefix + dataSaver.getString(i), + source = source, + ) + } + } - override suspend fun getTags(): Set { - val tags = webClient.httpGet("https://api.${domain}/manga/tag").parseJson() - .getJSONArray("data") - return tags.mapJSONToSet { jo -> - MangaTag( - title = jo.getJSONObject("attributes").getJSONObject("name").firstStringValue().toTitleCase(), - key = jo.getString("id"), - source = source, - ) - } - } + override suspend fun getTags(): Set { + val tags = webClient.httpGet("https://api.${domain}/manga/tag").parseJson() + .getJSONArray("data") + return tags.mapJSONToSet { jo -> + MangaTag( + title = jo.getJSONObject("attributes").getJSONObject("name").firstStringValue().toTitleCase(), + key = jo.getString("id"), + source = source, + ) + } + } - private fun JSONObject.firstStringValue() = values().next() as String + private fun JSONObject.firstStringValue() = values().next() as String - private fun JSONObject.selectByLocale(): String? { - val preferredLocales = context.getPreferredLocales() - for (locale in preferredLocales) { - getStringOrNull(locale.language)?.let { return it } - getStringOrNull(locale.toLanguageTag())?.let { return it } - } - return getStringOrNull(LOCALE_FALLBACK) ?: values().nextOrNull() as? String - } + private fun JSONObject.selectByLocale(): String? { + val preferredLocales = context.getPreferredLocales() + for (locale in preferredLocales) { + getStringOrNull(locale.language)?.let { return it } + getStringOrNull(locale.toLanguageTag())?.let { return it } + } + return getStringOrNull(LOCALE_FALLBACK) ?: values().nextOrNull() as? String + } - private suspend fun loadChapters(mangaId: String): List { - val firstPage = loadChapters(mangaId, offset = 0, limit = CHAPTERS_FIRST_PAGE_SIZE) - if (firstPage.size >= firstPage.total) { - return firstPage.data - } - val tail = coroutineScope { - val leftCount = firstPage.total.coerceAtMost(CHAPTERS_MAX_COUNT) - firstPage.size - val pages = (leftCount / CHAPTERS_MAX_PAGE_SIZE.toFloat()).toIntUp() - val dispatcher = Dispatchers.Default.limitedParallelism(CHAPTERS_PARALLELISM) - List(pages) { page -> - val offset = page * CHAPTERS_MAX_PAGE_SIZE + firstPage.size - async(dispatcher) { - loadChapters(mangaId, offset, CHAPTERS_MAX_PAGE_SIZE) - } - }.awaitAll() - } - val result = ArrayList(firstPage.total) - result += firstPage.data - tail.flatMapTo(result) { it.data } - return result - } + private suspend fun loadChapters(mangaId: String): List { + val firstPage = loadChapters(mangaId, offset = 0, limit = CHAPTERS_FIRST_PAGE_SIZE) + if (firstPage.size >= firstPage.total) { + return firstPage.data + } + val tail = coroutineScope { + val leftCount = firstPage.total.coerceAtMost(CHAPTERS_MAX_COUNT) - firstPage.size + val pages = (leftCount / CHAPTERS_MAX_PAGE_SIZE.toFloat()).toIntUp() + val dispatcher = Dispatchers.Default.limitedParallelism(CHAPTERS_PARALLELISM) + List(pages) { page -> + val offset = page * CHAPTERS_MAX_PAGE_SIZE + firstPage.size + async(dispatcher) { + loadChapters(mangaId, offset, CHAPTERS_MAX_PAGE_SIZE) + } + }.awaitAll() + } + val result = ArrayList(firstPage.total) + result += firstPage.data + tail.flatMapTo(result) { it.data } + return result + } - private suspend fun loadChapters(mangaId: String, offset: Int, limit: Int): Chapters { - val limitedLimit = when { - offset >= CHAPTERS_MAX_COUNT -> return Chapters(emptyList(), CHAPTERS_MAX_COUNT) - offset + limit > CHAPTERS_MAX_COUNT -> CHAPTERS_MAX_COUNT - offset - else -> limit - } - val url = buildString { - append("https://api.") - append(domain) - append("/manga/") - append(mangaId) - append("/feed") - append("?limit=") - append(limitedLimit) - append("&includes[]=scanlation_group&order[volume]=asc&order[chapter]=asc&offset=") - append(offset) - append('&') - append(CONTENT_RATING) - } - val json = webClient.httpGet(url).parseJson() - if (json.getString("result") == "ok") { - return Chapters( - data = json.optJSONArray("data")?.toJSONList().orEmpty(), - total = json.getInt("total"), - ) - } else { - val error = json.optJSONArray("errors").mapJSON { jo -> - jo.getString("detail") - }.joinToString("\n") - throw ParseException(error, url) - } - } + private suspend fun loadChapters(mangaId: String, offset: Int, limit: Int): Chapters { + val limitedLimit = when { + offset >= CHAPTERS_MAX_COUNT -> return Chapters(emptyList(), CHAPTERS_MAX_COUNT) + offset + limit > CHAPTERS_MAX_COUNT -> CHAPTERS_MAX_COUNT - offset + else -> limit + } + val url = buildString { + append("https://api.") + append(domain) + append("/manga/") + append(mangaId) + append("/feed") + append("?limit=") + append(limitedLimit) + append("&includes[]=scanlation_group&order[volume]=asc&order[chapter]=asc&offset=") + append(offset) + append('&') + append(CONTENT_RATING) + } + val json = webClient.httpGet(url).parseJson() + if (json.getString("result") == "ok") { + return Chapters( + data = json.optJSONArray("data")?.toJSONList().orEmpty(), + total = json.getInt("total"), + ) + } else { + val error = json.optJSONArray("errors").mapJSON { jo -> + jo.getString("detail") + }.joinToString("\n") + throw ParseException(error, url) + } + } - private class Chapters( - val data: List, - val total: Int, - ) { + private class Chapters( + val data: List, + val total: Int, + ) { - val size: Int - get() = data.size - } + val size: Int + get() = data.size + } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaInUaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaInUaParser.kt index 68726425..549fa3f1 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaInUaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaInUaParser.kt @@ -13,141 +13,140 @@ private const val DEF_BRANCH_NAME = "Основний переклад" @MangaSourceParser("MANGAINUA", "MANGA/in/UA", "uk") class MangaInUaParser(context: MangaLoaderContext) : PagedMangaParser( - context = context, - source = MangaSource.MANGAINUA, - pageSize = 24, - searchPageSize = 10, + context = context, + source = MangaSource.MANGAINUA, + pageSize = 24, + searchPageSize = 10, ) { - override val sortOrders: Set - get() = Collections.singleton(SortOrder.UPDATED) + override val sortOrders: Set + get() = Collections.singleton(SortOrder.UPDATED) - override val configKeyDomain: ConfigKey.Domain = ConfigKey.Domain("manga.in.ua", null) + override val configKeyDomain: ConfigKey.Domain = ConfigKey.Domain("manga.in.ua", null) - override suspend fun getListPage( - page: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - val url = when { - !query.isNullOrEmpty() -> ( - "/index.php?do=search" + - "&subaction=search" + - "&search_start=$page" + - "&full_search=1" + - "&story=$query" + - "&titleonly=3" - ).toAbsoluteUrl(domain) + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + val url = when { + !query.isNullOrEmpty() -> ( + "/index.php?do=search" + + "&subaction=search" + + "&search_start=$page" + + "&full_search=1" + + "&story=$query" + + "&titleonly=3" + ).toAbsoluteUrl(domain) - tags.isNullOrEmpty() -> "/mangas/page/$page".toAbsoluteUrl(domain) - tags.size == 1 -> "${tags.first().key}/page/$page" - tags.size > 1 -> throw IllegalArgumentException("This source supports only 1 genre") - else -> "/mangas/page/$page".toAbsoluteUrl(domain) - } - val doc = webClient.httpGet(url).parseHtml() - val container = doc.body().requireElementById("site-content") - val items = container.select("div.col-6") - return items.mapNotNull { item -> - val href = item.selectFirst("a")?.attrAsRelativeUrl("href") ?: return@mapNotNull null - Manga( - id = generateUid(href), - title = item.selectFirst("h3.card__title")?.text() ?: return@mapNotNull null, - coverUrl = item.selectFirst("header.card__cover")?.selectFirst("img")?.run { - attrAsAbsoluteUrlOrNull("data-src") ?: attrAsAbsoluteUrlOrNull("src") - }.orEmpty(), - altTitle = null, - author = null, - rating = item.selectFirst("div.card__short-rate--num") - ?.text() - ?.toFloatOrNull() - ?.div(10F) ?: RATING_UNKNOWN, - url = href, - isNsfw = item.selectFirst("ul.card__list")?.select("li")?.lastOrNull()?.text() == "18+", - tags = runCatching { - item.selectFirst("div.card__category")?.select("a")?.mapToSet { - MangaTag( - title = it.ownText(), - key = it.attr("href").removeSuffix("/"), - source = source, - ) - } - }.getOrNull().orEmpty(), - state = null, - publicUrl = href.toAbsoluteUrl(container.host ?: domain), - source = source, - ) - } - } + tags.isNullOrEmpty() -> "/mangas/page/$page".toAbsoluteUrl(domain) + tags.size == 1 -> "${tags.first().key}/page/$page" + tags.size > 1 -> throw IllegalArgumentException("This source supports only 1 genre") + else -> "/mangas/page/$page".toAbsoluteUrl(domain) + } + val doc = webClient.httpGet(url).parseHtml() + val container = doc.body().requireElementById("site-content") + val items = container.select("div.col-6") + return items.mapNotNull { item -> + val href = item.selectFirst("a")?.attrAsRelativeUrl("href") ?: return@mapNotNull null + Manga( + id = generateUid(href), + title = item.selectFirst("h3.card__title")?.text() ?: return@mapNotNull null, + coverUrl = item.selectFirst("header.card__cover")?.selectFirst("img")?.run { + attrAsAbsoluteUrlOrNull("data-src") ?: attrAsAbsoluteUrlOrNull("src") + }.orEmpty(), + altTitle = null, + author = null, + rating = item.selectFirst("div.card__short-rate--num") + ?.text() + ?.toFloatOrNull() + ?.div(10F) ?: RATING_UNKNOWN, + url = href, + isNsfw = item.selectFirst("ul.card__list")?.select("li")?.lastOrNull()?.text() == "18+", + tags = runCatching { + item.selectFirst("div.card__category")?.select("a")?.mapToSet { + MangaTag( + title = it.ownText(), + key = it.attr("href").removeSuffix("/"), + source = source, + ) + } + }.getOrNull().orEmpty(), + state = null, + publicUrl = href.toAbsoluteUrl(container.host ?: domain), + source = source, + ) + } + } - override suspend fun getDetails(manga: Manga): Manga { - val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - val root = doc.body().requireElementById("site-content") - val dateFormat = SimpleDateFormat("dd.MM.yyyy", Locale.US) - val chapterNodes = root.selectFirstOrThrow(".linkstocomics").select(".ltcitems") - var prevChapterName: String? = null - var i = 0 - return manga.copy( - description = root.selectFirst("div.item__full-description")?.text(), - largeCoverUrl = root.selectFirst("div.item__full-sidebar--poster")?.selectFirst("img") - ?.attrAsAbsoluteUrlOrNull("src"), - chapters = chapterNodes.mapChapters { _, item -> - val href = item?.selectFirst("a")?.attrAsRelativeUrlOrNull("href") - ?: return@mapChapters null - val isAlternative = item.styleValueOrNull("background") != null - val name = item.selectFirst("a")?.text().orEmpty() - if (!isAlternative) i++ - MangaChapter( - id = generateUid(href), - name = if (isAlternative) { - prevChapterName ?: return@mapChapters null - } else { - prevChapterName = name - name - }, - number = i, - url = href, - scanlator = null, - branch = if (isAlternative) { - name.substringAfterLast(':').trim() - } else { - DEF_BRANCH_NAME - }, - uploadDate = dateFormat.tryParse(item.selectFirst("div.ltcright")?.text()), - source = source, - ) - }, - ) - } + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val root = doc.body().requireElementById("site-content") + val dateFormat = SimpleDateFormat("dd.MM.yyyy", Locale.US) + val chapterNodes = root.selectFirstOrThrow(".linkstocomics").select(".ltcitems") + var prevChapterName: String? = null + var i = 0 + return manga.copy( + description = root.selectFirst("div.item__full-description")?.text(), + largeCoverUrl = root.selectFirst("div.item__full-sidebar--poster")?.selectFirst("img") + ?.attrAsAbsoluteUrlOrNull("src"), + chapters = chapterNodes.mapChapters { _, item -> + val href = item?.selectFirst("a")?.attrAsRelativeUrlOrNull("href") + ?: return@mapChapters null + val isAlternative = item.styleValueOrNull("background") != null + val name = item.selectFirst("a")?.text().orEmpty() + if (!isAlternative) i++ + MangaChapter( + id = generateUid(href), + name = if (isAlternative) { + prevChapterName ?: return@mapChapters null + } else { + prevChapterName = name + name + }, + number = i, + url = href, + scanlator = null, + branch = if (isAlternative) { + name.substringAfterLast(':').trim() + } else { + DEF_BRANCH_NAME + }, + uploadDate = dateFormat.tryParse(item.selectFirst("div.ltcright")?.text()), + source = source, + ) + }, + ) + } - override suspend fun getPages(chapter: MangaChapter): List { - val fullUrl = chapter.url.toAbsoluteUrl(domain) - val doc = webClient.httpGet(fullUrl).parseHtml() - val root = doc.body().requireElementById("comics").selectFirstOrThrow("ul.xfieldimagegallery") - return root.select("li").map { ul -> - val img = ul.selectFirstOrThrow("img") - val url = img.attrAsAbsoluteUrl("data-src") - MangaPage( - id = generateUid(url), - url = url, - preview = null, - referer = fullUrl, - source = source, - ) - } - } + override suspend fun getPages(chapter: MangaChapter): List { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + val root = doc.body().requireElementById("comics").selectFirstOrThrow("ul.xfieldimagegallery") + return root.select("li").map { ul -> + val img = ul.selectFirstOrThrow("img") + val url = img.attrAsAbsoluteUrl("data-src") + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } - override suspend fun getTags(): Set { - val domain = domain - val doc = webClient.httpGet("https://$domain/mangas").parseHtml() - val root = doc.body().requireElementById("menu_1").selectFirstOrThrow("div.menu__wrapper") - return root.select("li").mapNotNullToSet { li -> - val a = li.selectFirst("a") ?: return@mapNotNullToSet null - MangaTag( - title = a.ownText(), - key = a.attr("href").removeSuffix("/"), - source = source, - ) - } - } + override suspend fun getTags(): Set { + val domain = domain + val doc = webClient.httpGet("https://$domain/mangas").parseHtml() + val root = doc.body().requireElementById("menu_1").selectFirstOrThrow("div.menu__wrapper") + return root.select("li").mapNotNullToSet { li -> + val a = li.selectFirst("a") ?: return@mapNotNullToSet null + MangaTag( + title = a.ownText(), + key = a.attr("href").removeSuffix("/"), + source = source, + ) + } + } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaTownParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaTownParser.kt index dbc5ee62..60f8de5e 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaTownParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/MangaTownParser.kt @@ -13,205 +13,204 @@ import java.util.* @MangaSourceParser("MANGATOWN", "MangaTown", "en") internal class MangaTownParser(context: MangaLoaderContext) : MangaParser(context, MangaSource.MANGATOWN) { - override val configKeyDomain = ConfigKey.Domain("www.mangatown.com", null) + override val configKeyDomain = ConfigKey.Domain("www.mangatown.com", null) - override val sortOrders: Set = EnumSet.of( - SortOrder.ALPHABETICAL, - SortOrder.RATING, - SortOrder.POPULARITY, - SortOrder.UPDATED, - ) + override val sortOrders: Set = EnumSet.of( + SortOrder.ALPHABETICAL, + SortOrder.RATING, + SortOrder.POPULARITY, + SortOrder.UPDATED, + ) - private val regexTag = Regex("[^\\-]+-[^\\-]+-[^\\-]+-[^\\-]+-[^\\-]+-[^\\-]+") + private val regexTag = Regex("[^\\-]+-[^\\-]+-[^\\-]+-[^\\-]+-[^\\-]+-[^\\-]+") - override suspend fun getList( - offset: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - val sortKey = when (sortOrder) { - SortOrder.ALPHABETICAL -> "?name.az" - SortOrder.RATING -> "?rating.za" - SortOrder.UPDATED -> "?last_chapter_time.za" - else -> "" - } - val page = (offset / 30) + 1 - val url = when { - !query.isNullOrEmpty() -> { - if (offset != 0) { - return emptyList() - } - "/search?name=${query.urlEncoded()}".toAbsoluteUrl(domain) - } + override suspend fun getList( + offset: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + val sortKey = when (sortOrder) { + SortOrder.ALPHABETICAL -> "?name.az" + SortOrder.RATING -> "?rating.za" + SortOrder.UPDATED -> "?last_chapter_time.za" + else -> "" + } + val page = (offset / 30) + 1 + val url = when { + !query.isNullOrEmpty() -> { + if (offset != 0) { + return emptyList() + } + "/search?name=${query.urlEncoded()}".toAbsoluteUrl(domain) + } - tags.isNullOrEmpty() -> "/directory/$page.htm$sortKey".toAbsoluteUrl(domain) - tags.size == 1 -> "/directory/${tags.first().key}/$page.htm$sortKey".toAbsoluteUrl(domain) - else -> tags.joinToString( - prefix = "/search?page=$page".toAbsoluteUrl(domain), - ) { tag -> - "&genres[${tag.key}]=1" - } - } - val doc = webClient.httpGet(url).parseHtml() - val root = doc.body().selectFirstOrThrow("ul.manga_pic_list") - return root.select("li").mapNotNull { li -> - val a = li.selectFirst("a.manga_cover") - val href = a?.attrAsRelativeUrlOrNull("href") - ?: return@mapNotNull null - val views = li.select("p.view") - val status = views.firstNotNullOfOrNull { it.ownText().takeIf { x -> x.startsWith("Status:") } } - ?.substringAfter(':')?.trim()?.lowercase(Locale.ROOT) - Manga( - id = generateUid(href), - title = a.attr("title"), - coverUrl = a.selectFirst("img")?.absUrl("src").orEmpty(), - source = MangaSource.MANGATOWN, - altTitle = null, - rating = li.selectFirst("p.score")?.selectFirst("b") - ?.ownText()?.toFloatOrNull()?.div(5f) ?: RATING_UNKNOWN, - author = views.firstNotNullOfOrNull { it.text().takeIf { x -> x.startsWith("Author:") } } - ?.substringAfter(':') - ?.trim(), - state = when (status) { - "ongoing" -> MangaState.ONGOING - "completed" -> MangaState.FINISHED - else -> null - }, - tags = li.selectFirst("p.keyWord")?.select("a")?.mapNotNullToSet tags@{ x -> - MangaTag( - title = x.attr("title").toTitleCase(), - key = x.attr("href").parseTagKey() ?: return@tags null, - source = MangaSource.MANGATOWN, - ) - }.orEmpty(), - url = href, - isNsfw = false, - publicUrl = href.toAbsoluteUrl(a.host ?: domain), - ) - } - } + tags.isNullOrEmpty() -> "/directory/$page.htm$sortKey".toAbsoluteUrl(domain) + tags.size == 1 -> "/directory/${tags.first().key}/$page.htm$sortKey".toAbsoluteUrl(domain) + else -> tags.joinToString( + prefix = "/search?page=$page".toAbsoluteUrl(domain), + ) { tag -> + "&genres[${tag.key}]=1" + } + } + val doc = webClient.httpGet(url).parseHtml() + val root = doc.body().selectFirstOrThrow("ul.manga_pic_list") + return root.select("li").mapNotNull { li -> + val a = li.selectFirst("a.manga_cover") + val href = a?.attrAsRelativeUrlOrNull("href") + ?: return@mapNotNull null + val views = li.select("p.view") + val status = views.firstNotNullOfOrNull { it.ownText().takeIf { x -> x.startsWith("Status:") } } + ?.substringAfter(':')?.trim()?.lowercase(Locale.ROOT) + Manga( + id = generateUid(href), + title = a.attr("title"), + coverUrl = a.selectFirst("img")?.absUrl("src").orEmpty(), + source = MangaSource.MANGATOWN, + altTitle = null, + rating = li.selectFirst("p.score")?.selectFirst("b") + ?.ownText()?.toFloatOrNull()?.div(5f) ?: RATING_UNKNOWN, + author = views.firstNotNullOfOrNull { it.text().takeIf { x -> x.startsWith("Author:") } } + ?.substringAfter(':') + ?.trim(), + state = when (status) { + "ongoing" -> MangaState.ONGOING + "completed" -> MangaState.FINISHED + else -> null + }, + tags = li.selectFirst("p.keyWord")?.select("a")?.mapNotNullToSet tags@{ x -> + MangaTag( + title = x.attr("title").toTitleCase(), + key = x.attr("href").parseTagKey() ?: return@tags null, + source = MangaSource.MANGATOWN, + ) + }.orEmpty(), + url = href, + isNsfw = false, + publicUrl = href.toAbsoluteUrl(a.host ?: domain), + ) + } + } - override suspend fun getDetails(manga: Manga): Manga { - val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - val root = doc.body().selectFirstOrThrow("section.main") - .selectFirstOrThrow("div.article_content") - val info = root.selectFirst("div.detail_info")?.selectFirst("ul") - val chaptersList = root.selectFirst("div.chapter_content") - ?.selectFirst("ul.chapter_list")?.select("li")?.asReversed() - val dateFormat = SimpleDateFormat("MMM dd,yyyy", Locale.US) - return manga.copy( - tags = manga.tags + info?.select("li")?.find { x -> - x.selectFirst("b")?.ownText() == "Genre(s):" - }?.select("a")?.mapNotNull { a -> - MangaTag( - title = a.attr("title").toTitleCase(), - key = a.attr("href").parseTagKey() ?: return@mapNotNull null, - source = MangaSource.MANGATOWN, - ) - }.orEmpty(), - description = info?.getElementById("show")?.ownText(), - chapters = chaptersList?.mapChapters { i, li -> - val href = li.selectFirst("a")?.attrAsRelativeUrlOrNull("href") - ?: return@mapChapters null - val name = li.select("span") - .filter { x -> x.className().isEmpty() } - .joinToString(" - ") { it.text() }.trim() - MangaChapter( - id = generateUid(href), - url = href, - source = MangaSource.MANGATOWN, - number = i + 1, - uploadDate = parseChapterDate( - dateFormat, - li.selectFirst("span.time")?.text(), - ), - name = name.ifEmpty { "${manga.title} - ${i + 1}" }, - scanlator = null, - branch = null, - ) - } ?: bypassLicensedChapters(manga), - ) - } + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val root = doc.body().selectFirstOrThrow("section.main") + .selectFirstOrThrow("div.article_content") + val info = root.selectFirst("div.detail_info")?.selectFirst("ul") + val chaptersList = root.selectFirst("div.chapter_content") + ?.selectFirst("ul.chapter_list")?.select("li")?.asReversed() + val dateFormat = SimpleDateFormat("MMM dd,yyyy", Locale.US) + return manga.copy( + tags = manga.tags + info?.select("li")?.find { x -> + x.selectFirst("b")?.ownText() == "Genre(s):" + }?.select("a")?.mapNotNull { a -> + MangaTag( + title = a.attr("title").toTitleCase(), + key = a.attr("href").parseTagKey() ?: return@mapNotNull null, + source = MangaSource.MANGATOWN, + ) + }.orEmpty(), + description = info?.getElementById("show")?.ownText(), + chapters = chaptersList?.mapChapters { i, li -> + val href = li.selectFirst("a")?.attrAsRelativeUrlOrNull("href") + ?: return@mapChapters null + val name = li.select("span") + .filter { x -> x.className().isEmpty() } + .joinToString(" - ") { it.text() }.trim() + MangaChapter( + id = generateUid(href), + url = href, + source = MangaSource.MANGATOWN, + number = i + 1, + uploadDate = parseChapterDate( + dateFormat, + li.selectFirst("span.time")?.text(), + ), + name = name.ifEmpty { "${manga.title} - ${i + 1}" }, + scanlator = null, + branch = null, + ) + } ?: bypassLicensedChapters(manga), + ) + } - override suspend fun getPages(chapter: MangaChapter): List { - val fullUrl = chapter.url.toAbsoluteUrl(domain) - val doc = webClient.httpGet(fullUrl).parseHtml() - val root = doc.body().selectFirstOrThrow("div.page_select") - return root.selectFirstOrThrow("select").selectOrThrow("option").mapNotNull { - val href = it.attrAsRelativeUrlOrNull("value") - if (href == null || href.endsWith("featured.html")) { - return@mapNotNull null - } - MangaPage( - id = generateUid(href), - url = href, - preview = null, - referer = fullUrl, - source = MangaSource.MANGATOWN, - ) - } - } + override suspend fun getPages(chapter: MangaChapter): List { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + val root = doc.body().selectFirstOrThrow("div.page_select") + return root.selectFirstOrThrow("select").selectOrThrow("option").mapNotNull { + val href = it.attrAsRelativeUrlOrNull("value") + if (href == null || href.endsWith("featured.html")) { + return@mapNotNull null + } + MangaPage( + id = generateUid(href), + url = href, + preview = null, + source = MangaSource.MANGATOWN, + ) + } + } - override suspend fun getPageUrl(page: MangaPage): String { - val doc = webClient.httpGet(page.url.toAbsoluteUrl(domain)).parseHtml() - return doc.requireElementById("image").attrAsAbsoluteUrl("src") - } + override suspend fun getPageUrl(page: MangaPage): String { + val doc = webClient.httpGet(page.url.toAbsoluteUrl(domain)).parseHtml() + return doc.requireElementById("image").attrAsAbsoluteUrl("src") + } - override suspend fun getTags(): Set { - val doc = webClient.httpGet("/directory/".toAbsoluteUrl(domain)).parseHtml() - val root = doc.body().selectFirst("aside.right") - ?.getElementsContainingOwnText("Genres") - ?.first() - ?.nextElementSibling() ?: doc.parseFailed("Root not found") - return root.select("li").mapNotNullToSet { li -> - val a = li.selectFirst("a") ?: return@mapNotNullToSet null - val key = a.attr("href").parseTagKey() - if (key.isNullOrEmpty()) { - return@mapNotNullToSet null - } - MangaTag( - source = MangaSource.MANGATOWN, - key = key, - title = a.text().toTitleCase(), - ) - } - } + override suspend fun getTags(): Set { + val doc = webClient.httpGet("/directory/".toAbsoluteUrl(domain)).parseHtml() + val root = doc.body().selectFirst("aside.right") + ?.getElementsContainingOwnText("Genres") + ?.first() + ?.nextElementSibling() ?: doc.parseFailed("Root not found") + return root.select("li").mapNotNullToSet { li -> + val a = li.selectFirst("a") ?: return@mapNotNullToSet null + val key = a.attr("href").parseTagKey() + if (key.isNullOrEmpty()) { + return@mapNotNullToSet null + } + MangaTag( + source = MangaSource.MANGATOWN, + key = key, + title = a.text().toTitleCase(), + ) + } + } - private fun parseChapterDate(dateFormat: DateFormat, date: String?): Long { - return when { - date.isNullOrEmpty() -> 0L - date.contains("Today") -> Calendar.getInstance().timeInMillis - date.contains("Yesterday") -> Calendar.getInstance().apply { add(Calendar.DAY_OF_MONTH, -1) }.timeInMillis - else -> dateFormat.tryParse(date) - } - } + private fun parseChapterDate(dateFormat: DateFormat, date: String?): Long { + return when { + date.isNullOrEmpty() -> 0L + date.contains("Today") -> Calendar.getInstance().timeInMillis + date.contains("Yesterday") -> Calendar.getInstance().apply { add(Calendar.DAY_OF_MONTH, -1) }.timeInMillis + else -> dateFormat.tryParse(date) + } + } - private suspend fun bypassLicensedChapters(manga: Manga): List { - val doc = webClient.httpGet(manga.url.toAbsoluteUrl(getDomain("m"))).parseHtml() - val list = doc.body().selectFirst("ul.detail-ch-list") ?: return emptyList() - val dateFormat = SimpleDateFormat("MMM dd,yyyy", Locale.US) - return list.select("li").asReversed().mapIndexedNotNull { i, li -> - val a = li.selectFirst("a") ?: return@mapIndexedNotNull null - val href = a.attrAsRelativeUrl("href") - val name = a.selectFirst("span.vol")?.text().orEmpty().ifEmpty { - a.ownText() - } - MangaChapter( - id = generateUid(href), - url = href, - source = MangaSource.MANGATOWN, - number = i + 1, - uploadDate = parseChapterDate( - dateFormat, - li.selectFirst("span.time")?.text(), - ), - name = name.ifEmpty { "${manga.title} - ${i + 1}" }, - scanlator = null, - branch = null, - ) - } - } + private suspend fun bypassLicensedChapters(manga: Manga): List { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(getDomain("m"))).parseHtml() + val list = doc.body().selectFirst("ul.detail-ch-list") ?: return emptyList() + val dateFormat = SimpleDateFormat("MMM dd,yyyy", Locale.US) + return list.select("li").asReversed().mapIndexedNotNull { i, li -> + val a = li.selectFirst("a") ?: return@mapIndexedNotNull null + val href = a.attrAsRelativeUrl("href") + val name = a.selectFirst("span.vol")?.text().orEmpty().ifEmpty { + a.ownText() + } + MangaChapter( + id = generateUid(href), + url = href, + source = MangaSource.MANGATOWN, + number = i + 1, + uploadDate = parseChapterDate( + dateFormat, + li.selectFirst("span.time")?.text(), + ), + name = name.ifEmpty { "${manga.title} - ${i + 1}" }, + scanlator = null, + branch = null, + ) + } + } - private fun String.parseTagKey() = split('/').findLast { regexTag matches it } + private fun String.parseTagKey() = split('/').findLast { regexTag matches it } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/Manhwa18Parser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/Manhwa18Parser.kt index 55bc37cd..93d1e214 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/Manhwa18Parser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/Manhwa18Parser.kt @@ -165,7 +165,6 @@ class Manhwa18Parser(context: MangaLoaderContext) : MangaPage( id = generateUid(url), url = url, - referer = chapterUrl, preview = null, source = MangaSource.MANHWA18, ) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NHentaiParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NHentaiParser.kt index 5ca2f515..8c1c53be 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NHentaiParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NHentaiParser.kt @@ -17,172 +17,171 @@ import java.util.* @MangaSourceParser("NHENTAI", "N-Hentai") class NHentaiParser(context: MangaLoaderContext) : PagedMangaParser(context, MangaSource.NHENTAI, pageSize = 25) { - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("nhentai.net", null) + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("nhentai.net", null) - override val sortOrders: Set - get() = EnumSet.of(SortOrder.NEWEST, SortOrder.POPULARITY) + override val sortOrders: Set + get() = EnumSet.of(SortOrder.NEWEST, SortOrder.POPULARITY) - override suspend fun getListPage( - page: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - if (query.isNullOrEmpty() && tags != null && tags.size > 1) { - return getListPage(page, buildQuery(tags), emptySet(), sortOrder) - } - val domain = domain - val url = buildString { - append("https://") - append(domain) - if (!query.isNullOrEmpty()) { - append("/search/?q=") - append(query.urlEncoded()) - append("&page=") - append(page) - if (sortOrder == SortOrder.POPULARITY) { - append("&sort=popular") - } - } else { - append('/') - if (!tags.isNullOrEmpty()) { - val tag = tags.single() - append("tag/") - append(tag.key) - append('/') - if (sortOrder == SortOrder.POPULARITY) { - append("popular") - } - append("?page=") - append(page) - } else { - if (sortOrder == SortOrder.POPULARITY) { - append("?sort=popular&page=") - } else { - append("?page=") - } - append(page) - } - } - } - val root = webClient.httpGet(url).parseHtml().body().requireElementById("content") - .selectLastOrThrow("div.index-container") - val regexBrackets = Regex("\\[[^]]+]|\\([^)]+\\)") - val regexSpaces = Regex("\\s+") - return root.select(".gallery").map { div -> - val a = div.selectFirstOrThrow("a.cover") - val href = a.attrAsRelativeUrl("href") - val img = div.selectFirstOrThrow("img") - val title = div.selectFirstOrThrow(".caption").text() - Manga( - id = generateUid(href), - title = title.replace(regexBrackets, "") - .replace(regexSpaces, " ") - .trim(), - altTitle = null, - url = href, - publicUrl = href.toAbsoluteUrl(domain), - rating = RATING_UNKNOWN, - isNsfw = true, - coverUrl = img.attrAsAbsoluteUrlOrNull("data-src") - ?: img.attrAsAbsoluteUrl("src"), - tags = setOf(), - state = null, - author = null, - largeCoverUrl = null, - description = null, - chapters = listOf(), - source = source, - ) - } - } + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + if (query.isNullOrEmpty() && tags != null && tags.size > 1) { + return getListPage(page, buildQuery(tags), emptySet(), sortOrder) + } + val domain = domain + val url = buildString { + append("https://") + append(domain) + if (!query.isNullOrEmpty()) { + append("/search/?q=") + append(query.urlEncoded()) + append("&page=") + append(page) + if (sortOrder == SortOrder.POPULARITY) { + append("&sort=popular") + } + } else { + append('/') + if (!tags.isNullOrEmpty()) { + val tag = tags.single() + append("tag/") + append(tag.key) + append('/') + if (sortOrder == SortOrder.POPULARITY) { + append("popular") + } + append("?page=") + append(page) + } else { + if (sortOrder == SortOrder.POPULARITY) { + append("?sort=popular&page=") + } else { + append("?page=") + } + append(page) + } + } + } + val root = webClient.httpGet(url).parseHtml().body().requireElementById("content") + .selectLastOrThrow("div.index-container") + val regexBrackets = Regex("\\[[^]]+]|\\([^)]+\\)") + val regexSpaces = Regex("\\s+") + return root.select(".gallery").map { div -> + val a = div.selectFirstOrThrow("a.cover") + val href = a.attrAsRelativeUrl("href") + val img = div.selectFirstOrThrow("img") + val title = div.selectFirstOrThrow(".caption").text() + Manga( + id = generateUid(href), + title = title.replace(regexBrackets, "") + .replace(regexSpaces, " ") + .trim(), + altTitle = null, + url = href, + publicUrl = href.toAbsoluteUrl(domain), + rating = RATING_UNKNOWN, + isNsfw = true, + coverUrl = img.attrAsAbsoluteUrlOrNull("data-src") + ?: img.attrAsAbsoluteUrl("src"), + tags = setOf(), + state = null, + author = null, + largeCoverUrl = null, + description = null, + chapters = listOf(), + source = source, + ) + } + } - override suspend fun getDetails(manga: Manga): Manga { - val root = webClient.httpGet( - url = manga.url.toAbsoluteUrl(domain), - ).parseHtml().body().requireElementById("bigcontainer") - val img = root.requireElementById("cover").selectFirstOrThrow("img") - val tagContainers = root.requireElementById("tags").select(".tag-container") - val dateFormat = SimpleDateFormat( - "yyyy-MM-dd'T'HH:mm:ss.SSSSSS'+00:00'", - Locale.ROOT, - ) - return manga.copy( - tags = tagContainers.find { x -> x.ownText() == "Tags:" }?.parseTags() ?: manga.tags, - author = tagContainers.find { x -> x.ownText() == "Artists:" } - ?.selectFirst("span.name")?.text()?.toCamelCase(), - largeCoverUrl = img.attrAsAbsoluteUrlOrNull("data-src") - ?: img.attrAsAbsoluteUrl("src"), - description = null, - chapters = listOf( - MangaChapter( - id = manga.id, - name = manga.title, - number = 1, - url = manga.url, - scanlator = null, - uploadDate = dateFormat.tryParse( - tagContainers.find { x -> x.ownText() == "Uploaded:" } - ?.selectFirst("time") - ?.attr("datetime"), - ), - branch = null, - source = source, - ), - ), - ) - } + override suspend fun getDetails(manga: Manga): Manga { + val root = webClient.httpGet( + url = manga.url.toAbsoluteUrl(domain), + ).parseHtml().body().requireElementById("bigcontainer") + val img = root.requireElementById("cover").selectFirstOrThrow("img") + val tagContainers = root.requireElementById("tags").select(".tag-container") + val dateFormat = SimpleDateFormat( + "yyyy-MM-dd'T'HH:mm:ss.SSSSSS'+00:00'", + Locale.ROOT, + ) + return manga.copy( + tags = tagContainers.find { x -> x.ownText() == "Tags:" }?.parseTags() ?: manga.tags, + author = tagContainers.find { x -> x.ownText() == "Artists:" } + ?.selectFirst("span.name")?.text()?.toCamelCase(), + largeCoverUrl = img.attrAsAbsoluteUrlOrNull("data-src") + ?: img.attrAsAbsoluteUrl("src"), + description = null, + chapters = listOf( + MangaChapter( + id = manga.id, + name = manga.title, + number = 1, + url = manga.url, + scanlator = null, + uploadDate = dateFormat.tryParse( + tagContainers.find { x -> x.ownText() == "Uploaded:" } + ?.selectFirst("time") + ?.attr("datetime"), + ), + branch = null, + source = source, + ), + ), + ) + } - override suspend fun getPages(chapter: MangaChapter): List { - val url = chapter.url.toAbsoluteUrl(domain) - val root = webClient.httpGet(url).parseHtml().requireElementById("thumbnail-container") - return root.select(".thumb-container").map { div -> - val a = div.selectFirstOrThrow("a") - val img = div.selectFirstOrThrow("img") - val href = a.attrAsRelativeUrl("href") - MangaPage( - id = generateUid(href), - url = href, - referer = url, - preview = img.attrAsAbsoluteUrlOrNull("data-src") - ?: img.attrAsAbsoluteUrl("src"), - source = source, - ) - } - } + override suspend fun getPages(chapter: MangaChapter): List { + val url = chapter.url.toAbsoluteUrl(domain) + val root = webClient.httpGet(url).parseHtml().requireElementById("thumbnail-container") + return root.select(".thumb-container").map { div -> + val a = div.selectFirstOrThrow("a") + val img = div.selectFirstOrThrow("img") + val href = a.attrAsRelativeUrl("href") + MangaPage( + id = generateUid(href), + url = href, + preview = img.attrAsAbsoluteUrlOrNull("data-src") + ?: img.attrAsAbsoluteUrl("src"), + source = source, + ) + } + } - override suspend fun getPageUrl(page: MangaPage): String { - val root = webClient.httpGet(page.url.toAbsoluteUrl(domain)).parseHtml().body() - .requireElementById("image-container") - return root.selectFirstOrThrow("img").attrAsAbsoluteUrl("src") - } + override suspend fun getPageUrl(page: MangaPage): String { + val root = webClient.httpGet(page.url.toAbsoluteUrl(domain)).parseHtml().body() + .requireElementById("image-container") + return root.selectFirstOrThrow("img").attrAsAbsoluteUrl("src") + } - override suspend fun getTags(): Set { - return coroutineScope { - // parse first 3 pages of tags - (1..3).map { page -> - async { getTags(page) } - } - }.awaitAll().flattenTo(ArraySet(360)) - } + override suspend fun getTags(): Set { + return coroutineScope { + // parse first 3 pages of tags + (1..3).map { page -> + async { getTags(page) } + } + }.awaitAll().flattenTo(ArraySet(360)) + } - private suspend fun getTags(page: Int): Set { - val root = webClient.httpGet("https://${domain}/tags/popular?page=$page").parseHtml().body() - .getElementById("tag-container") - return root?.parseTags().orEmpty() - } + private suspend fun getTags(page: Int): Set { + val root = webClient.httpGet("https://${domain}/tags/popular?page=$page").parseHtml().body() + .getElementById("tag-container") + return root?.parseTags().orEmpty() + } - private fun Element.parseTags() = select("a.tag").mapToSet { a -> - val href = a.attr("href").removeSuffix('/') - MangaTag( - title = a.selectFirstOrThrow(".name").text().toTitleCase(), - key = href.substringAfterLast('/'), - source = source, - ) - } + private fun Element.parseTags() = select("a.tag").mapToSet { a -> + val href = a.attr("href").removeSuffix('/') + MangaTag( + title = a.selectFirstOrThrow(".name").text().toTitleCase(), + key = href.substringAfterLast('/'), + source = source, + ) + } - private fun buildQuery(tags: Collection) = tags.joinToString(separator = " ") { tag -> - "tag:\"${tag.key}\"" - } + private fun buildQuery(tags: Collection) = tags.joinToString(separator = " ") { tag -> + "tag:\"${tag.key}\"" + } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NetTruyenParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NetTruyenParser.kt index aaf4c956..6b89233e 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NetTruyenParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NetTruyenParser.kt @@ -16,209 +16,208 @@ import java.util.* @MangaSourceParser("NETTRUYEN", "NetTruyen", "vi") class NetTruyenParser(context: MangaLoaderContext) : - PagedMangaParser(context, MangaSource.NETTRUYEN, pageSize = 36) { - - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("nettruyenin.com", null) - - override val sortOrders: Set - get() = EnumSet.of(SortOrder.UPDATED, SortOrder.POPULARITY, SortOrder.NEWEST, SortOrder.RATING) - - private val mutex = Mutex() - private val dateFormat = SimpleDateFormat("dd/MM/yy", Locale.US) - private var tagCache: ArrayMap? = null - - override suspend fun getDetails(manga: Manga): Manga { - val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - val rating = doc.selectFirst("span[itemprop=ratingValue]") - ?.ownText() - ?.toFloatOrNull() ?: 0f - - val chapterElements = doc.getElementById("nt_listchapter")?.select("ul > li") ?: doc.parseFailed() - val chapters = chapterElements.asReversed().mapChapters { index, element -> - val a = element.selectFirst("div.chapter > a") ?: return@mapChapters null - val relativeUrl = a.attrAsRelativeUrlOrNull("href") ?: return@mapChapters null - val timeText = element.selectFirst("div.col-xs-4.text-center.no-wrap.small")?.text() - - MangaChapter( - id = generateUid(relativeUrl), - name = a.text(), - number = index + 1, - url = relativeUrl, - scanlator = null, - uploadDate = parseChapterTime(timeText), - branch = null, - source = source, - ) - } - - return manga.copy( - rating = rating / 5, - chapters = chapters, - description = doc.selectFirst("div.detail-content > p")?.html(), - isNsfw = doc.selectFirst("div.alert.alert-danger > strong:contains(Cảnh báo độ tuổi)") != null, - ) - } - - // 20 giây trước - // 52 phút trước - // 6 giờ trước - // 2 ngày trước - // 19:09 30/07 - // 23/12/21 - private fun parseChapterTime(timeText: String?): Long { - if (timeText.isNullOrEmpty()) { - return 0L - } - - val timeWords = arrayOf("giây", "phút", "giờ", "ngày") - val calendar = Calendar.getInstance() - val timeArr = timeText.split(' ') - if (WordSet(*timeWords).anyWordIn(timeText)) { - val timeSuffix = timeArr.getOrNull(1) - val timeDiff = timeArr.getOrNull(0)?.toIntOrNull() ?: return 0L - when (timeSuffix) { - timeWords[0] -> calendar.add(Calendar.SECOND, -timeDiff) - timeWords[1] -> calendar.add(Calendar.MINUTE, -timeDiff) - timeWords[2] -> calendar.add(Calendar.HOUR, -timeDiff) - timeWords[3] -> calendar.add(Calendar.DATE, -timeDiff) - else -> return 0L - } - } else { - val relativeDate = timeArr.lastOrNull() ?: return 0L - val dateString = when (relativeDate.split('/').size) { - 2 -> { - val currentYear = calendar.get(Calendar.YEAR).toString().takeLast(2) - "$relativeDate/$currentYear" - } - - 3 -> relativeDate - else -> return 0L - } - - calendar.timeInMillis = dateFormat.tryParse(dateString) - } - - - return calendar.time.time - } - - override suspend fun getListPage( - page: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - val isSearching = !query.isNullOrEmpty() - val url = buildString { - append("https://") - append(domain) - if (isSearching) { - append("/tim-truyen?keyword=") - append(query!!.urlEncoded()) - append("&page=") - append(page) - } else { - val tagQuery = tags.orEmpty().joinToString(",") { it.key } - append("/tim-truyen-nang-cao?genres=$tagQuery") - append("¬genres=&gender=-1&status=-1&minchapter=1&sort=${getSortOrderKey(sortOrder)}") - append("&page=$page") - } - } - - val response = if (isSearching) { - val result = runCatchingCancellable { webClient.httpGet(url) } - val exception = result.exceptionOrNull() - if (exception is NotFoundException) { - return emptyList() - } - - result.getOrThrow() - } else { - webClient.httpGet(url) - } - - val itemsElements = response.parseHtml() - .select("div.ModuleContent > div.items") - .select("div.item") - return itemsElements.mapNotNull { item -> - val tooltipElement = item.selectFirst("div.box_tootip") ?: return@mapNotNull null - val absUrl = item.selectFirst("div.image > a")?.attrAsAbsoluteUrlOrNull("href") ?: return@mapNotNull null - val slug = absUrl.substringAfterLast('/') - val mangaState = when (tooltipElement.selectFirst("div.message_main > p:contains(Tình trạng)")?.ownText()) { - "Đang tiến hành" -> MangaState.ONGOING - "Hoàn thành" -> MangaState.FINISHED - else -> null - } - - val tagMap = getOrCreateTagMap() - val tagsElement = tooltipElement.selectFirst("div.message_main > p:contains(Thể loại)")?.ownText().orEmpty() - val mangaTags = tagsElement.split(',').mapNotNullToSet { tagMap[it.trim()] } - Manga( - id = generateUid(slug), - title = tooltipElement.selectFirst("div.title")?.text().orEmpty(), - altTitle = null, - url = absUrl.toRelativeUrl(domain), - publicUrl = absUrl, - rating = RATING_UNKNOWN, - isNsfw = false, - coverUrl = item.selectFirst("div.image a img")?.absUrl("data-original").orEmpty(), - largeCoverUrl = null, - tags = mangaTags, - state = mangaState, - author = tooltipElement.selectFirst("div.message_main > p:contains(Tác giả)")?.ownText(), - description = tooltipElement.selectFirst("div.box_text")?.text(), - chapters = null, - source = source, - ) - } - } - - - override suspend fun getPages(chapter: MangaChapter): List { - val pageElements = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() - .select("div.reading-detail.box_doc > div img") - return pageElements.map { element -> - val url = element.attrAsAbsoluteUrl("data-original") - MangaPage( - id = generateUid(url), - url = url, - referer = domain, - preview = null, - source = source, - ) - } - } - - override suspend fun getTags(): Set { - val map = getOrCreateTagMap() - val tagSet = ArraySet(map.size) - for (entry in map) { - tagSet.add(entry.value) - } - - return tagSet - } - - private suspend fun getOrCreateTagMap(): ArrayMap = mutex.withLock { - tagCache?.let { return@withLock it } - val doc = webClient.httpGet("/tim-truyen-nang-cao".toAbsoluteUrl(domain)).parseHtml() - val tagItems = doc.select("div.genre-item") - val result = ArrayMap(tagItems.size) - for (item in tagItems) { - val title = item.text().trim() - val key = item.select("span[data-id]").attr("data-id") - result[title] = MangaTag(title = title, key = key, source = source) - } - tagCache = result - result - } - - private fun getSortOrderKey(sortOrder: SortOrder) = when (sortOrder) { - SortOrder.UPDATED -> 0 - SortOrder.POPULARITY -> 10 - SortOrder.NEWEST -> 15 - SortOrder.RATING -> 20 - else -> throw IllegalArgumentException("Sort order ${sortOrder.name} not supported") - } + PagedMangaParser(context, MangaSource.NETTRUYEN, pageSize = 36) { + + override val configKeyDomain: ConfigKey.Domain + get() = ConfigKey.Domain("nettruyenin.com", null) + + override val sortOrders: Set + get() = EnumSet.of(SortOrder.UPDATED, SortOrder.POPULARITY, SortOrder.NEWEST, SortOrder.RATING) + + private val mutex = Mutex() + private val dateFormat = SimpleDateFormat("dd/MM/yy", Locale.US) + private var tagCache: ArrayMap? = null + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val rating = doc.selectFirst("span[itemprop=ratingValue]") + ?.ownText() + ?.toFloatOrNull() ?: 0f + + val chapterElements = doc.getElementById("nt_listchapter")?.select("ul > li") ?: doc.parseFailed() + val chapters = chapterElements.asReversed().mapChapters { index, element -> + val a = element.selectFirst("div.chapter > a") ?: return@mapChapters null + val relativeUrl = a.attrAsRelativeUrlOrNull("href") ?: return@mapChapters null + val timeText = element.selectFirst("div.col-xs-4.text-center.no-wrap.small")?.text() + + MangaChapter( + id = generateUid(relativeUrl), + name = a.text(), + number = index + 1, + url = relativeUrl, + scanlator = null, + uploadDate = parseChapterTime(timeText), + branch = null, + source = source, + ) + } + + return manga.copy( + rating = rating / 5, + chapters = chapters, + description = doc.selectFirst("div.detail-content > p")?.html(), + isNsfw = doc.selectFirst("div.alert.alert-danger > strong:contains(Cảnh báo độ tuổi)") != null, + ) + } + + // 20 giây trước + // 52 phút trước + // 6 giờ trước + // 2 ngày trước + // 19:09 30/07 + // 23/12/21 + private fun parseChapterTime(timeText: String?): Long { + if (timeText.isNullOrEmpty()) { + return 0L + } + + val timeWords = arrayOf("giây", "phút", "giờ", "ngày") + val calendar = Calendar.getInstance() + val timeArr = timeText.split(' ') + if (WordSet(*timeWords).anyWordIn(timeText)) { + val timeSuffix = timeArr.getOrNull(1) + val timeDiff = timeArr.getOrNull(0)?.toIntOrNull() ?: return 0L + when (timeSuffix) { + timeWords[0] -> calendar.add(Calendar.SECOND, -timeDiff) + timeWords[1] -> calendar.add(Calendar.MINUTE, -timeDiff) + timeWords[2] -> calendar.add(Calendar.HOUR, -timeDiff) + timeWords[3] -> calendar.add(Calendar.DATE, -timeDiff) + else -> return 0L + } + } else { + val relativeDate = timeArr.lastOrNull() ?: return 0L + val dateString = when (relativeDate.split('/').size) { + 2 -> { + val currentYear = calendar.get(Calendar.YEAR).toString().takeLast(2) + "$relativeDate/$currentYear" + } + + 3 -> relativeDate + else -> return 0L + } + + calendar.timeInMillis = dateFormat.tryParse(dateString) + } + + + return calendar.time.time + } + + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + val isSearching = !query.isNullOrEmpty() + val url = buildString { + append("https://") + append(domain) + if (isSearching) { + append("/tim-truyen?keyword=") + append(query!!.urlEncoded()) + append("&page=") + append(page) + } else { + val tagQuery = tags.orEmpty().joinToString(",") { it.key } + append("/tim-truyen-nang-cao?genres=$tagQuery") + append("¬genres=&gender=-1&status=-1&minchapter=1&sort=${getSortOrderKey(sortOrder)}") + append("&page=$page") + } + } + + val response = if (isSearching) { + val result = runCatchingCancellable { webClient.httpGet(url) } + val exception = result.exceptionOrNull() + if (exception is NotFoundException) { + return emptyList() + } + + result.getOrThrow() + } else { + webClient.httpGet(url) + } + + val itemsElements = response.parseHtml() + .select("div.ModuleContent > div.items") + .select("div.item") + return itemsElements.mapNotNull { item -> + val tooltipElement = item.selectFirst("div.box_tootip") ?: return@mapNotNull null + val absUrl = item.selectFirst("div.image > a")?.attrAsAbsoluteUrlOrNull("href") ?: return@mapNotNull null + val slug = absUrl.substringAfterLast('/') + val mangaState = when (tooltipElement.selectFirst("div.message_main > p:contains(Tình trạng)")?.ownText()) { + "Đang tiến hành" -> MangaState.ONGOING + "Hoàn thành" -> MangaState.FINISHED + else -> null + } + + val tagMap = getOrCreateTagMap() + val tagsElement = tooltipElement.selectFirst("div.message_main > p:contains(Thể loại)")?.ownText().orEmpty() + val mangaTags = tagsElement.split(',').mapNotNullToSet { tagMap[it.trim()] } + Manga( + id = generateUid(slug), + title = tooltipElement.selectFirst("div.title")?.text().orEmpty(), + altTitle = null, + url = absUrl.toRelativeUrl(domain), + publicUrl = absUrl, + rating = RATING_UNKNOWN, + isNsfw = false, + coverUrl = item.selectFirst("div.image a img")?.absUrl("data-original").orEmpty(), + largeCoverUrl = null, + tags = mangaTags, + state = mangaState, + author = tooltipElement.selectFirst("div.message_main > p:contains(Tác giả)")?.ownText(), + description = tooltipElement.selectFirst("div.box_text")?.text(), + chapters = null, + source = source, + ) + } + } + + + override suspend fun getPages(chapter: MangaChapter): List { + val pageElements = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + .select("div.reading-detail.box_doc > div img") + return pageElements.map { element -> + val url = element.attrAsAbsoluteUrl("data-original") + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } + + override suspend fun getTags(): Set { + val map = getOrCreateTagMap() + val tagSet = ArraySet(map.size) + for (entry in map) { + tagSet.add(entry.value) + } + + return tagSet + } + + private suspend fun getOrCreateTagMap(): ArrayMap = mutex.withLock { + tagCache?.let { return@withLock it } + val doc = webClient.httpGet("/tim-truyen-nang-cao".toAbsoluteUrl(domain)).parseHtml() + val tagItems = doc.select("div.genre-item") + val result = ArrayMap(tagItems.size) + for (item in tagItems) { + val title = item.text().trim() + val key = item.select("span[data-id]").attr("data-id") + result[title] = MangaTag(title = title, key = key, source = source) + } + tagCache = result + result + } + + private fun getSortOrderKey(sortOrder: SortOrder) = when (sortOrder) { + SortOrder.UPDATED -> 0 + SortOrder.POPULARITY -> 10 + SortOrder.NEWEST -> 15 + SortOrder.RATING -> 20 + else -> throw IllegalArgumentException("Sort order ${sortOrder.name} not supported") + } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NicovideoSeigaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NicovideoSeigaParser.kt index 2434199b..7be9b20a 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NicovideoSeigaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NicovideoSeigaParser.kt @@ -12,174 +12,173 @@ private const val STATUS_FINISHED = "完結" @MangaSourceParser("NICOVIDEO_SEIGA", "Nicovideo Seiga", "ja") class NicovideoSeigaParser(context: MangaLoaderContext) : - MangaParser(context, MangaSource.NICOVIDEO_SEIGA), - MangaParserAuthProvider { + MangaParser(context, MangaSource.NICOVIDEO_SEIGA), + MangaParserAuthProvider { - override val authUrl: String - get() = "https://${getDomain("account")}/login?site=seiga" + override val authUrl: String + get() = "https://${getDomain("account")}/login?site=seiga" - override val isAuthorized: Boolean - get() = context.cookieJar.getCookies(getDomain("seiga")).any { - it.name == "user_session" - } + override val isAuthorized: Boolean + get() = context.cookieJar.getCookies(getDomain("seiga")).any { + it.name == "user_session" + } - override suspend fun getUsername(): String { - val body = webClient.httpGet("https://${getDomain("app")}/my/apps").parseHtml().body() - return body.selectFirst("#userinfo > div > div > strong")?.text() ?: throw AuthRequiredException(source) - } + override suspend fun getUsername(): String { + val body = webClient.httpGet("https://${getDomain("app")}/my/apps").parseHtml().body() + return body.selectFirst("#userinfo > div > div > strong")?.text() ?: throw AuthRequiredException(source) + } - override val sortOrders: Set = EnumSet.of( - SortOrder.UPDATED, - SortOrder.POPULARITY, - ) + override val sortOrders: Set = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + ) - override val configKeyDomain: ConfigKey.Domain = ConfigKey.Domain("nicovideo.jp", null) + override val configKeyDomain: ConfigKey.Domain = ConfigKey.Domain("nicovideo.jp", null) - @InternalParsersApi - override suspend fun getList( - offset: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - val page = (offset / 20f).toIntUp().inc() - val domain = getDomain("seiga") - val url = when { - !query.isNullOrEmpty() -> return if (offset == 0) getSearchList(query, page) else emptyList() - tags.isNullOrEmpty() -> "https://$domain/manga/list?page=$page&sort=${getSortKey(sortOrder)}" - tags.size == 1 -> "https://$domain/manga/list?category=${tags.first().key}&page=$page" + - "&sort=${getSortKey(sortOrder)}" + @InternalParsersApi + override suspend fun getList( + offset: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + val page = (offset / 20f).toIntUp().inc() + val domain = getDomain("seiga") + val url = when { + !query.isNullOrEmpty() -> return if (offset == 0) getSearchList(query, page) else emptyList() + tags.isNullOrEmpty() -> "https://$domain/manga/list?page=$page&sort=${getSortKey(sortOrder)}" + tags.size == 1 -> "https://$domain/manga/list?category=${tags.first().key}&page=$page" + + "&sort=${getSortKey(sortOrder)}" - tags.size > 1 -> throw IllegalArgumentException("This source supports only 1 category") - else -> "https://$domain/manga/list?page=$page&sort=${getSortKey(sortOrder)}" - } - val doc = webClient.httpGet(url).parseHtml() - val comicList = doc.body().select("#comic_list > ul > li") ?: doc.parseFailed("Container not found") - val items = comicList.select("div > .description > div > div") - return items.mapNotNull { item -> - val href = - item.selectFirst(".comic_icon > div > a")?.attrAsRelativeUrlOrNull("href") ?: return@mapNotNull null - val statusText = item.selectFirst(".mg_description_header > .mg_icon > .content_status > span")?.text() - Manga( - id = generateUid(href), - title = item.selectFirst(".mg_body > .title > a")?.text() ?: return@mapNotNull null, - coverUrl = item.selectFirst(".comic_icon > div > a > img")?.attrAsAbsoluteUrl("src").orEmpty(), - altTitle = null, - author = item.selectFirst(".mg_description_header > .mg_author > a")?.text(), - rating = RATING_UNKNOWN, - url = href, - isNsfw = false, - tags = item.getElementsByAttributeValueContaining("href", "?category=").mapToSet { a -> - MangaTag( - key = a.attr("href").substringAfterLast('='), - title = a.ownText().trim(), - source = source, - ) - }, - state = when (statusText) { - STATUS_ONGOING -> MangaState.ONGOING - STATUS_FINISHED -> MangaState.FINISHED - else -> null - }, - publicUrl = href.toAbsoluteUrl(item.host ?: getDomain("seiga")), - source = source, - ) - } - } + tags.size > 1 -> throw IllegalArgumentException("This source supports only 1 category") + else -> "https://$domain/manga/list?page=$page&sort=${getSortKey(sortOrder)}" + } + val doc = webClient.httpGet(url).parseHtml() + val comicList = doc.body().select("#comic_list > ul > li") ?: doc.parseFailed("Container not found") + val items = comicList.select("div > .description > div > div") + return items.mapNotNull { item -> + val href = + item.selectFirst(".comic_icon > div > a")?.attrAsRelativeUrlOrNull("href") ?: return@mapNotNull null + val statusText = item.selectFirst(".mg_description_header > .mg_icon > .content_status > span")?.text() + Manga( + id = generateUid(href), + title = item.selectFirst(".mg_body > .title > a")?.text() ?: return@mapNotNull null, + coverUrl = item.selectFirst(".comic_icon > div > a > img")?.attrAsAbsoluteUrl("src").orEmpty(), + altTitle = null, + author = item.selectFirst(".mg_description_header > .mg_author > a")?.text(), + rating = RATING_UNKNOWN, + url = href, + isNsfw = false, + tags = item.getElementsByAttributeValueContaining("href", "?category=").mapToSet { a -> + MangaTag( + key = a.attr("href").substringAfterLast('='), + title = a.ownText().trim(), + source = source, + ) + }, + state = when (statusText) { + STATUS_ONGOING -> MangaState.ONGOING + STATUS_FINISHED -> MangaState.FINISHED + else -> null + }, + publicUrl = href.toAbsoluteUrl(item.host ?: getDomain("seiga")), + source = source, + ) + } + } - override suspend fun getDetails(manga: Manga): Manga { - val doc = webClient.httpGet(manga.url.toAbsoluteUrl(getDomain("seiga"))).parseHtml() - val contents = doc.body().selectFirstOrThrow("#contents") - val statusText = contents - .select("div.mg_work_detail > div > div:nth-child(2) > div.tip.content_status.status_series > span") - .text() - return manga.copy( - description = contents.selectFirst("div.mg_work_detail > div > div.row > div.description_text")?.html(), - largeCoverUrl = contents.selectFirst("div.primaries > div.main_visual > a > img") - ?.attrAsAbsoluteUrlOrNull("src"), - state = when (statusText) { - STATUS_ONGOING -> MangaState.ONGOING - STATUS_FINISHED -> MangaState.FINISHED - else -> null - }, - isNsfw = contents.select(".icon_adult").isNotEmpty(), - chapters = contents.select("#episode_list > ul > li").mapChapters { i, li -> - val href = li.selectFirst("div > div.description > div.title > a") - ?.attrAsRelativeUrl("href") ?: li.parseFailed() - MangaChapter( - id = generateUid(href), - name = li.select("div > div.description > div.title > a").text(), - number = i + 1, - url = href, - scanlator = null, - branch = null, - uploadDate = 0, - source = source, - ) - }, - ) - } + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(getDomain("seiga"))).parseHtml() + val contents = doc.body().selectFirstOrThrow("#contents") + val statusText = contents + .select("div.mg_work_detail > div > div:nth-child(2) > div.tip.content_status.status_series > span") + .text() + return manga.copy( + description = contents.selectFirst("div.mg_work_detail > div > div.row > div.description_text")?.html(), + largeCoverUrl = contents.selectFirst("div.primaries > div.main_visual > a > img") + ?.attrAsAbsoluteUrlOrNull("src"), + state = when (statusText) { + STATUS_ONGOING -> MangaState.ONGOING + STATUS_FINISHED -> MangaState.FINISHED + else -> null + }, + isNsfw = contents.select(".icon_adult").isNotEmpty(), + chapters = contents.select("#episode_list > ul > li").mapChapters { i, li -> + val href = li.selectFirst("div > div.description > div.title > a") + ?.attrAsRelativeUrl("href") ?: li.parseFailed() + MangaChapter( + id = generateUid(href), + name = li.select("div > div.description > div.title > a").text(), + number = i + 1, + url = href, + scanlator = null, + branch = null, + uploadDate = 0, + source = source, + ) + }, + ) + } - override suspend fun getPages(chapter: MangaChapter): List { - val fullUrl = chapter.url.toAbsoluteUrl(getDomain("seiga")) - val doc = webClient.httpGet(fullUrl).parseHtml() - if (!doc.select("#login_manga").isEmpty()) - throw AuthRequiredException(source) - val root = doc.body().select("#page_contents > li") - return root.map { li -> - val url = li.select("div > img").attr("data-original") - MangaPage( - id = generateUid(url), - url = url, - preview = null, - referer = fullUrl, - source = source, - ) - } - } + override suspend fun getPages(chapter: MangaChapter): List { + val fullUrl = chapter.url.toAbsoluteUrl(getDomain("seiga")) + val doc = webClient.httpGet(fullUrl).parseHtml() + if (!doc.select("#login_manga").isEmpty()) + throw AuthRequiredException(source) + val root = doc.body().select("#page_contents > li") + return root.map { li -> + val url = li.select("div > img").attr("data-original") + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } - override suspend fun getTags(): Set { - val doc = webClient.httpGet("https://${getDomain("seiga")}/manga/list").parseHtml() - val root = doc.body().selectOrThrow("#mg_category_list > ul > li") - return root.mapToSet { li -> - val a = li.selectFirstOrThrow("a") - MangaTag( - title = a.text(), - key = a.attrAsRelativeUrlOrNull("href").orEmpty(), - source = source, - ) - } - } + override suspend fun getTags(): Set { + val doc = webClient.httpGet("https://${getDomain("seiga")}/manga/list").parseHtml() + val root = doc.body().selectOrThrow("#mg_category_list > ul > li") + return root.mapToSet { li -> + val a = li.selectFirstOrThrow("a") + MangaTag( + title = a.text(), + key = a.attrAsRelativeUrlOrNull("href").orEmpty(), + source = source, + ) + } + } - private suspend fun getSearchList(query: String, page: Int): List { - val domain = getDomain("seiga") - val doc = webClient.httpGet("https://$domain/manga/search/?q=$query&page=$page&sort=score").parseHtml() - val root = doc.body().select(".search_result__item") - return root.mapNotNull { item -> - val href = item.selectFirst(".search_result__item__thumbnail > a") - ?.attrAsRelativeUrl("href") ?: doc.parseFailed() - Manga( - id = generateUid(href), - url = href, - publicUrl = href.toAbsoluteUrl(item.host ?: domain), - title = item.selectFirst(".search_result__item__info > .search_result__item__info--title > a") - ?.text()?.trim() ?: return@mapNotNull null, - altTitle = null, - author = null, - tags = emptySet(), - rating = RATING_UNKNOWN, - state = null, - isNsfw = false, - source = source, - coverUrl = item.selectFirst(".search_result__item__thumbnail > a > img") - ?.attrAsAbsoluteUrl("data-original").orEmpty(), - ) - } - } + private suspend fun getSearchList(query: String, page: Int): List { + val domain = getDomain("seiga") + val doc = webClient.httpGet("https://$domain/manga/search/?q=$query&page=$page&sort=score").parseHtml() + val root = doc.body().select(".search_result__item") + return root.mapNotNull { item -> + val href = item.selectFirst(".search_result__item__thumbnail > a") + ?.attrAsRelativeUrl("href") ?: doc.parseFailed() + Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(item.host ?: domain), + title = item.selectFirst(".search_result__item__info > .search_result__item__info--title > a") + ?.text()?.trim() ?: return@mapNotNull null, + altTitle = null, + author = null, + tags = emptySet(), + rating = RATING_UNKNOWN, + state = null, + isNsfw = false, + source = source, + coverUrl = item.selectFirst(".search_result__item__thumbnail > a > img") + ?.attrAsAbsoluteUrl("data-original").orEmpty(), + ) + } + } - private fun getSortKey(sortOrder: SortOrder) = when (sortOrder) { - SortOrder.POPULARITY -> "manga_view" - SortOrder.UPDATED -> "manga_updated" - else -> "manga_view" - } + private fun getSortKey(sortOrder: SortOrder) = when (sortOrder) { + SortOrder.POPULARITY -> "manga_view" + SortOrder.UPDATED -> "manga_updated" + else -> "manga_view" + } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NineMangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NineMangaParser.kt index f3d6d2ce..f3b2f6b8 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NineMangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NineMangaParser.kt @@ -12,248 +12,247 @@ import java.text.SimpleDateFormat import java.util.* internal abstract class NineMangaParser( - context: MangaLoaderContext, - source: MangaSource, - defaultDomain: String, + context: MangaLoaderContext, + source: MangaSource, + defaultDomain: String, ) : PagedMangaParser(context, source, pageSize = 26) { - override val configKeyDomain = ConfigKey.Domain(defaultDomain, null) + override val configKeyDomain = ConfigKey.Domain(defaultDomain, null) - init { - context.cookieJar.insertCookies(domain, "ninemanga_template_desk=yes") - } + init { + context.cookieJar.insertCookies(domain, "ninemanga_template_desk=yes") + } - override val headers = Headers.Builder() - .add("Accept-Language", "en-US;q=0.7,en;q=0.3") - .build() + override val headers = Headers.Builder() + .add("Accept-Language", "en-US;q=0.7,en;q=0.3") + .build() - override val sortOrders: Set = Collections.singleton( - SortOrder.POPULARITY, - ) + override val sortOrders: Set = Collections.singleton( + SortOrder.POPULARITY, + ) - override suspend fun getListPage( - page: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - val url = buildString { - append("https://") - append(domain) - when { - !query.isNullOrEmpty() -> { - append("/search/?name_sel=&wd=") - append(query.urlEncoded()) - append("&page=") - } + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + val url = buildString { + append("https://") + append(domain) + when { + !query.isNullOrEmpty() -> { + append("/search/?name_sel=&wd=") + append(query.urlEncoded()) + append("&page=") + } - !tags.isNullOrEmpty() -> { - append("/search/?category_id=") - for (tag in tags) { - append(tag.key) - append(',') - } - append("&page=") - } + !tags.isNullOrEmpty() -> { + append("/search/?category_id=") + for (tag in tags) { + append(tag.key) + append(',') + } + append("&page=") + } - else -> { - append("/category/index_") - } - } - append(page) - append(".html") - } - val doc = webClient.httpGet(url).parseHtml() - val root = doc.body().selectFirst("ul.direlist") - ?: doc.parseFailed("Cannot find root") - val baseHost = root.baseUri().toHttpUrl().host - return root.select("li").map { node -> - val href = node.selectFirst("a")?.absUrl("href") - ?: node.parseFailed("Link not found") - val relUrl = href.toRelativeUrl(baseHost) - val dd = node.selectFirst("dd") - Manga( - id = generateUid(relUrl), - url = relUrl, - publicUrl = href, - title = dd?.selectFirst("a.bookname")?.text()?.toCamelCase().orEmpty(), - altTitle = null, - coverUrl = node.selectFirst("img")?.absUrl("src").orEmpty(), - rating = RATING_UNKNOWN, - author = null, - isNsfw = false, - tags = emptySet(), - state = null, - source = source, - description = dd?.selectFirst("p")?.html(), - ) - } - } + else -> { + append("/category/index_") + } + } + append(page) + append(".html") + } + val doc = webClient.httpGet(url).parseHtml() + val root = doc.body().selectFirst("ul.direlist") + ?: doc.parseFailed("Cannot find root") + val baseHost = root.baseUri().toHttpUrl().host + return root.select("li").map { node -> + val href = node.selectFirst("a")?.absUrl("href") + ?: node.parseFailed("Link not found") + val relUrl = href.toRelativeUrl(baseHost) + val dd = node.selectFirst("dd") + Manga( + id = generateUid(relUrl), + url = relUrl, + publicUrl = href, + title = dd?.selectFirst("a.bookname")?.text()?.toCamelCase().orEmpty(), + altTitle = null, + coverUrl = node.selectFirst("img")?.absUrl("src").orEmpty(), + rating = RATING_UNKNOWN, + author = null, + isNsfw = false, + tags = emptySet(), + state = null, + source = source, + description = dd?.selectFirst("p")?.html(), + ) + } + } - override suspend fun getDetails(manga: Manga): Manga { - val doc = webClient.httpGet( - manga.url.toAbsoluteUrl(domain) + "?waring=1", - ).parseHtml() - val root = doc.body().selectFirstOrThrow("div.manga") - val infoRoot = root.selectFirstOrThrow("div.bookintro") - return manga.copy( - tags = infoRoot.getElementsByAttributeValue("itemprop", "genre").first() - ?.select("a")?.mapToSet { a -> - MangaTag( - title = a.text().toTitleCase(), - key = a.attr("href").substringBetween("/", "."), - source = source, - ) - }.orEmpty(), - author = infoRoot.getElementsByAttributeValue("itemprop", "author").first()?.text(), - state = parseStatus(infoRoot.select("li a.red").text()), - description = infoRoot.getElementsByAttributeValue("itemprop", "description").first() - ?.html()?.substringAfter(""), - chapters = root.selectFirst("div.chapterbox")?.select("ul.sub_vol_ul > li") - ?.asReversed()?.mapChapters { i, li -> - val a = li.selectFirst("a.chapter_list_a") - val href = a?.attrAsRelativeUrlOrNull("href") - ?.replace("%20", " ") ?: li.parseFailed("Link not found") - MangaChapter( - id = generateUid(href), - name = a.text(), - number = i + 1, - url = href, - uploadDate = parseChapterDateByLang(li.selectFirst("span")?.text().orEmpty()), - source = source, - scanlator = null, - branch = null, - ) - }, - ) - } + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet( + manga.url.toAbsoluteUrl(domain) + "?waring=1", + ).parseHtml() + val root = doc.body().selectFirstOrThrow("div.manga") + val infoRoot = root.selectFirstOrThrow("div.bookintro") + return manga.copy( + tags = infoRoot.getElementsByAttributeValue("itemprop", "genre").first() + ?.select("a")?.mapToSet { a -> + MangaTag( + title = a.text().toTitleCase(), + key = a.attr("href").substringBetween("/", "."), + source = source, + ) + }.orEmpty(), + author = infoRoot.getElementsByAttributeValue("itemprop", "author").first()?.text(), + state = parseStatus(infoRoot.select("li a.red").text()), + description = infoRoot.getElementsByAttributeValue("itemprop", "description").first() + ?.html()?.substringAfter(""), + chapters = root.selectFirst("div.chapterbox")?.select("ul.sub_vol_ul > li") + ?.asReversed()?.mapChapters { i, li -> + val a = li.selectFirst("a.chapter_list_a") + val href = a?.attrAsRelativeUrlOrNull("href") + ?.replace("%20", " ") ?: li.parseFailed("Link not found") + MangaChapter( + id = generateUid(href), + name = a.text(), + number = i + 1, + url = href, + uploadDate = parseChapterDateByLang(li.selectFirst("span")?.text().orEmpty()), + source = source, + scanlator = null, + branch = null, + ) + }, + ) + } - override suspend fun getPages(chapter: MangaChapter): List { - val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() - return doc.body().getElementById("page")?.select("option")?.map { option -> - val url = option.attr("value") - MangaPage( - id = generateUid(url), - url = url, - referer = chapter.url.toAbsoluteUrl(domain), - preview = null, - source = source, - ) - } ?: doc.parseFailed("Pages list not found") - } + override suspend fun getPages(chapter: MangaChapter): List { + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + return doc.body().getElementById("page")?.select("option")?.map { option -> + val url = option.attr("value") + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } ?: doc.parseFailed("Pages list not found") + } - override suspend fun getPageUrl(page: MangaPage): String { - val doc = webClient.httpGet(page.url.toAbsoluteUrl(domain)).parseHtml() - val root = doc.body() - return root.selectFirst("a.pic_download")?.absUrl("href") - ?: doc.parseFailed("Page image not found") - } + override suspend fun getPageUrl(page: MangaPage): String { + val doc = webClient.httpGet(page.url.toAbsoluteUrl(domain)).parseHtml() + val root = doc.body() + return root.selectFirst("a.pic_download")?.absUrl("href") + ?: doc.parseFailed("Page image not found") + } - override suspend fun getTags(): Set { - val doc = webClient.httpGet("https://${domain}/search/?type=high") - .parseHtml() - val root = doc.body().getElementById("search_form") - return root?.select("li.cate_list")?.mapNotNullToSet { li -> - val cateId = li.attr("cate_id") ?: return@mapNotNullToSet null - val a = li.selectFirst("a") ?: return@mapNotNullToSet null - MangaTag( - title = a.text().toTitleCase(), - key = cateId, - source = source, - ) - } ?: doc.parseFailed("Root not found") - } + override suspend fun getTags(): Set { + val doc = webClient.httpGet("https://${domain}/search/?type=high") + .parseHtml() + val root = doc.body().getElementById("search_form") + return root?.select("li.cate_list")?.mapNotNullToSet { li -> + val cateId = li.attr("cate_id") ?: return@mapNotNullToSet null + val a = li.selectFirst("a") ?: return@mapNotNullToSet null + MangaTag( + title = a.text().toTitleCase(), + key = cateId, + source = source, + ) + } ?: doc.parseFailed("Root not found") + } - private fun parseStatus(status: String) = when { - status.contains("Ongoing") -> MangaState.ONGOING - status.contains("Completed") -> MangaState.FINISHED - else -> null - } + private fun parseStatus(status: String) = when { + status.contains("Ongoing") -> MangaState.ONGOING + status.contains("Completed") -> MangaState.FINISHED + else -> null + } - private fun parseChapterDateByLang(date: String): Long { - val dateWords = date.split(" ") + private fun parseChapterDateByLang(date: String): Long { + val dateWords = date.split(" ") - if (dateWords.size == 3) { - if (dateWords[1].contains(",")) { - SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH).tryParse(date) - } else { - val timeAgo = Integer.parseInt(dateWords[0]) - return Calendar.getInstance().apply { - when (dateWords[1]) { - "minutes" -> Calendar.MINUTE // EN-FR - "hours" -> Calendar.HOUR // EN + if (dateWords.size == 3) { + if (dateWords[1].contains(",")) { + SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH).tryParse(date) + } else { + val timeAgo = Integer.parseInt(dateWords[0]) + return Calendar.getInstance().apply { + when (dateWords[1]) { + "minutes" -> Calendar.MINUTE // EN-FR + "hours" -> Calendar.HOUR // EN - "minutos" -> Calendar.MINUTE // ES - "horas" -> Calendar.HOUR + "minutos" -> Calendar.MINUTE // ES + "horas" -> Calendar.HOUR - // "minutos" -> Calendar.MINUTE // BR - "hora" -> Calendar.HOUR + // "minutos" -> Calendar.MINUTE // BR + "hora" -> Calendar.HOUR - "минут" -> Calendar.MINUTE // RU - "часа" -> Calendar.HOUR + "минут" -> Calendar.MINUTE // RU + "часа" -> Calendar.HOUR - "Stunden" -> Calendar.HOUR // DE + "Stunden" -> Calendar.HOUR // DE - "minuti" -> Calendar.MINUTE // IT - "ore" -> Calendar.HOUR + "minuti" -> Calendar.MINUTE // IT + "ore" -> Calendar.HOUR - "heures" -> Calendar.HOUR // FR ("minutes" also French word) - else -> null - }?.let { - add(it, -timeAgo) - } - }.timeInMillis - } - } - return 0L - } + "heures" -> Calendar.HOUR // FR ("minutes" also French word) + else -> null + }?.let { + add(it, -timeAgo) + } + }.timeInMillis + } + } + return 0L + } - @MangaSourceParser("NINEMANGA_EN", "NineManga English", "en") - class English(context: MangaLoaderContext) : NineMangaParser( - context, - MangaSource.NINEMANGA_EN, - "www.ninemanga.com", - ) + @MangaSourceParser("NINEMANGA_EN", "NineManga English", "en") + class English(context: MangaLoaderContext) : NineMangaParser( + context, + MangaSource.NINEMANGA_EN, + "www.ninemanga.com", + ) - @MangaSourceParser("NINEMANGA_ES", "NineManga Español", "es") - class Spanish(context: MangaLoaderContext) : NineMangaParser( - context, - MangaSource.NINEMANGA_ES, - "es.ninemanga.com", - ) + @MangaSourceParser("NINEMANGA_ES", "NineManga Español", "es") + class Spanish(context: MangaLoaderContext) : NineMangaParser( + context, + MangaSource.NINEMANGA_ES, + "es.ninemanga.com", + ) - @MangaSourceParser("NINEMANGA_RU", "NineManga Русский", "ru") - class Russian(context: MangaLoaderContext) : NineMangaParser( - context, - MangaSource.NINEMANGA_RU, - "ru.ninemanga.com", - ) + @MangaSourceParser("NINEMANGA_RU", "NineManga Русский", "ru") + class Russian(context: MangaLoaderContext) : NineMangaParser( + context, + MangaSource.NINEMANGA_RU, + "ru.ninemanga.com", + ) - @MangaSourceParser("NINEMANGA_DE", "NineManga Deutsch", "de") - class Deutsch(context: MangaLoaderContext) : NineMangaParser( - context, - MangaSource.NINEMANGA_DE, - "de.ninemanga.com", - ) + @MangaSourceParser("NINEMANGA_DE", "NineManga Deutsch", "de") + class Deutsch(context: MangaLoaderContext) : NineMangaParser( + context, + MangaSource.NINEMANGA_DE, + "de.ninemanga.com", + ) - @MangaSourceParser("NINEMANGA_BR", "NineManga Brasil", "pt") - class Brazil(context: MangaLoaderContext) : NineMangaParser( - context, - MangaSource.NINEMANGA_BR, - "br.ninemanga.com", - ) + @MangaSourceParser("NINEMANGA_BR", "NineManga Brasil", "pt") + class Brazil(context: MangaLoaderContext) : NineMangaParser( + context, + MangaSource.NINEMANGA_BR, + "br.ninemanga.com", + ) - @MangaSourceParser("NINEMANGA_IT", "NineManga Italiano", "it") - class Italiano(context: MangaLoaderContext) : NineMangaParser( - context, - MangaSource.NINEMANGA_IT, - "it.ninemanga.com", - ) + @MangaSourceParser("NINEMANGA_IT", "NineManga Italiano", "it") + class Italiano(context: MangaLoaderContext) : NineMangaParser( + context, + MangaSource.NINEMANGA_IT, + "it.ninemanga.com", + ) - @MangaSourceParser("NINEMANGA_FR", "NineManga Français", "fr") - class Francais(context: MangaLoaderContext) : NineMangaParser( - context, - MangaSource.NINEMANGA_FR, - "fr.ninemanga.com", - ) + @MangaSourceParser("NINEMANGA_FR", "NineManga Français", "fr") + class Francais(context: MangaLoaderContext) : NineMangaParser( + context, + MangaSource.NINEMANGA_FR, + "fr.ninemanga.com", + ) } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NudeMoonParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NudeMoonParser.kt index 823cd37b..3b8a38d3 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NudeMoonParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/NudeMoonParser.kt @@ -15,213 +15,212 @@ private const val MAX_THUMB_INDEX = 19 @MangaSourceParser("NUDEMOON", "Nude-Moon", "ru") internal class NudeMoonParser( - context: MangaLoaderContext, + context: MangaLoaderContext, ) : MangaParser(context, MangaSource.NUDEMOON), MangaParserAuthProvider { - override val configKeyDomain = ConfigKey.Domain( - defaultValue = "nude-moon.org", - presetValues = arrayOf("nude-moon.org", "nude-moon.net"), - ) - override val authUrl: String - get() = "https://${domain}/index.php" - - override val isAuthorized: Boolean - get() { - return context.cookieJar.getCookies(domain).any { - it.name == "fusion_user" - } - } - - override val sortOrders: Set = EnumSet.of( - SortOrder.NEWEST, - SortOrder.POPULARITY, - SortOrder.RATING, - ) - - init { - context.cookieJar.insertCookies( - domain, - "NMfYa=1;", - "nm_mobile=0;", - ) - } - - override suspend fun getList( - offset: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - val domain = domain - val url = when { - !query.isNullOrEmpty() -> "https://$domain/search?stext=${query.urlEncoded()}&rowstart=$offset" - !tags.isNullOrEmpty() -> tags.joinToString( - separator = "_", - prefix = "https://$domain/tags/", - postfix = "&rowstart=$offset", - transform = { it.key.urlEncoded() }, - ) - - else -> "https://$domain/all_manga?${getSortKey(sortOrder)}&rowstart=$offset" - } - val doc = webClient.httpGet(url).parseHtml() - val root = doc.body().run { - selectFirst("td.main-bg") ?: selectFirst("td.main-body") - } ?: doc.parseFailed("Cannot find root") - return root.select("table.news_pic2").mapNotNull { row -> - val a = row.selectFirst("td.bg_style1")?.selectFirst("a") - ?: return@mapNotNull null - val href = a.attrAsRelativeUrl("href") - val title = a.selectFirst("h2")?.text().orEmpty() - val info = row.selectFirst("td[width=100%]") ?: return@mapNotNull null - Manga( - id = generateUid(href), - url = href, - title = title.substringAfter(" / "), - altTitle = title.substringBefore(" / ", "") - .takeUnless { it.isBlank() }, - author = info.getElementsContainingOwnText("Автор:").firstOrNull() - ?.nextElementSibling()?.ownText(), - coverUrl = row.selectFirst("img.news_pic2")?.absUrl("data-src") - .orEmpty(), - tags = row.selectFirst("span.tag-links")?.select("a") - ?.mapToSet { - MangaTag( - title = it.text().toTitleCase(), - key = it.attr("href").substringAfterLast('/'), - source = source, - ) - }.orEmpty(), - source = source, - publicUrl = a.absUrl("href"), - rating = RATING_UNKNOWN, - isNsfw = true, - description = row.selectFirst("div.description")?.html(), - state = null, - ) - } - } - - override suspend fun getDetails(manga: Manga): Manga { - val body = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml().body() - val root = body.selectFirst("table.shoutbox") - ?: body.parseFailed("Cannot find root") - val info = root.select("div.tbl2") - val lastInfo = info.last() - return manga.copy( - largeCoverUrl = body.selectFirst("img.news_pic2")?.absUrl("src"), - description = info.select("div.blockquote").lastOrNull()?.html() ?: manga.description, - tags = info.select("span.tag-links").firstOrNull()?.select("a")?.mapToSet { - MangaTag( - title = it.text().toTitleCase(), - key = it.attr("href").substringAfterLast('/'), - source = source, - ) - }?.plus(manga.tags) ?: manga.tags, - author = lastInfo?.getElementsByAttributeValueContaining("href", "mangaka/")?.text() - ?: manga.author, - chapters = listOf( - MangaChapter( - id = manga.id, - url = getReadLink(manga.url), - source = source, - number = 1, - name = manga.title, - scanlator = lastInfo?.getElementsByAttributeValueContaining("href", "perevod/")?.text(), - uploadDate = lastInfo?.getElementsContainingOwnText("Дата:") - ?.firstOrNull() - ?.html() - ?.parseDate() ?: 0L, - branch = null, - ), - ), - ) - } - - override suspend fun getPages(chapter: MangaChapter): List { - val fullUrl = chapter.url.toAbsoluteUrl(domain) - val doc = webClient.httpGet(fullUrl).parseHtml() - val mangaId = chapter.url.substringAfterLast('/').substringBefore('-').toIntOrNull() - - val script = doc.select("script").firstNotNullOfOrNull { - it.html().takeIf { x -> x.contains(" images = new ") } - } ?: if (isAuthorized) { - doc.parseFailed("Cannot find pages list") - } else { - throw AuthRequiredException(source) - } - val pagesRegex = Regex("images\\[(\\d+)].src\\s*=\\s*'([^']+)'", RegexOption.MULTILINE) - return pagesRegex.findAll(script).map { match -> - val i = match.groupValues[1].toInt() - val url = match.groupValues[2] - MangaPage( - id = generateUid(url), - url = url, - referer = fullUrl, - preview = if (i <= MAX_THUMB_INDEX && mangaId != null) { - val part2 = url.substringBeforeLast('/') - val part3 = url.substringAfterLast('/') - val part1 = part2.substringBeforeLast('/') - "$part1/thumb/$mangaId/thumb_$part3" - } else { - null - }, - source = source, - ) - }.toList() - } - - override suspend fun getTags(): Set { - val domain = domain - val doc = webClient.httpGet("https://$domain/all_manga").parseHtml() - val root = doc.body().getElementsContainingOwnText("Поиск манги по тегам") - .firstOrNull()?.parents()?.find { it.tag().normalName() == "tbody" } - ?.selectFirst("td.textbox")?.selectFirst("td.small") - ?: doc.parseFailed("Tags root not found") - return root.select("a").mapToSet { - MangaTag( - title = it.text().toTitleCase(), - key = it.attr("href").substringAfterLast('/') - .removeSuffix("+"), - source = source, - ) - } - } - - override suspend fun getUsername(): String { - val body = webClient.httpGet("https://${domain}/").parseHtml() - .body() - return body - .getElementsContainingOwnText("Профиль") - .firstOrNull() - ?.attr("href") - ?.substringAfterLast('/') - ?: run { - throw if (body.selectFirst("form[name=\"loginform\"]") != null) { - AuthRequiredException(source) - } else { - body.parseFailed("Cannot find username") - } - } - } - - private fun getSortKey(sortOrder: SortOrder) = - when (sortOrder) { - SortOrder.POPULARITY -> "views" - SortOrder.NEWEST -> "date" - SortOrder.RATING -> "like" - else -> "like" - } - - private fun String.parseDate(): Long { - val dateString = substringBetweenFirst("Дата:", "<")?.trim() ?: return 0 - val dateFormat = SimpleDateFormat("d MMMM yyyy", Locale("ru")) - return dateFormat.tryParse(dateString) - } - - private fun getReadLink(url: String): String { - val prefix = url.substringBefore('-', "") - val suffix = url.substringAfter('-').trimStart('-') - return "$prefix-online-$suffix" - } + override val configKeyDomain = ConfigKey.Domain( + defaultValue = "nude-moon.org", + presetValues = arrayOf("nude-moon.org", "nude-moon.net"), + ) + override val authUrl: String + get() = "https://${domain}/index.php" + + override val isAuthorized: Boolean + get() { + return context.cookieJar.getCookies(domain).any { + it.name == "fusion_user" + } + } + + override val sortOrders: Set = EnumSet.of( + SortOrder.NEWEST, + SortOrder.POPULARITY, + SortOrder.RATING, + ) + + init { + context.cookieJar.insertCookies( + domain, + "NMfYa=1;", + "nm_mobile=0;", + ) + } + + override suspend fun getList( + offset: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + val domain = domain + val url = when { + !query.isNullOrEmpty() -> "https://$domain/search?stext=${query.urlEncoded()}&rowstart=$offset" + !tags.isNullOrEmpty() -> tags.joinToString( + separator = "_", + prefix = "https://$domain/tags/", + postfix = "&rowstart=$offset", + transform = { it.key.urlEncoded() }, + ) + + else -> "https://$domain/all_manga?${getSortKey(sortOrder)}&rowstart=$offset" + } + val doc = webClient.httpGet(url).parseHtml() + val root = doc.body().run { + selectFirst("td.main-bg") ?: selectFirst("td.main-body") + } ?: doc.parseFailed("Cannot find root") + return root.select("table.news_pic2").mapNotNull { row -> + val a = row.selectFirst("td.bg_style1")?.selectFirst("a") + ?: return@mapNotNull null + val href = a.attrAsRelativeUrl("href") + val title = a.selectFirst("h2")?.text().orEmpty() + val info = row.selectFirst("td[width=100%]") ?: return@mapNotNull null + Manga( + id = generateUid(href), + url = href, + title = title.substringAfter(" / "), + altTitle = title.substringBefore(" / ", "") + .takeUnless { it.isBlank() }, + author = info.getElementsContainingOwnText("Автор:").firstOrNull() + ?.nextElementSibling()?.ownText(), + coverUrl = row.selectFirst("img.news_pic2")?.absUrl("data-src") + .orEmpty(), + tags = row.selectFirst("span.tag-links")?.select("a") + ?.mapToSet { + MangaTag( + title = it.text().toTitleCase(), + key = it.attr("href").substringAfterLast('/'), + source = source, + ) + }.orEmpty(), + source = source, + publicUrl = a.absUrl("href"), + rating = RATING_UNKNOWN, + isNsfw = true, + description = row.selectFirst("div.description")?.html(), + state = null, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val body = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml().body() + val root = body.selectFirst("table.shoutbox") + ?: body.parseFailed("Cannot find root") + val info = root.select("div.tbl2") + val lastInfo = info.last() + return manga.copy( + largeCoverUrl = body.selectFirst("img.news_pic2")?.absUrl("src"), + description = info.select("div.blockquote").lastOrNull()?.html() ?: manga.description, + tags = info.select("span.tag-links").firstOrNull()?.select("a")?.mapToSet { + MangaTag( + title = it.text().toTitleCase(), + key = it.attr("href").substringAfterLast('/'), + source = source, + ) + }?.plus(manga.tags) ?: manga.tags, + author = lastInfo?.getElementsByAttributeValueContaining("href", "mangaka/")?.text() + ?: manga.author, + chapters = listOf( + MangaChapter( + id = manga.id, + url = getReadLink(manga.url), + source = source, + number = 1, + name = manga.title, + scanlator = lastInfo?.getElementsByAttributeValueContaining("href", "perevod/")?.text(), + uploadDate = lastInfo?.getElementsContainingOwnText("Дата:") + ?.firstOrNull() + ?.html() + ?.parseDate() ?: 0L, + branch = null, + ), + ), + ) + } + + override suspend fun getPages(chapter: MangaChapter): List { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + val mangaId = chapter.url.substringAfterLast('/').substringBefore('-').toIntOrNull() + + val script = doc.select("script").firstNotNullOfOrNull { + it.html().takeIf { x -> x.contains(" images = new ") } + } ?: if (isAuthorized) { + doc.parseFailed("Cannot find pages list") + } else { + throw AuthRequiredException(source) + } + val pagesRegex = Regex("images\\[(\\d+)].src\\s*=\\s*'([^']+)'", RegexOption.MULTILINE) + return pagesRegex.findAll(script).map { match -> + val i = match.groupValues[1].toInt() + val url = match.groupValues[2] + MangaPage( + id = generateUid(url), + url = url, + preview = if (i <= MAX_THUMB_INDEX && mangaId != null) { + val part2 = url.substringBeforeLast('/') + val part3 = url.substringAfterLast('/') + val part1 = part2.substringBeforeLast('/') + "$part1/thumb/$mangaId/thumb_$part3" + } else { + null + }, + source = source, + ) + }.toList() + } + + override suspend fun getTags(): Set { + val domain = domain + val doc = webClient.httpGet("https://$domain/all_manga").parseHtml() + val root = doc.body().getElementsContainingOwnText("Поиск манги по тегам") + .firstOrNull()?.parents()?.find { it.tag().normalName() == "tbody" } + ?.selectFirst("td.textbox")?.selectFirst("td.small") + ?: doc.parseFailed("Tags root not found") + return root.select("a").mapToSet { + MangaTag( + title = it.text().toTitleCase(), + key = it.attr("href").substringAfterLast('/') + .removeSuffix("+"), + source = source, + ) + } + } + + override suspend fun getUsername(): String { + val body = webClient.httpGet("https://${domain}/").parseHtml() + .body() + return body + .getElementsContainingOwnText("Профиль") + .firstOrNull() + ?.attr("href") + ?.substringAfterLast('/') + ?: run { + throw if (body.selectFirst("form[name=\"loginform\"]") != null) { + AuthRequiredException(source) + } else { + body.parseFailed("Cannot find username") + } + } + } + + private fun getSortKey(sortOrder: SortOrder) = + when (sortOrder) { + SortOrder.POPULARITY -> "views" + SortOrder.NEWEST -> "date" + SortOrder.RATING -> "like" + else -> "like" + } + + private fun String.parseDate(): Long { + val dateString = substringBetweenFirst("Дата:", "<")?.trim() ?: return 0 + val dateFormat = SimpleDateFormat("d MMMM yyyy", Locale("ru")) + return dateFormat.tryParse(dateString) + } + + private fun getReadLink(url: String): String { + val prefix = url.substringBefore('-', "") + val suffix = url.substringAfter('-').trimStart('-') + return "$prefix-online-$suffix" + } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/RemangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/RemangaParser.kt index fa3a24dc..4ae288d5 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/RemangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/RemangaParser.kt @@ -28,259 +28,258 @@ private const val STATUS_FINISHED = 0 @MangaSourceParser("REMANGA", "Remanga", "ru") internal class RemangaParser( - context: MangaLoaderContext, + context: MangaLoaderContext, ) : PagedMangaParser(context, MangaSource.REMANGA, PAGE_SIZE), MangaParserAuthProvider { - private val baseHeaders = Headers.Builder() - .add("User-Agent", "Mozilla/5.0 (Android 13; Mobile; rv:68.0) Gecko/68.0 Firefox/109.0") - .build() + private val baseHeaders = Headers.Builder() + .add("User-Agent", "Mozilla/5.0 (Android 13; Mobile; rv:68.0) Gecko/68.0 Firefox/109.0") + .build() - override val headers - get() = getApiHeaders() + override val headers + get() = getApiHeaders() - override val configKeyDomain = ConfigKey.Domain("remanga.org", null) - override val authUrl: String - get() = "https://${domain}/user/login" + override val configKeyDomain = ConfigKey.Domain("remanga.org", null) + override val authUrl: String + get() = "https://${domain}/user/login" - override val sortOrders: Set = EnumSet.of( - SortOrder.UPDATED, - SortOrder.POPULARITY, - SortOrder.RATING, - SortOrder.NEWEST, - ) + override val sortOrders: Set = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.RATING, + SortOrder.NEWEST, + ) - override val isAuthorized: Boolean - get() { - return context.cookieJar.getCookies(domain).any { - it.name == "user" - } - } + override val isAuthorized: Boolean + get() { + return context.cookieJar.getCookies(domain).any { + it.name == "user" + } + } - private val regexLastUrlPath = Regex("/[^/]+/?$") + private val regexLastUrlPath = Regex("/[^/]+/?$") - override suspend fun getListPage( - page: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - copyCookies() - val domain = domain - val urlBuilder = StringBuilder() - .append("https://api.") - .append(domain) - if (query != null) { - urlBuilder.append("/api/search/?query=") - .append(query.urlEncoded()) - } else { - urlBuilder.append("/api/search/catalog/?ordering=") - .append(getSortKey(sortOrder)) - tags?.forEach { tag -> - urlBuilder.append("&genres=") - urlBuilder.append(tag.key) - } - } - urlBuilder - .append("&page=") - .append(page) - .append("&count=") - .append(PAGE_SIZE) - val content = webClient.httpGet(urlBuilder.toString()).parseJson() - .getJSONArray("content") - return content.mapJSON { jo -> - val url = "/manga/${jo.getString("dir")}" - val img = jo.getJSONObject("img") - Manga( - id = generateUid(url), - url = url, - publicUrl = "https://$domain$url", - title = jo.getString("rus_name"), - altTitle = jo.getString("en_name"), - rating = jo.getString("avg_rating").toFloatOrNull()?.div(10f) ?: RATING_UNKNOWN, - coverUrl = "https://api.$domain${img.getString("mid")}", - largeCoverUrl = "https://api.$domain${img.getString("high")}", - author = null, - isNsfw = false, - state = null, - tags = jo.optJSONArray("genres")?.mapJSONToSet { g -> - MangaTag( - title = g.getString("name").toTitleCase(), - key = g.getInt("id").toString(), - source = MangaSource.REMANGA, - ) - }.orEmpty(), - source = MangaSource.REMANGA, - ) - } - } + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + copyCookies() + val domain = domain + val urlBuilder = StringBuilder() + .append("https://api.") + .append(domain) + if (query != null) { + urlBuilder.append("/api/search/?query=") + .append(query.urlEncoded()) + } else { + urlBuilder.append("/api/search/catalog/?ordering=") + .append(getSortKey(sortOrder)) + tags?.forEach { tag -> + urlBuilder.append("&genres=") + urlBuilder.append(tag.key) + } + } + urlBuilder + .append("&page=") + .append(page) + .append("&count=") + .append(PAGE_SIZE) + val content = webClient.httpGet(urlBuilder.toString()).parseJson() + .getJSONArray("content") + return content.mapJSON { jo -> + val url = "/manga/${jo.getString("dir")}" + val img = jo.getJSONObject("img") + Manga( + id = generateUid(url), + url = url, + publicUrl = "https://$domain$url", + title = jo.getString("rus_name"), + altTitle = jo.getString("en_name"), + rating = jo.getString("avg_rating").toFloatOrNull()?.div(10f) ?: RATING_UNKNOWN, + coverUrl = "https://api.$domain${img.getString("mid")}", + largeCoverUrl = "https://api.$domain${img.getString("high")}", + author = null, + isNsfw = false, + state = null, + tags = jo.optJSONArray("genres")?.mapJSONToSet { g -> + MangaTag( + title = g.getString("name").toTitleCase(), + key = g.getInt("id").toString(), + source = MangaSource.REMANGA, + ) + }.orEmpty(), + source = MangaSource.REMANGA, + ) + } + } - override suspend fun getDetails(manga: Manga): Manga { - copyCookies() - val domain = domain - val slug = manga.url.find(regexLastUrlPath) - ?: throw ParseException("Cannot obtain slug from ${manga.url}", manga.publicUrl) - val data = webClient.httpGet( - url = "https://api.$domain/api/titles$slug/", - ).handle401().parseJson() - val content = try { - data.getJSONObject("content") - } catch (e: JSONException) { - throw ParseException(data.optString("msg"), manga.publicUrl, e) - } - val branchId = content.getJSONArray("branches").optJSONObject(0) - ?.getLong("id") ?: throw ParseException("No branches found", manga.publicUrl) - val chapters = grabChapters(domain, branchId) - val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.US) - return manga.copy( - description = content.getString("description"), - state = when (content.optJSONObject("status")?.getInt("id")) { - STATUS_ONGOING -> MangaState.ONGOING - STATUS_FINISHED -> MangaState.FINISHED - else -> null - }, - tags = content.getJSONArray("genres").mapJSONToSet { g -> - MangaTag( - title = g.getString("name").toTitleCase(), - key = g.getInt("id").toString(), - source = MangaSource.REMANGA, - ) - }, - chapters = chapters.mapChapters { i, jo -> - if ( - jo.getBooleanOrDefault("is_paid", false) && - !jo.getBooleanOrDefault("is_bought", false) - ) { - return@mapChapters null - } - val id = jo.getLong("id") - val name = jo.getString("name").toTitleCase(Locale.ROOT) - val publishers = jo.optJSONArray("publishers") - MangaChapter( - id = generateUid(id), - url = "/api/titles/chapters/$id/", - number = chapters.size - i, - name = buildString { - append("Том ") - append(jo.optString("tome", "0")) - append(". ") - append("Глава ") - append(jo.optString("chapter", "0")) - if (name.isNotEmpty()) { - append(" - ") - append(name) - } - }, - uploadDate = dateFormat.tryParse(jo.getString("upload_date")), - scanlator = publishers?.optJSONObject(0)?.getStringOrNull("name"), - source = MangaSource.REMANGA, - branch = null, - ) - }.asReversed(), - ) - } + override suspend fun getDetails(manga: Manga): Manga { + copyCookies() + val domain = domain + val slug = manga.url.find(regexLastUrlPath) + ?: throw ParseException("Cannot obtain slug from ${manga.url}", manga.publicUrl) + val data = webClient.httpGet( + url = "https://api.$domain/api/titles$slug/", + ).handle401().parseJson() + val content = try { + data.getJSONObject("content") + } catch (e: JSONException) { + throw ParseException(data.optString("msg"), manga.publicUrl, e) + } + val branchId = content.getJSONArray("branches").optJSONObject(0) + ?.getLong("id") ?: throw ParseException("No branches found", manga.publicUrl) + val chapters = grabChapters(domain, branchId) + val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.US) + return manga.copy( + description = content.getString("description"), + state = when (content.optJSONObject("status")?.getInt("id")) { + STATUS_ONGOING -> MangaState.ONGOING + STATUS_FINISHED -> MangaState.FINISHED + else -> null + }, + tags = content.getJSONArray("genres").mapJSONToSet { g -> + MangaTag( + title = g.getString("name").toTitleCase(), + key = g.getInt("id").toString(), + source = MangaSource.REMANGA, + ) + }, + chapters = chapters.mapChapters { i, jo -> + if ( + jo.getBooleanOrDefault("is_paid", false) && + !jo.getBooleanOrDefault("is_bought", false) + ) { + return@mapChapters null + } + val id = jo.getLong("id") + val name = jo.getString("name").toTitleCase(Locale.ROOT) + val publishers = jo.optJSONArray("publishers") + MangaChapter( + id = generateUid(id), + url = "/api/titles/chapters/$id/", + number = chapters.size - i, + name = buildString { + append("Том ") + append(jo.optString("tome", "0")) + append(". ") + append("Глава ") + append(jo.optString("chapter", "0")) + if (name.isNotEmpty()) { + append(" - ") + append(name) + } + }, + uploadDate = dateFormat.tryParse(jo.getString("upload_date")), + scanlator = publishers?.optJSONObject(0)?.getStringOrNull("name"), + source = MangaSource.REMANGA, + branch = null, + ) + }.asReversed(), + ) + } - override suspend fun getPages(chapter: MangaChapter): List { - val referer = "https://${domain}/" - val content = webClient.httpGet(chapter.url.toAbsoluteUrl(getDomain("api"))) - .handle401() - .parseJson() - .getJSONObject("content") - val pages = content.optJSONArray("pages") - if (pages == null) { - val pubDate = content.getStringOrNull("pub_date")?.let { - SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.US).tryParse(it) - } - if (pubDate != null && pubDate > System.currentTimeMillis()) { - val at = SimpleDateFormat.getDateInstance(DateFormat.LONG).format(Date(pubDate)) - throw ContentUnavailableException("Глава станет доступной $at") - } else { - throw ContentUnavailableException("Глава недоступна") - } - } - val result = ArrayList(pages.length()) - for (i in 0 until pages.length()) { - when (val item = pages.get(i)) { - is JSONObject -> result += parsePage(item, referer) - is JSONArray -> item.mapJSONTo(result) { parsePage(it, referer) } - else -> throw ParseException("Unknown json item $item", chapter.url) - } - } - return result - } + override suspend fun getPages(chapter: MangaChapter): List { + val referer = "https://${domain}/" + val content = webClient.httpGet(chapter.url.toAbsoluteUrl(getDomain("api"))) + .handle401() + .parseJson() + .getJSONObject("content") + val pages = content.optJSONArray("pages") + if (pages == null) { + val pubDate = content.getStringOrNull("pub_date")?.let { + SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.US).tryParse(it) + } + if (pubDate != null && pubDate > System.currentTimeMillis()) { + val at = SimpleDateFormat.getDateInstance(DateFormat.LONG).format(Date(pubDate)) + throw ContentUnavailableException("Глава станет доступной $at") + } else { + throw ContentUnavailableException("Глава недоступна") + } + } + val result = ArrayList(pages.length()) + for (i in 0 until pages.length()) { + when (val item = pages.get(i)) { + is JSONObject -> result += parsePage(item, referer) + is JSONArray -> item.mapJSONTo(result) { parsePage(it, referer) } + else -> throw ParseException("Unknown json item $item", chapter.url) + } + } + return result + } - override suspend fun getTags(): Set { - val domain = domain - val content = webClient.httpGet("https://api.$domain/api/forms/titles/?get=genres") - .parseJson().getJSONObject("content").getJSONArray("genres") - return content.mapJSONToSet { jo -> - MangaTag( - title = jo.getString("name").toTitleCase(), - key = jo.getInt("id").toString(), - source = source, - ) - } - } + override suspend fun getTags(): Set { + val domain = domain + val content = webClient.httpGet("https://api.$domain/api/forms/titles/?get=genres") + .parseJson().getJSONObject("content").getJSONArray("genres") + return content.mapJSONToSet { jo -> + MangaTag( + title = jo.getString("name").toTitleCase(), + key = jo.getInt("id").toString(), + source = source, + ) + } + } - override suspend fun getUsername(): String { - val jo = webClient.httpGet( - url = "https://api.${domain}/api/users/current/", - ).handle401().parseJson() - return jo.getJSONObject("content").getString("username") - } + override suspend fun getUsername(): String { + val jo = webClient.httpGet( + url = "https://api.${domain}/api/users/current/", + ).handle401().parseJson() + return jo.getJSONObject("content").getString("username") + } - private fun getApiHeaders(): Headers { - val userCookie = context.cookieJar.getCookies(domain).find { - it.name == "user" - } ?: return baseHeaders - val jo = JSONObject(URLDecoder.decode(userCookie.value, Charsets.UTF_8.name())) - val accessToken = jo.getStringOrNull("access_token") ?: return baseHeaders - return baseHeaders.newBuilder().add("authorization", "bearer $accessToken").build() - } + private fun getApiHeaders(): Headers { + val userCookie = context.cookieJar.getCookies(domain).find { + it.name == "user" + } ?: return baseHeaders + val jo = JSONObject(URLDecoder.decode(userCookie.value, Charsets.UTF_8.name())) + val accessToken = jo.getStringOrNull("access_token") ?: return baseHeaders + return baseHeaders.newBuilder().add("authorization", "bearer $accessToken").build() + } - private fun copyCookies() { - val domain = domain - context.cookieJar.copyCookies(domain, "api.$domain") - } + private fun copyCookies() { + val domain = domain + context.cookieJar.copyCookies(domain, "api.$domain") + } - private fun getSortKey(order: SortOrder?) = when (order) { - SortOrder.UPDATED -> "-chapter_date" - SortOrder.POPULARITY -> "-rating" - SortOrder.RATING -> "-votes" - SortOrder.NEWEST -> "-id" - else -> "-chapter_date" - } + private fun getSortKey(order: SortOrder?) = when (order) { + SortOrder.UPDATED -> "-chapter_date" + SortOrder.POPULARITY -> "-rating" + SortOrder.RATING -> "-votes" + SortOrder.NEWEST -> "-id" + else -> "-chapter_date" + } - private fun parsePage(jo: JSONObject, referer: String) = MangaPage( - id = generateUid(jo.getLong("id")), - url = jo.getString("link"), - preview = null, - referer = referer, - source = source, - ) + private fun parsePage(jo: JSONObject, referer: String) = MangaPage( + id = generateUid(jo.getLong("id")), + url = jo.getString("link"), + preview = null, + source = source, + ) - private suspend fun grabChapters(domain: String, branchId: Long): List { - val result = ArrayList(100) - var page = 1 - while (true) { - val content = webClient.httpGet( - url = "https://api.$domain/api/titles/chapters/?branch_id=$branchId&page=$page&count=100", - ).handle401().parseJson().getJSONArray("content") - val len = content.length() - if (len == 0) { - break - } - result.ensureCapacity(result.size + len) - for (i in 0 until len) { - result.add(content.getJSONObject(i)) - } - page++ - } - return result - } + private suspend fun grabChapters(domain: String, branchId: Long): List { + val result = ArrayList(100) + var page = 1 + while (true) { + val content = webClient.httpGet( + url = "https://api.$domain/api/titles/chapters/?branch_id=$branchId&page=$page&count=100", + ).handle401().parseJson().getJSONArray("content") + val len = content.length() + if (len == 0) { + break + } + result.ensureCapacity(result.size + len) + for (i in 0 until len) { + result.add(content.getJSONObject(i)) + } + page++ + } + return result + } - private fun Response.handle401() = apply { - if (code == HttpURLConnection.HTTP_UNAUTHORIZED) { - throw AuthRequiredException(source) - } - } + private fun Response.handle401() = apply { + if (code == HttpURLConnection.HTTP_UNAUTHORIZED) { + throw AuthRequiredException(source) + } + } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/TruyentranhLHParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/TruyentranhLHParser.kt index 715c3a1a..9c97582a 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/TruyentranhLHParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/TruyentranhLHParser.kt @@ -124,7 +124,6 @@ class TruyentranhLHParser(context: MangaLoaderContext) : MangaPage( id = generateUid(imageUrl), url = imageUrl, - referer = url, preview = null, source = source, ) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/UnionMangasParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/UnionMangasParser.kt index 53b959d0..491923a4 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/UnionMangasParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/UnionMangasParser.kt @@ -122,7 +122,6 @@ class UnionMangasParser(context: MangaLoaderContext) : PagedMangaParser(context, MangaPage( id = generateUid(href), url = href, - referer = fullUrl, preview = null, source = source, ) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt index fd975206..ca1bccb6 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt @@ -226,7 +226,6 @@ internal abstract class GroupleParser( id = generateUid(url), url = "$primaryServer|$serversStr|$url", preview = null, - referer = chapter.url, source = source, ) } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Madara5Parser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Madara5Parser.kt index 1b480d01..5c699324 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Madara5Parser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/Madara5Parser.kt @@ -112,7 +112,6 @@ abstract class Madara5Parser @InternalParsersApi constructor( MangaPage( id = generateUid(url), url = url, - referer = fullUrl, preview = null, source = source, ) @@ -180,7 +179,5 @@ abstract class Madara5Parser @InternalParsersApi constructor( ) @MangaSourceParser("MANGAOWLS", "BeautyManga", "en") - class BeautyManga(context: MangaLoaderContext) : Madara5Parser(context, MangaSource.MANGAOWLS, "beautymanga.com") { - - } + class BeautyManga(context: MangaLoaderContext) : Madara5Parser(context, MangaSource.MANGAOWLS, "beautymanga.com") } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt index e03e8550..4091e616 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/MadaraParser.kt @@ -186,7 +186,6 @@ internal abstract class MadaraParser( id = generateUid(url), url = url, preview = null, - referer = fullUrl, source = source, ) } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/NeatManga.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/NeatManga.kt index 3e8e4bb7..691938fc 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/NeatManga.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/madara/NeatManga.kt @@ -88,7 +88,6 @@ internal class NeatManga(context: MangaLoaderContext) : MadaraParser(context, Ma id = generateUid(url), url = url, preview = null, - referer = fullUrl, source = source, ) } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt index ef776dc5..6199f401 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/mangareader/MangaReaderParser.kt @@ -187,7 +187,6 @@ internal abstract class MangaReaderParser( MangaPage( id = generateUid(images.getString(i)), url = images.getString(i), - referer = chapterUrl, preview = null, source = source, ), diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/multichan/ChanParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/multichan/ChanParser.kt index 2a886fca..082cfb5b 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/multichan/ChanParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/multichan/ChanParser.kt @@ -10,174 +10,173 @@ import java.text.SimpleDateFormat import java.util.* internal abstract class ChanParser( - context: MangaLoaderContext, - source: MangaSource, + context: MangaLoaderContext, + source: MangaSource, ) : MangaParser(context, source), MangaParserAuthProvider { - override val sortOrders: Set = EnumSet.of( - SortOrder.NEWEST, - SortOrder.POPULARITY, - SortOrder.ALPHABETICAL, - ) + override val sortOrders: Set = EnumSet.of( + SortOrder.NEWEST, + SortOrder.POPULARITY, + SortOrder.ALPHABETICAL, + ) - override val authUrl: String - get() = "https://${domain}" + override val authUrl: String + get() = "https://${domain}" - override val isAuthorized: Boolean - get() = context.cookieJar.getCookies(domain).any { it.name == "dle_user_id" } + override val isAuthorized: Boolean + get() = context.cookieJar.getCookies(domain).any { it.name == "dle_user_id" } - override suspend fun getList( - offset: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - val domain = domain - val url = when { - !query.isNullOrEmpty() -> { - if (offset != 0) { - return emptyList() - } - "https://$domain/?do=search&subaction=search&story=${query.urlEncoded()}" - } + override suspend fun getList( + offset: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + val domain = domain + val url = when { + !query.isNullOrEmpty() -> { + if (offset != 0) { + return emptyList() + } + "https://$domain/?do=search&subaction=search&story=${query.urlEncoded()}" + } - !tags.isNullOrEmpty() -> tags.joinToString( - prefix = "https://$domain/tags/", - postfix = "&n=${getSortKey2(sortOrder)}?offset=$offset", - separator = "+", - ) { tag -> tag.key } + !tags.isNullOrEmpty() -> tags.joinToString( + prefix = "https://$domain/tags/", + postfix = "&n=${getSortKey2(sortOrder)}?offset=$offset", + separator = "+", + ) { tag -> tag.key } - else -> "https://$domain/${getSortKey(sortOrder)}?offset=$offset" - } - val doc = webClient.httpGet(url).parseHtml() - val root = doc.body().selectFirst("div.main_fon")?.getElementById("content") - ?: doc.parseFailed("Cannot find root") - return root.select("div.content_row").mapNotNull { row -> - val a = row.selectFirst("div.manga_row1")?.selectFirst("h2")?.selectFirst("a") - ?: return@mapNotNull null - val href = a.attrAsRelativeUrl("href") - Manga( - id = generateUid(href), - url = href, - publicUrl = href.toAbsoluteUrl(a.host ?: domain), - altTitle = a.attr("title"), - title = a.text().substringAfterLast('(').substringBeforeLast(')'), - author = row.getElementsByAttributeValueStarting( - "href", - "/mangaka", - ).firstOrNull()?.text(), - coverUrl = row.selectFirst("div.manga_images")?.selectFirst("img") - ?.absUrl("src").orEmpty(), - tags = runCatching { - row.selectFirst("div.genre")?.select("a")?.mapToSet { - MangaTag( - title = it.text().toTagName(), - key = it.attr("href").substringAfterLast('/').urlEncoded(), - source = source, - ) - } - }.getOrNull().orEmpty(), - rating = RATING_UNKNOWN, - state = null, - isNsfw = false, - source = source, - ) - } - } + else -> "https://$domain/${getSortKey(sortOrder)}?offset=$offset" + } + val doc = webClient.httpGet(url).parseHtml() + val root = doc.body().selectFirst("div.main_fon")?.getElementById("content") + ?: doc.parseFailed("Cannot find root") + return root.select("div.content_row").mapNotNull { row -> + val a = row.selectFirst("div.manga_row1")?.selectFirst("h2")?.selectFirst("a") + ?: return@mapNotNull null + val href = a.attrAsRelativeUrl("href") + Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(a.host ?: domain), + altTitle = a.attr("title"), + title = a.text().substringAfterLast('(').substringBeforeLast(')'), + author = row.getElementsByAttributeValueStarting( + "href", + "/mangaka", + ).firstOrNull()?.text(), + coverUrl = row.selectFirst("div.manga_images")?.selectFirst("img") + ?.absUrl("src").orEmpty(), + tags = runCatching { + row.selectFirst("div.genre")?.select("a")?.mapToSet { + MangaTag( + title = it.text().toTagName(), + key = it.attr("href").substringAfterLast('/').urlEncoded(), + source = source, + ) + } + }.getOrNull().orEmpty(), + rating = RATING_UNKNOWN, + state = null, + isNsfw = false, + source = source, + ) + } + } - override suspend fun getDetails(manga: Manga): Manga { - val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - val root = doc.body().getElementById("dle-content") ?: doc.parseFailed("Cannot find root") - val dateFormat = SimpleDateFormat("yyyy-MM-dd", Locale.US) - return manga.copy( - description = root.getElementById("description")?.html()?.substringBeforeLast(" - val href = tr?.selectFirst("a")?.attrAsRelativeUrlOrNull("href") - ?: return@mapChapters null - MangaChapter( - id = generateUid(href), - name = tr.selectFirst("a")?.text().orEmpty(), - number = i + 1, - url = href, - scanlator = null, - branch = null, - uploadDate = dateFormat.tryParse(tr.selectFirst("div.date")?.text()), - source = source, - ) - }, - ) - } + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val root = doc.body().getElementById("dle-content") ?: doc.parseFailed("Cannot find root") + val dateFormat = SimpleDateFormat("yyyy-MM-dd", Locale.US) + return manga.copy( + description = root.getElementById("description")?.html()?.substringBeforeLast(" + val href = tr?.selectFirst("a")?.attrAsRelativeUrlOrNull("href") + ?: return@mapChapters null + MangaChapter( + id = generateUid(href), + name = tr.selectFirst("a")?.text().orEmpty(), + number = i + 1, + url = href, + scanlator = null, + branch = null, + uploadDate = dateFormat.tryParse(tr.selectFirst("div.date")?.text()), + source = source, + ) + }, + ) + } - override suspend fun getPages(chapter: MangaChapter): List { - val fullUrl = chapter.url.toAbsoluteUrl(domain) - val doc = webClient.httpGet(fullUrl).parseHtml() - val scripts = doc.select("script") - for (script in scripts) { - val data = script.html() - val pos = data.indexOf("\"fullimg") - if (pos == -1) { - continue - } - val json = data.substring(pos).substringAfter('[').substringBefore(';') - .substringBeforeLast(']') - val domain = domain - return json.split(",").mapNotNull { - it.trim() - .removeSurrounding('"', '\'') - .toRelativeUrl(domain) - .takeUnless(String::isBlank) - }.map { url -> - MangaPage( - id = generateUid(url), - url = url, - preview = null, - referer = fullUrl, - source = source, - ) - } - } - doc.parseFailed("Pages list not found at ${chapter.url}") - } + override suspend fun getPages(chapter: MangaChapter): List { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + val scripts = doc.select("script") + for (script in scripts) { + val data = script.html() + val pos = data.indexOf("\"fullimg") + if (pos == -1) { + continue + } + val json = data.substring(pos).substringAfter('[').substringBefore(';') + .substringBeforeLast(']') + val domain = domain + return json.split(",").mapNotNull { + it.trim() + .removeSurrounding('"', '\'') + .toRelativeUrl(domain) + .takeUnless(String::isBlank) + }.map { url -> + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } + doc.parseFailed("Pages list not found at ${chapter.url}") + } - override suspend fun getTags(): Set { - val domain = domain - val doc = webClient.httpGet("https://$domain/mostfavorites&sort=manga").parseHtml() - val root = doc.body().selectFirst("div.main_fon")?.getElementById("side") - ?.select("ul")?.last() ?: doc.parseFailed("Cannot find root") - return root.select("li.sidetag").mapToSet { li -> - val a = li.children().lastOrNull() ?: li.parseFailed("a is null") - MangaTag( - title = a.text().toTagName(), - key = a.attr("href").substringAfterLast('/'), - source = source, - ) - } - } + override suspend fun getTags(): Set { + val domain = domain + val doc = webClient.httpGet("https://$domain/mostfavorites&sort=manga").parseHtml() + val root = doc.body().selectFirst("div.main_fon")?.getElementById("side") + ?.select("ul")?.last() ?: doc.parseFailed("Cannot find root") + return root.select("li.sidetag").mapToSet { li -> + val a = li.children().lastOrNull() ?: li.parseFailed("a is null") + MangaTag( + title = a.text().toTagName(), + key = a.attr("href").substringAfterLast('/'), + source = source, + ) + } + } - override suspend fun getUsername(): String { - val doc = webClient.httpGet("https://${domain}").parseHtml().body() - val root = doc.requireElementById("top_user") - val a = root.getElementsByAttributeValueContaining("href", "/user/").firstOrNull() - ?: throw AuthRequiredException(source) - return a.attr("href").removeSuffix('/').substringAfterLast('/') - } + override suspend fun getUsername(): String { + val doc = webClient.httpGet("https://${domain}").parseHtml().body() + val root = doc.requireElementById("top_user") + val a = root.getElementsByAttributeValueContaining("href", "/user/").firstOrNull() + ?: throw AuthRequiredException(source) + return a.attr("href").removeSuffix('/').substringAfterLast('/') + } - private fun getSortKey(sortOrder: SortOrder) = - when (sortOrder) { - SortOrder.ALPHABETICAL -> "catalog" - SortOrder.POPULARITY -> "mostfavorites" - SortOrder.NEWEST -> "manga/new" - else -> "mostfavorites" - } + private fun getSortKey(sortOrder: SortOrder) = + when (sortOrder) { + SortOrder.ALPHABETICAL -> "catalog" + SortOrder.POPULARITY -> "mostfavorites" + SortOrder.NEWEST -> "manga/new" + else -> "mostfavorites" + } - private fun getSortKey2(sortOrder: SortOrder) = - when (sortOrder) { - SortOrder.ALPHABETICAL -> "abcasc" - SortOrder.POPULARITY -> "favdesc" - SortOrder.NEWEST -> "datedesc" - else -> "favdesc" - } + private fun getSortKey2(sortOrder: SortOrder) = + when (sortOrder) { + SortOrder.ALPHABETICAL -> "abcasc" + SortOrder.POPULARITY -> "favdesc" + SortOrder.NEWEST -> "datedesc" + else -> "favdesc" + } - private fun String.toTagName() = replace('_', ' ').toTitleCase() + private fun String.toTagName() = replace('_', ' ').toTitleCase() } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/rulib/MangaLibParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/rulib/MangaLibParser.kt index d00dd2dd..ed6fba15 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/rulib/MangaLibParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/rulib/MangaLibParser.kt @@ -20,284 +20,283 @@ import java.text.SimpleDateFormat import java.util.* internal open class MangaLibParser( - context: MangaLoaderContext, - source: MangaSource, + context: MangaLoaderContext, + source: MangaSource, ) : PagedMangaParser(context, source, pageSize = 60), MangaParserAuthProvider { - override val configKeyDomain = ConfigKey.Domain("mangalib.me", null) + override val configKeyDomain = ConfigKey.Domain("mangalib.me", null) - override val authUrl: String - get() = "https://${domain}/login" + override val authUrl: String + get() = "https://${domain}/login" - override val sortOrders: Set = EnumSet.of( - SortOrder.RATING, - SortOrder.ALPHABETICAL, - SortOrder.POPULARITY, - SortOrder.UPDATED, - SortOrder.NEWEST, - ) + override val sortOrders: Set = EnumSet.of( + SortOrder.RATING, + SortOrder.ALPHABETICAL, + SortOrder.POPULARITY, + SortOrder.UPDATED, + SortOrder.NEWEST, + ) - override suspend fun getListPage( - page: Int, - query: String?, - tags: Set?, - sortOrder: SortOrder, - ): List { - if (!query.isNullOrEmpty()) { - return if (page == searchPaginator.firstPage) search(query) else emptyList() - } - val url = buildString { - append("https://") - append(domain) - append("/manga-list?dir=") - append(getSortKey(sortOrder)) - append("&page=") - append(page) - tags?.forEach { tag -> - append("&genres[include][]=") - append(tag.key) - } - } - val doc = webClient.httpGet(url).parseHtml() - val root = doc.body().getElementById("manga-list") ?: doc.parseFailed("Root not found") - val items = root.selectFirst("div.media-cards-grid")?.select("div.media-card-wrap") - ?: return emptyList() - return items.mapNotNull { card -> - val a = card.selectFirst("a.media-card") ?: return@mapNotNull null - val href = a.attrAsRelativeUrl("href") - Manga( - id = generateUid(href), - title = card.selectFirst("h3")?.text().orEmpty(), - coverUrl = a.absUrl("data-src"), - altTitle = null, - author = null, - rating = RATING_UNKNOWN, - url = href, - publicUrl = href.toAbsoluteUrl(a.host ?: domain), - tags = emptySet(), - state = null, - isNsfw = false, - source = source, - ) - } - } + override suspend fun getListPage( + page: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder, + ): List { + if (!query.isNullOrEmpty()) { + return if (page == searchPaginator.firstPage) search(query) else emptyList() + } + val url = buildString { + append("https://") + append(domain) + append("/manga-list?dir=") + append(getSortKey(sortOrder)) + append("&page=") + append(page) + tags?.forEach { tag -> + append("&genres[include][]=") + append(tag.key) + } + } + val doc = webClient.httpGet(url).parseHtml() + val root = doc.body().getElementById("manga-list") ?: doc.parseFailed("Root not found") + val items = root.selectFirst("div.media-cards-grid")?.select("div.media-card-wrap") + ?: return emptyList() + return items.mapNotNull { card -> + val a = card.selectFirst("a.media-card") ?: return@mapNotNull null + val href = a.attrAsRelativeUrl("href") + Manga( + id = generateUid(href), + title = card.selectFirst("h3")?.text().orEmpty(), + coverUrl = a.absUrl("data-src"), + altTitle = null, + author = null, + rating = RATING_UNKNOWN, + url = href, + publicUrl = href.toAbsoluteUrl(a.host ?: domain), + tags = emptySet(), + state = null, + isNsfw = false, + source = source, + ) + } + } - override suspend fun getDetails(manga: Manga): Manga { - val fullUrl = manga.url.toAbsoluteUrl(domain) - val doc = webClient.httpGet("$fullUrl?section=info").parseHtml() - val root = doc.body().getElementById("main-page") ?: throw ParseException("Root not found", fullUrl) - val title = root.selectFirst("div.media-header__wrap")?.children() - val info = root.selectFirst("div.media-content") - val chaptersDoc = webClient.httpGet("$fullUrl?section=chapters").parseHtml() - val scripts = chaptersDoc.select("script") - val dateFormat = SimpleDateFormat("yyy-MM-dd", Locale.US) - var chapters: ChaptersListBuilder? = null - scripts@ for (script in scripts) { - val raw = script.html().lines() - for (line in raw) { - if (line.startsWith("window.__DATA__")) { - val json = JSONObject(line.substringAfter('=').substringBeforeLast(';')) - val list = json.getJSONObject("chapters").getJSONArray("list") - val id = json.optJSONObject("user")?.getLong("id")?.toString() ?: "not" - val total = list.length() - chapters = ChaptersListBuilder(total) - for (i in 0 until total) { - val item = list.getJSONObject(i) - val chapterId = item.getLong("chapter_id") - val scanlator = item.getStringOrNull("username") - val url = buildString { - if (isAuthorized) { - append(manga.url) - append("/v") - append(item.getInt("chapter_volume")) - append("/c") - append(item.getString("chapter_number")) - append("?ui=") - append(id) - } else { - append(manga.url) - append("/v") - append(item.getInt("chapter_volume")) - append("/c") - append(item.getString("chapter_number")) - } - } - val nameChapter = item.getStringOrNull("chapter_name") - val volume = item.getInt("chapter_volume") - val number = item.getString("chapter_number") - val fullNameChapter = "Том $volume. Глава $number" - chapters.add( - MangaChapter( - id = generateUid(chapterId), - url = url, - source = source, - number = total - i, - uploadDate = dateFormat.tryParse( - item.getString("chapter_created_at").substringBefore(" "), - ), - scanlator = scanlator, - branch = null, - name = if (nameChapter.isNullOrBlank()) fullNameChapter else "$fullNameChapter - $nameChapter", - ), - ) - } - chapters.reverse() - break@scripts - } - } - } - return manga.copy( - title = title?.getOrNull(0)?.text()?.takeUnless(String::isBlank) ?: manga.title, - altTitle = title?.getOrNull(1)?.text()?.substringBefore('/')?.trim(), - rating = root.selectFirst("div.media-stats-item__score") - ?.selectFirst("span") - ?.text()?.toFloatOrNull()?.div(5f) ?: manga.rating, - author = info?.getElementsMatchingOwnText("Автор")?.firstOrNull() - ?.nextElementSibling()?.text() ?: manga.author, - tags = info?.selectFirst("div.media-tags") - ?.select("a.media-tag-item")?.mapNotNullToSet { a -> - val href = a.attr("href") - if (href.contains("genres")) { - MangaTag( - title = a.text().toTitleCase(), - key = href.substringAfterLast('='), - source = source, - ) - } else null - } ?: manga.tags, - isNsfw = isNsfw(doc), - description = info?.selectFirst("div.media-description__text")?.html(), - chapters = chapters?.toList(), - ) - } + override suspend fun getDetails(manga: Manga): Manga { + val fullUrl = manga.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet("$fullUrl?section=info").parseHtml() + val root = doc.body().getElementById("main-page") ?: throw ParseException("Root not found", fullUrl) + val title = root.selectFirst("div.media-header__wrap")?.children() + val info = root.selectFirst("div.media-content") + val chaptersDoc = webClient.httpGet("$fullUrl?section=chapters").parseHtml() + val scripts = chaptersDoc.select("script") + val dateFormat = SimpleDateFormat("yyy-MM-dd", Locale.US) + var chapters: ChaptersListBuilder? = null + scripts@ for (script in scripts) { + val raw = script.html().lines() + for (line in raw) { + if (line.startsWith("window.__DATA__")) { + val json = JSONObject(line.substringAfter('=').substringBeforeLast(';')) + val list = json.getJSONObject("chapters").getJSONArray("list") + val id = json.optJSONObject("user")?.getLong("id")?.toString() ?: "not" + val total = list.length() + chapters = ChaptersListBuilder(total) + for (i in 0 until total) { + val item = list.getJSONObject(i) + val chapterId = item.getLong("chapter_id") + val scanlator = item.getStringOrNull("username") + val url = buildString { + if (isAuthorized) { + append(manga.url) + append("/v") + append(item.getInt("chapter_volume")) + append("/c") + append(item.getString("chapter_number")) + append("?ui=") + append(id) + } else { + append(manga.url) + append("/v") + append(item.getInt("chapter_volume")) + append("/c") + append(item.getString("chapter_number")) + } + } + val nameChapter = item.getStringOrNull("chapter_name") + val volume = item.getInt("chapter_volume") + val number = item.getString("chapter_number") + val fullNameChapter = "Том $volume. Глава $number" + chapters.add( + MangaChapter( + id = generateUid(chapterId), + url = url, + source = source, + number = total - i, + uploadDate = dateFormat.tryParse( + item.getString("chapter_created_at").substringBefore(" "), + ), + scanlator = scanlator, + branch = null, + name = if (nameChapter.isNullOrBlank()) fullNameChapter else "$fullNameChapter - $nameChapter", + ), + ) + } + chapters.reverse() + break@scripts + } + } + } + return manga.copy( + title = title?.getOrNull(0)?.text()?.takeUnless(String::isBlank) ?: manga.title, + altTitle = title?.getOrNull(1)?.text()?.substringBefore('/')?.trim(), + rating = root.selectFirst("div.media-stats-item__score") + ?.selectFirst("span") + ?.text()?.toFloatOrNull()?.div(5f) ?: manga.rating, + author = info?.getElementsMatchingOwnText("Автор")?.firstOrNull() + ?.nextElementSibling()?.text() ?: manga.author, + tags = info?.selectFirst("div.media-tags") + ?.select("a.media-tag-item")?.mapNotNullToSet { a -> + val href = a.attr("href") + if (href.contains("genres")) { + MangaTag( + title = a.text().toTitleCase(), + key = href.substringAfterLast('='), + source = source, + ) + } else null + } ?: manga.tags, + isNsfw = isNsfw(doc), + description = info?.selectFirst("div.media-description__text")?.html(), + chapters = chapters?.toList(), + ) + } - override suspend fun getPages(chapter: MangaChapter): List { - val fullUrl = chapter.url.toAbsoluteUrl(domain) - val doc = webClient.httpGet(fullUrl).parseHtml() - if (doc.location().substringBefore('?').endsWith("/register")) { - throw AuthRequiredException(source) - } - val scripts = doc.head().select("script") - val pg = (doc.body().getElementById("pg")?.html() ?: doc.parseFailed("Element #pg not found")) - .substringAfter('=') - .substringBeforeLast(';') - val pages = JSONArray(pg) - for (script in scripts) { - val raw = script.html().trim() - if (raw.contains("window.__info")) { - val json = JSONObject( - raw.substringAfter("window.__info") - .substringAfter('=') - .substringBeforeLast(';'), - ) - val domain = json.getJSONObject("servers").run { - getStringOrNull("main") ?: getString( - json.getJSONObject("img").getString("server"), - ) - } - val url = json.getJSONObject("img").getString("url") - return pages.mapJSON { x -> - val pageUrl = "$domain/$url${x.getString("u")}" - MangaPage( - id = generateUid(pageUrl), - url = pageUrl, - preview = null, - referer = fullUrl, - source = source, - ) - } - } - } - throw ParseException("Script with info not found", fullUrl) - } + override suspend fun getPages(chapter: MangaChapter): List { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + if (doc.location().substringBefore('?').endsWith("/register")) { + throw AuthRequiredException(source) + } + val scripts = doc.head().select("script") + val pg = (doc.body().getElementById("pg")?.html() ?: doc.parseFailed("Element #pg not found")) + .substringAfter('=') + .substringBeforeLast(';') + val pages = JSONArray(pg) + for (script in scripts) { + val raw = script.html().trim() + if (raw.contains("window.__info")) { + val json = JSONObject( + raw.substringAfter("window.__info") + .substringAfter('=') + .substringBeforeLast(';'), + ) + val domain = json.getJSONObject("servers").run { + getStringOrNull("main") ?: getString( + json.getJSONObject("img").getString("server"), + ) + } + val url = json.getJSONObject("img").getString("url") + return pages.mapJSON { x -> + val pageUrl = "$domain/$url${x.getString("u")}" + MangaPage( + id = generateUid(pageUrl), + url = pageUrl, + preview = null, + source = source, + ) + } + } + } + throw ParseException("Script with info not found", fullUrl) + } - override suspend fun getTags(): Set { - val url = "https://${domain}/manga-list" - val doc = webClient.httpGet(url).parseHtml() - val scripts = doc.body().select("script") - for (script in scripts) { - val raw = script.html().trim() - if (raw.startsWith("window.__DATA")) { - val json = JSONObject(raw.substringAfter('=').substringBeforeLast(';')) - val genres = json.getJSONObject("filters").getJSONArray("genres") - val result = ArraySet(genres.length()) - for (x in genres.JSONIterator()) { - result += MangaTag( - source = source, - key = x.getInt("id").toString(), - title = x.getString("name").toTitleCase(), - ) - } - return result - } - } - throw ParseException("Script with genres not found", url) - } + override suspend fun getTags(): Set { + val url = "https://${domain}/manga-list" + val doc = webClient.httpGet(url).parseHtml() + val scripts = doc.body().select("script") + for (script in scripts) { + val raw = script.html().trim() + if (raw.startsWith("window.__DATA")) { + val json = JSONObject(raw.substringAfter('=').substringBeforeLast(';')) + val genres = json.getJSONObject("filters").getJSONArray("genres") + val result = ArraySet(genres.length()) + for (x in genres.JSONIterator()) { + result += MangaTag( + source = source, + key = x.getInt("id").toString(), + title = x.getString("name").toTitleCase(), + ) + } + return result + } + } + throw ParseException("Script with genres not found", url) + } - override val isAuthorized: Boolean - get() { - return context.cookieJar.getCookies(domain).any { - it.name.startsWith("remember_web_") - } - } + override val isAuthorized: Boolean + get() { + return context.cookieJar.getCookies(domain).any { + it.name.startsWith("remember_web_") + } + } - override suspend fun getUsername(): String { - val body = webClient.httpGet("https://${LibConst.LIB_SOCIAL_LINK}/messages").parseHtml().body() - if (body.baseUri().endsWith("/login")) { - throw AuthRequiredException(source) - } - return body.selectFirst(".profile-user__username")?.text() ?: body.parseFailed("Cannot find username") - } + override suspend fun getUsername(): String { + val body = webClient.httpGet("https://${LibConst.LIB_SOCIAL_LINK}/messages").parseHtml().body() + if (body.baseUri().endsWith("/login")) { + throw AuthRequiredException(source) + } + return body.selectFirst(".profile-user__username")?.text() ?: body.parseFailed("Cannot find username") + } - protected open fun isNsfw(doc: Document): Boolean { - val sidebar = doc.body().run { - selectFirst(".media-sidebar") ?: selectFirst(".media-info") - } ?: doc.parseFailed("Sidebar not found") - return sidebar.getElementsContainingOwnText("18+").isNotEmpty() - } + protected open fun isNsfw(doc: Document): Boolean { + val sidebar = doc.body().run { + selectFirst(".media-sidebar") ?: selectFirst(".media-info") + } ?: doc.parseFailed("Sidebar not found") + return sidebar.getElementsContainingOwnText("18+").isNotEmpty() + } - private fun getSortKey(sortOrder: SortOrder?) = when (sortOrder) { - SortOrder.RATING -> "desc&sort=rate" - SortOrder.ALPHABETICAL -> "asc&sort=name" - SortOrder.POPULARITY -> "desc&sort=views" - SortOrder.UPDATED -> "desc&sort=last_chapter_at" - SortOrder.NEWEST -> "desc&sort=created_at" - else -> "desc&sort=last_chapter_at" - } + private fun getSortKey(sortOrder: SortOrder?) = when (sortOrder) { + SortOrder.RATING -> "desc&sort=rate" + SortOrder.ALPHABETICAL -> "asc&sort=name" + SortOrder.POPULARITY -> "desc&sort=views" + SortOrder.UPDATED -> "desc&sort=last_chapter_at" + SortOrder.NEWEST -> "desc&sort=created_at" + else -> "desc&sort=last_chapter_at" + } - private suspend fun search(query: String): List { - val domain = domain - val json = webClient.httpGet("https://$domain/search?type=manga&q=$query") - .parseJsonArray() - return json.mapJSON { jo -> - val slug = jo.getString("slug") - val url = "/$slug" - val covers = jo.getJSONObject("covers") - val title = jo.getString("rus_name").ifEmpty { jo.getString("name") } - Manga( - id = generateUid(url), - url = url, - publicUrl = "https://$domain/$slug", - title = title, - altTitle = jo.getString("name").takeUnless { it == title }, - author = null, - tags = emptySet(), - rating = jo.getString("rate_avg").toFloatOrNull()?.div(5f) ?: RATING_UNKNOWN, - state = null, - isNsfw = false, - source = source, - coverUrl = covers.getString("thumbnail").toAbsoluteUrl(domain), - largeCoverUrl = covers.getString("default").toAbsoluteUrl(domain), - ) - } - } + private suspend fun search(query: String): List { + val domain = domain + val json = webClient.httpGet("https://$domain/search?type=manga&q=$query") + .parseJsonArray() + return json.mapJSON { jo -> + val slug = jo.getString("slug") + val url = "/$slug" + val covers = jo.getJSONObject("covers") + val title = jo.getString("rus_name").ifEmpty { jo.getString("name") } + Manga( + id = generateUid(url), + url = url, + publicUrl = "https://$domain/$slug", + title = title, + altTitle = jo.getString("name").takeUnless { it == title }, + author = null, + tags = emptySet(), + rating = jo.getString("rate_avg").toFloatOrNull()?.div(5f) ?: RATING_UNKNOWN, + state = null, + isNsfw = false, + source = source, + coverUrl = covers.getString("thumbnail").toAbsoluteUrl(domain), + largeCoverUrl = covers.getString("default").toAbsoluteUrl(domain), + ) + } + } - @MangaSourceParser("MANGALIB", "MangaLib", "ru") - class Impl(context: MangaLoaderContext) : MangaLibParser(context, MangaSource.MANGALIB) + @MangaSourceParser("MANGALIB", "MangaLib", "ru") + class Impl(context: MangaLoaderContext) : MangaLibParser(context, MangaSource.MANGALIB) - object LibConst { + object LibConst { - val LIB_SOCIAL_LINK = "lib.social" + val LIB_SOCIAL_LINK = "lib.social" - } + } }