From 81975977adfc702210bb5feefcf405f9955dd5ff Mon Sep 17 00:00:00 2001 From: Naga <94557604+NagaYZ@users.noreply.github.com> Date: Sat, 9 Mar 2024 23:44:01 +0100 Subject: [PATCH 01/10] added search support for reaperscans --- .../kotatsu/parsers/site/en/ReaperComics.kt | 63 +++++++++++++++---- 1 file changed, 51 insertions(+), 12 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/ReaperComics.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/ReaperComics.kt index 73b7e760..c265d016 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/ReaperComics.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/ReaperComics.kt @@ -2,7 +2,6 @@ package org.koitharu.kotatsu.parsers.site.en import okhttp3.Headers import org.jsoup.nodes.Document -import org.koitharu.kotatsu.parsers.ErrorMessages import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaSourceParser import org.koitharu.kotatsu.parsers.PagedMangaParser @@ -12,30 +11,36 @@ import org.koitharu.kotatsu.parsers.network.UserAgents import org.koitharu.kotatsu.parsers.util.* import java.text.DateFormat import java.text.SimpleDateFormat -import java.util.* +import java.util.Calendar +import java.util.EnumSet @MangaSourceParser("REAPERCOMICS", "ReaperComics", "en") internal class ReaperComics(context: MangaLoaderContext) : - PagedMangaParser(context, MangaSource.REAPERCOMICS, pageSize = 30) { + PagedMangaParser(context, MangaSource.REAPERCOMICS, pageSize = 32) { override val availableSortOrders: Set = EnumSet.of(SortOrder.UPDATED, SortOrder.ALPHABETICAL) override val configKeyDomain = ConfigKey.Domain("reaperscans.com") - override val isSearchSupported = false + override val headers: Headers = Headers.Builder().add("User-Agent", UserAgents.CHROME_DESKTOP).build() - override val headers: Headers = Headers.Builder() - .add("User-Agent", UserAgents.CHROME_DESKTOP) - .build() + private val searchCache = mutableSetOf() // Cache search results override suspend fun getListPage(page: Int, filter: MangaListFilter?): List { - val url = buildString { append("https://") append(domain) when (filter) { is MangaListFilter.Search -> { - throw IllegalArgumentException(ErrorMessages.SEARCH_NOT_SUPPORTED) // TODO + val searchTitle = filter.query.trim() + if (searchCache.isNotEmpty()) { + if (page > 1) { + return emptyList() + } + return searchCache.filter { it.title.contains(searchTitle, ignoreCase = true) } + } else { + return searchAllPage(page, searchTitle) + } } is MangaListFilter.Advanced -> { @@ -53,13 +58,47 @@ internal class ReaperComics(context: MangaLoaderContext) : } } } - return parseMangaList(webClient.httpGet(url).parseHtml()) } - private fun parseMangaList(docs: Document): List { - + /** + * Search once all pages and stores them in cache + * + * @param page the page to start from + * @param searchTitle the title to search for + * @return the list of manga + */ + private suspend fun searchAllPage(page: Int, searchTitle: String): List { + var currentPage = page + val url = buildString { + append("https://") + append(domain) + append("/comics?page=") + } + while (true) { + try { + val allEntries = parseMangaList(webClient.httpGet(url + currentPage).parseHtml()) + if (allEntries.isEmpty()) { + break + } + searchCache.addAll(allEntries) + currentPage++ + } catch (e: Exception) { + println("Error parsing page $currentPage: ${e.message}") + break + } + } + return searchCache.filter { it.title.contains(searchTitle, ignoreCase = true) }.toList() + } + /** + * Parse the list of manga from the given document + * + * @param docs the document to parse + * @param title the title to search for + * @return the list of manga + */ + private fun parseMangaList(docs: Document): List { return docs.select("main div.relative, main li.col-span-1").map { val a = it.selectFirstOrThrow("a") val url = a.attrAsAbsoluteUrl("href") From 6d8d75779846de57f490a2293c1742b4d304fa7d Mon Sep 17 00:00:00 2001 From: Naga <94557604+NagaYZ@users.noreply.github.com> Date: Sun, 10 Mar 2024 23:38:28 +0100 Subject: [PATCH 02/10] fix manga chapters list --- build.gradle | 2 + .../kotatsu/parsers/site/en/ReaperComics.kt | 224 ++++++++++++++++-- 2 files changed, 211 insertions(+), 15 deletions(-) diff --git a/build.gradle b/build.gradle index c5412694..667c6b8d 100644 --- a/build.gradle +++ b/build.gradle @@ -5,6 +5,7 @@ plugins { id 'org.jetbrains.kotlin.jvm' version '1.9.22' id 'com.google.devtools.ksp' version '1.9.22-1.0.17' id 'maven-publish' + id 'org.jetbrains.kotlin.plugin.serialization' version '1.9.22' } group = 'org.koitharu' @@ -59,6 +60,7 @@ dependencies { api 'org.jsoup:jsoup:1.17.2' implementation 'org.json:json:20231013' implementation 'androidx.collection:collection:1.4.0' + implementation "org.jetbrains.kotlinx:kotlinx-serialization-json:1.6.3" ksp project(':kotatsu-parsers-ksp') diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/ReaperComics.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/ReaperComics.kt index c265d016..e0fa9045 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/ReaperComics.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/ReaperComics.kt @@ -1,18 +1,31 @@ package org.koitharu.kotatsu.parsers.site.en -import okhttp3.Headers +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.delay +import kotlinx.coroutines.withContext +import kotlinx.serialization.Serializable +import kotlinx.serialization.json.* +import okhttp3.* +import okhttp3.MediaType.Companion.toMediaType +import okhttp3.RequestBody.Companion.toRequestBody +import org.json.JSONObject +import org.jsoup.Jsoup import org.jsoup.nodes.Document import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaSourceParser import org.koitharu.kotatsu.parsers.PagedMangaParser import org.koitharu.kotatsu.parsers.config.ConfigKey import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.network.UserAgents import org.koitharu.kotatsu.parsers.util.* +import org.koitharu.kotatsu.parsers.util.json.getStringOrNull import java.text.DateFormat import java.text.SimpleDateFormat import java.util.Calendar import java.util.EnumSet +import kotlin.random.Random + +private const val TOO_MANY_REQUESTS = 429 +private const val MAX_RETRY_COUNT = 3 @MangaSourceParser("REAPERCOMICS", "ReaperComics", "en") internal class ReaperComics(context: MangaLoaderContext) : @@ -22,10 +35,31 @@ internal class ReaperComics(context: MangaLoaderContext) : override val configKeyDomain = ConfigKey.Domain("reaperscans.com") - override val headers: Headers = Headers.Builder().add("User-Agent", UserAgents.CHROME_DESKTOP).build() + private val userAgentKey = + ConfigKey.UserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36") + + private val baseHeaders: Headers + get() = Headers.Builder().add("User-Agent", config[userAgentKey]).build() + + override val headers + get() = getApiHeaders() + + private val selectTotalChapter = "dl.mt-2 div:nth-child(5) > dd" + private val selectState = "dl.mt-2 div:nth-child(4) > dd" private val searchCache = mutableSetOf() // Cache search results + private val baseUrl = "https://reaperscans.com" + + private fun getApiHeaders(): Headers { + val userCookie = context.cookieJar.getCookies(domain).find { + it.name == "user" + } ?: return baseHeaders + val jo = JSONObject(userCookie.value.urlDecode()) + val accessToken = jo.getStringOrNull("access_token") ?: return baseHeaders + return baseHeaders.newBuilder().add("authorization", "bearer $accessToken").build() + } + override suspend fun getListPage(page: Int, filter: MangaListFilter?): List { val url = buildString { append("https://") @@ -95,7 +129,6 @@ internal class ReaperComics(context: MangaLoaderContext) : * Parse the list of manga from the given document * * @param docs the document to parse - * @param title the title to search for * @return the list of manga */ private fun parseMangaList(docs: Document): List { @@ -121,23 +154,38 @@ internal class ReaperComics(context: MangaLoaderContext) : override suspend fun getAvailableTags(): Set = emptySet() + private inline fun Response.parseJson(): T = use { + it.body!!.string().parseJson() + } + + private inline fun String.parseJson(): T = json.decodeFromString(this) + + companion object { + private val JSON_MEDIA_TYPE = "application/json; charset=utf-8".toMediaType() + } + + private fun chapterListNextPageSelector(): String = "button[wire:click*=nextPage]" + + private val json = Json { + ignoreUnknownKeys = true + } + + private fun chapterListSelector() = "div[wire:id] > div > ul[role=list] > li" + override suspend fun getDetails(manga: Manga): Manga { - val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val doc = Jsoup.parse(webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseRaw()) val simpleDateFormat = SimpleDateFormat("dd/MM/yyyy", sourceLocale) - return manga.copy( - description = doc.selectFirst("div.p-4 p.prose")?.html(), - state = when (doc.selectFirst("dl.mt-2 div:contains(Status) dd")?.text()?.lowercase()) { - "ongoing" -> MangaState.ONGOING - "complete" -> MangaState.FINISHED - else -> null - }, - chapters = doc.select("div.p-2 div.pb-4 ul li").mapChapters(reversed = true) { i, li -> + var totalChapters = (doc.selectFirst(selectTotalChapter)?.text()?.toIntOrNull() ?: 0) - 1 + val chapters = mutableSetOf() + var hasNextPage = doc.selectFirst(chapterListNextPageSelector()) != null + chapters.addAll( + doc.select(chapterListSelector()).mapChapters { _, li -> val a = li.selectFirstOrThrow("a") - val chapterUrl = a.attrAsAbsoluteUrl("href").toRelativeUrl(domain) + val chapterUrl = a.attr("href").toRelativeUrl(domain) MangaChapter( id = generateUid(chapterUrl), name = li.selectFirst("div.truncate p.truncate")?.text().orEmpty(), - number = i + 1, + number = totalChapters--, url = chapterUrl, scanlator = null, uploadDate = parseChapterDate( @@ -149,6 +197,135 @@ internal class ReaperComics(context: MangaLoaderContext) : ) }, ) + + if (!hasNextPage) { + return manga.copy( + description = doc.selectFirst("div.p-4 p.prose")?.html(), + state = when (doc.selectFirst(selectState)?.text()?.lowercase()) { + "ongoing" -> MangaState.ONGOING + "complete" -> MangaState.FINISHED + else -> null + }, + chapters = chapters.reversed(), + ) + } + + val csrfToken = doc.selectFirst("meta[name=csrf-token]")?.attr("content") ?: error("Couldn't find csrf-token") + val livewareData = doc.selectFirst("div[wire:initial-data*=Models\\\\Comic]")?.attr("wire:initial-data") + ?.parseJson() ?: error("Couldn't find LiveWireData") + + val routeName = + livewareData.fingerprint["name"]?.jsonPrimitive?.contentOrNull ?: error("Couldn't find routeName") + + val fingerprint = livewareData.fingerprint + var serverMemo = livewareData.serverMemo + + var pageToQuery = 2 + + // Javascript: (Math.random() + 1).toString(36).substring(8) + val generateId = { -> + "1.${ + Random.nextLong().toString(36) + }".substring(10) + } // Not exactly the same, but results in a 3-5 character string + + while (hasNextPage) { + val payload = buildJsonObject { + put("fingerprint", fingerprint) + put("serverMemo", serverMemo) + putJsonArray("updates") { + addJsonObject { + put("type", "callMethod") + putJsonObject("payload") { + put("id", generateId()) + put("method", "gotoPage") + putJsonArray("params") { + add(pageToQuery) + add("page") + } + } + } + } + }.toString().toRequestBody(JSON_MEDIA_TYPE) + + val headers = Headers.Builder().add("x-csrf-token", csrfToken).add("x-livewire", "true").build() + + val responseData = makeRequest("$baseUrl/livewire/message/$routeName", payload, headers) + + // response contains state that we need to preserve + serverMemo = serverMemo.mergeLeft(responseData.serverMemo) + val chaptersHtml = Jsoup.parse(responseData.effects.html, baseUrl) + chapters.addAll( + chaptersHtml.select(chapterListSelector()).mapChapters { _, li -> + val a = li.selectFirstOrThrow("a") + val chapterUrl = a.attr("href").toRelativeUrl(domain) + MangaChapter( + id = generateUid(chapterUrl), + name = li.selectFirst("div.truncate p.truncate")?.text().orEmpty(), + number = totalChapters--, + url = chapterUrl, + scanlator = null, + uploadDate = parseChapterDate( + simpleDateFormat, + li.selectFirst("div.truncate div.items-center")?.text(), + ), + branch = null, + source = source, + ) + }, + ) + hasNextPage = chaptersHtml.selectFirst(chapterListNextPageSelector()) != null + pageToQuery++ + } + + return manga.copy( + description = doc.selectFirst("div.p-4 p.prose")?.html(), + state = when (doc.selectFirst(selectState)?.text()?.lowercase()) { + "ongoing" -> MangaState.ONGOING + "complete" -> MangaState.FINISHED + else -> null + }, + chapters = chapters.reversed(), + ) + } + + private suspend fun makeRequest(url: String, payload: RequestBody, headers: Headers): LiveWireResponseDto { + var retryCount = 0 + var backoffDelay = 2000L // Initial delay (milliseconds) + val request = Request.Builder().url(url).post(payload).headers(headers).build() + while (true) { + try { + println(backoffDelay) + return context.httpClient.newCall(request).execute().parseJson() + } catch (e: Exception) { + // Log or handle the exception as needed + if (++retryCount <= MAX_RETRY_COUNT) { + withContext(Dispatchers.Default) { + delay(backoffDelay) + backoffDelay += 500L + } + } else { + throw e + } + } + } + } + + /** + * Recursively merges j2 onto j1 in place + * If j1 and j2 both contain keys whose values aren't both jsonObjects, j2's value overwrites j1's + * + */ + private fun JsonObject.mergeLeft(j2: JsonObject): JsonObject = buildJsonObject { + val j1 = this@mergeLeft + j1.entries.forEach { (key, value) -> put(key, value) } + j2.entries.forEach { (key, value) -> + val j1Value = j1[key] + when { + j1Value !is JsonObject -> put(key, value) + value is JsonObject -> put(key, j1Value.mergeLeft(value)) + } + } } private fun parseChapterDate(dateFormat: DateFormat, date: String?): Long { @@ -187,3 +364,20 @@ internal class ReaperComics(context: MangaLoaderContext) : } } } + +@Serializable +data class LiveWireResponseDto( + val effects: LiveWireEffectsDto, + val serverMemo: JsonObject, +) + +@Serializable +data class LiveWireEffectsDto( + val html: String, +) + +@Serializable +data class LiveWireDataDto( + val fingerprint: JsonObject, + val serverMemo: JsonObject, +) From f39a9f191ac7b393ea66ec6ddd42476b04bffeaa Mon Sep 17 00:00:00 2001 From: Naga <94557604+NagaYZ@users.noreply.github.com> Date: Sun, 10 Mar 2024 23:39:31 +0100 Subject: [PATCH 03/10] cleanup --- .../kotlin/org/koitharu/kotatsu/parsers/site/en/ReaperComics.kt | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/ReaperComics.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/ReaperComics.kt index e0fa9045..5c0f176e 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/ReaperComics.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/ReaperComics.kt @@ -295,7 +295,6 @@ internal class ReaperComics(context: MangaLoaderContext) : val request = Request.Builder().url(url).post(payload).headers(headers).build() while (true) { try { - println(backoffDelay) return context.httpClient.newCall(request).execute().parseJson() } catch (e: Exception) { // Log or handle the exception as needed From 1926a73dee96caf2fad719527ca70fa765bee12a Mon Sep 17 00:00:00 2001 From: Naga <94557604+NagaYZ@users.noreply.github.com> Date: Mon, 11 Mar 2024 20:01:07 +0100 Subject: [PATCH 04/10] without serialization, and added cache --- .idea/.gitignore | 2 + build.gradle | 2 - .../kotatsu/parsers/site/en/ReaperComics.kt | 114 +++++++++--------- 3 files changed, 58 insertions(+), 60 deletions(-) diff --git a/.idea/.gitignore b/.idea/.gitignore index 26d33521..8f00030d 100644 --- a/.idea/.gitignore +++ b/.idea/.gitignore @@ -1,3 +1,5 @@ # Default ignored files /shelf/ /workspace.xml +# GitHub Copilot persisted chat sessions +/copilot/chatSessions diff --git a/build.gradle b/build.gradle index 667c6b8d..c5412694 100644 --- a/build.gradle +++ b/build.gradle @@ -5,7 +5,6 @@ plugins { id 'org.jetbrains.kotlin.jvm' version '1.9.22' id 'com.google.devtools.ksp' version '1.9.22-1.0.17' id 'maven-publish' - id 'org.jetbrains.kotlin.plugin.serialization' version '1.9.22' } group = 'org.koitharu' @@ -60,7 +59,6 @@ dependencies { api 'org.jsoup:jsoup:1.17.2' implementation 'org.json:json:20231013' implementation 'androidx.collection:collection:1.4.0' - implementation "org.jetbrains.kotlinx:kotlinx-serialization-json:1.6.3" ksp project(':kotatsu-parsers-ksp') diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/ReaperComics.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/ReaperComics.kt index 5c0f176e..5fa4a412 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/ReaperComics.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/ReaperComics.kt @@ -3,10 +3,10 @@ package org.koitharu.kotatsu.parsers.site.en import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.delay import kotlinx.coroutines.withContext -import kotlinx.serialization.Serializable -import kotlinx.serialization.json.* -import okhttp3.* +import okhttp3.Headers import okhttp3.MediaType.Companion.toMediaType +import okhttp3.Request +import okhttp3.RequestBody import okhttp3.RequestBody.Companion.toRequestBody import org.json.JSONObject import org.jsoup.Jsoup @@ -25,7 +25,7 @@ import java.util.EnumSet import kotlin.random.Random private const val TOO_MANY_REQUESTS = 429 -private const val MAX_RETRY_COUNT = 3 +private const val MAX_RETRY_COUNT = 5 @MangaSourceParser("REAPERCOMICS", "ReaperComics", "en") internal class ReaperComics(context: MangaLoaderContext) : @@ -48,6 +48,7 @@ internal class ReaperComics(context: MangaLoaderContext) : private val selectState = "dl.mt-2 div:nth-child(4) > dd" private val searchCache = mutableSetOf() // Cache search results + private val chapterCache = mutableMapOf() // Cache chapter lists private val baseUrl = "https://reaperscans.com" @@ -154,26 +155,21 @@ internal class ReaperComics(context: MangaLoaderContext) : override suspend fun getAvailableTags(): Set = emptySet() - private inline fun Response.parseJson(): T = use { - it.body!!.string().parseJson() - } - - private inline fun String.parseJson(): T = json.decodeFromString(this) - companion object { private val JSON_MEDIA_TYPE = "application/json; charset=utf-8".toMediaType() } private fun chapterListNextPageSelector(): String = "button[wire:click*=nextPage]" - private val json = Json { - ignoreUnknownKeys = true - } - private fun chapterListSelector() = "div[wire:id] > div > ul[role=list] > li" override suspend fun getDetails(manga: Manga): Manga { - val doc = Jsoup.parse(webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseRaw()) + val cachedChapters = chapterCache[manga.url] + if (cachedChapters != null) { + return cachedChapters + } + + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() val simpleDateFormat = SimpleDateFormat("dd/MM/yyyy", sourceLocale) var totalChapters = (doc.selectFirst(selectTotalChapter)?.text()?.toIntOrNull() ?: 0) - 1 val chapters = mutableSetOf() @@ -212,13 +208,13 @@ internal class ReaperComics(context: MangaLoaderContext) : val csrfToken = doc.selectFirst("meta[name=csrf-token]")?.attr("content") ?: error("Couldn't find csrf-token") val livewareData = doc.selectFirst("div[wire:initial-data*=Models\\\\Comic]")?.attr("wire:initial-data") - ?.parseJson() ?: error("Couldn't find LiveWireData") + ?.let { JSONObject(it) } ?: error("Couldn't find LiveWireData") val routeName = - livewareData.fingerprint["name"]?.jsonPrimitive?.contentOrNull ?: error("Couldn't find routeName") + livewareData.getJSONObject("fingerprint").getStringOrNull("name") ?: error("Couldn't find routeName") - val fingerprint = livewareData.fingerprint - var serverMemo = livewareData.serverMemo + val fingerprint = livewareData.getJSONObject("fingerprint") + var serverMemo = livewareData.getJSONObject("serverMemo") var pageToQuery = 2 @@ -230,30 +226,27 @@ internal class ReaperComics(context: MangaLoaderContext) : } // Not exactly the same, but results in a 3-5 character string while (hasNextPage) { - val payload = buildJsonObject { - put("fingerprint", fingerprint) - put("serverMemo", serverMemo) - putJsonArray("updates") { - addJsonObject { - put("type", "callMethod") - putJsonObject("payload") { - put("id", generateId()) - put("method", "gotoPage") - putJsonArray("params") { - add(pageToQuery) - add("page") - } - } - } - } - }.toString().toRequestBody(JSON_MEDIA_TYPE) + //need to format the payload to the expected response format since org.json.JSONObject are not ordered, and the server seems to care about the order of the keys + val payload = String.format( + responseTemplate, + fingerprint.getString("id"), + fingerprint.getString("path"), + serverMemo.getString("htmlHash"), + pageToQuery - 1, + pageToQuery - 1, + serverMemo.getJSONObject("dataMeta").getJSONObject("models").getJSONObject("comic").getString("id"), + serverMemo.getString("checksum"), + generateId(), + pageToQuery, + ).toRequestBody(JSON_MEDIA_TYPE) val headers = Headers.Builder().add("x-csrf-token", csrfToken).add("x-livewire", "true").build() - val responseData = makeRequest("$baseUrl/livewire/message/$routeName", payload, headers) + val responseData = + makeRequest("$baseUrl/livewire/message/$routeName", payload, headers) // response contains state that we need to preserve - serverMemo = serverMemo.mergeLeft(responseData.serverMemo) + serverMemo = mergeLeft(serverMemo, responseData.serverMemo) val chaptersHtml = Jsoup.parse(responseData.effects.html, baseUrl) chapters.addAll( chaptersHtml.select(chapterListSelector()).mapChapters { _, li -> @@ -278,7 +271,7 @@ internal class ReaperComics(context: MangaLoaderContext) : pageToQuery++ } - return manga.copy( + val copy = manga.copy( description = doc.selectFirst("div.p-4 p.prose")?.html(), state = when (doc.selectFirst(selectState)?.text()?.lowercase()) { "ongoing" -> MangaState.ONGOING @@ -287,21 +280,30 @@ internal class ReaperComics(context: MangaLoaderContext) : }, chapters = chapters.reversed(), ) + + chapterCache[manga.url] = copy + return copy + } private suspend fun makeRequest(url: String, payload: RequestBody, headers: Headers): LiveWireResponseDto { var retryCount = 0 - var backoffDelay = 2000L // Initial delay (milliseconds) + val backoffDelay = 2000L // Initial delay (milliseconds) val request = Request.Builder().url(url).post(payload).headers(headers).build() + while (true) { try { - return context.httpClient.newCall(request).execute().parseJson() + val response = context.httpClient.newCall(request).execute().parseJson() + val effectsJson = response.getJSONObject("effects") + val serverMemoJson = response.getJSONObject("serverMemo") + val effects = LiveWireEffectsDto(effectsJson.getString("html")) + return LiveWireResponseDto(effects, serverMemoJson) + } catch (e: Exception) { // Log or handle the exception as needed if (++retryCount <= MAX_RETRY_COUNT) { withContext(Dispatchers.Default) { delay(backoffDelay) - backoffDelay += 500L } } else { throw e @@ -315,16 +317,17 @@ internal class ReaperComics(context: MangaLoaderContext) : * If j1 and j2 both contain keys whose values aren't both jsonObjects, j2's value overwrites j1's * */ - private fun JsonObject.mergeLeft(j2: JsonObject): JsonObject = buildJsonObject { - val j1 = this@mergeLeft - j1.entries.forEach { (key, value) -> put(key, value) } - j2.entries.forEach { (key, value) -> - val j1Value = j1[key] - when { - j1Value !is JsonObject -> put(key, value) - value is JsonObject -> put(key, j1Value.mergeLeft(value)) + private fun mergeLeft(j1: JSONObject, j2: JSONObject): JSONObject { + for (key in j2.keys()) { + val j1Value = j1.opt(key) + + if (j1Value !is JSONObject) { + j1.put(key, j2[key]) + } else if (j2[key] is JSONObject) { + j1.put(key, mergeLeft(j1Value, j2.getJSONObject(key))) } } + return j1 } private fun parseChapterDate(dateFormat: DateFormat, date: String?): Long { @@ -364,19 +367,14 @@ internal class ReaperComics(context: MangaLoaderContext) : } } -@Serializable data class LiveWireResponseDto( val effects: LiveWireEffectsDto, - val serverMemo: JsonObject, + val serverMemo: JSONObject, ) -@Serializable data class LiveWireEffectsDto( val html: String, ) -@Serializable -data class LiveWireDataDto( - val fingerprint: JsonObject, - val serverMemo: JsonObject, -) +//!IMPORTANT +private val responseTemplate = """{"fingerprint":{"id":"%s","name":"frontend.comic-chapter-list","locale":"en","path":"%s","method":"GET","v":"acj"},"serverMemo":{"children":[],"errors":[],"htmlHash":"%s","data":{"comic":[],"page":%d,"paginators":{"page":%d}},"dataMeta":{"models":{"comic":{"class":"App\\Models\\Comic","id":"%s","relations":[],"connection":"pgsql","collectionClass":null}}},"checksum":"%s"},"updates":[{"type":"callMethod","payload":{"id":"%s","method":"gotoPage","params":[%d,"page"]}}]}""" From f733b85878e67ad1f2a9a49ac30861155c5ce6be Mon Sep 17 00:00:00 2001 From: Naga <94557604+NagaYZ@users.noreply.github.com> Date: Tue, 12 Mar 2024 23:10:31 +0100 Subject: [PATCH 05/10] fix webtoon sort newest --- .../kotatsu/parsers/site/all/WebtoonsParser.kt | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt index eb51e1df..f75e9ce8 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt @@ -231,21 +231,19 @@ internal abstract class WebtoonsParser( val genre = filter.tags.oneOrThrowIfMany()?.key ?: "ALL" val genres = getAllGenreList() - val result = getAllTitleList() + var result = getAllTitleList() - val sortedResult = when (filter.sortOrder) { - SortOrder.UPDATED -> result.sortedBy { it.date } + if (genre != "ALL") { + result = result.filter { it.manga.tags.contains(genres[genre]) } + } + + when (filter.sortOrder) { + SortOrder.UPDATED -> result.sortedByDescending { it.date } SortOrder.POPULARITY -> result.sortedByDescending { it.readCount } SortOrder.RATING -> result.sortedByDescending { it.manga.rating } //SortOrder.LIKE -> result.sortedBy { it.likeitCount } else -> throw IllegalArgumentException("Unsupported sort order: ${filter.sortOrder}") } - - if (genre != "ALL") { - sortedResult.filter { it.manga.tags.contains(genres[genre]) } - } else { - sortedResult - } } else -> getAllTitleList() From b60b2d83559f0466e8927307d01a43bcefad2ef4 Mon Sep 17 00:00:00 2001 From: Naga <94557604+NagaYZ@users.noreply.github.com> Date: Tue, 12 Mar 2024 23:11:18 +0100 Subject: [PATCH 06/10] added sort by popularity/updated --- .../parsers/site/nepnep/NepnepParser.kt | 105 +++++++++--------- 1 file changed, 50 insertions(+), 55 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/nepnep/NepnepParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/nepnep/NepnepParser.kt index ccc6f270..5de7975c 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/nepnep/NepnepParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/nepnep/NepnepParser.kt @@ -23,7 +23,7 @@ internal abstract class NepnepParser( override val configKeyDomain = ConfigKey.Domain(domain) - override val availableSortOrders: Set = EnumSet.of(SortOrder.ALPHABETICAL) + override val availableSortOrders: Set = EnumSet.of(SortOrder.ALPHABETICAL, SortOrder.POPULARITY, SortOrder.UPDATED) override val availableStates: Set = EnumSet.allOf(MangaState::class.java) override val isTagsExclusionSupported = true @@ -31,15 +31,16 @@ internal abstract class NepnepParser( .add("User-Agent", UserAgents.CHROME_DESKTOP) .build() + data class MangaWithLastUpdate( + val manga: Manga, + val lastUpdate: Long, + val views: String + ) override suspend fun getList(offset: Int, filter: MangaListFilter?): List { if (offset > 0) { return emptyList() } - var foundTag = true - var foundTagExclude = true - var foundState = true - val doc = webClient.httpGet("https://$domain/search/").parseHtml() val json = JSONArray( doc.selectFirstOrThrow("script:containsData(MainFunction)").data() @@ -48,80 +49,74 @@ internal abstract class NepnepParser( .trim() .replace(';', ' '), ) - val manga = ArrayList(json.length()) - + val mangaWithLastUpdateList = ArrayList(json.length()) + var sort = false for (i in 0 until json.length()) { val m = json.getJSONObject(i) val href = "/manga/" + m.getString("i") val imgUrl = "https://temp.compsci88.com/cover/" + m.getString("i") + ".jpg" + val lastUpdate = m.getLong("lt") + val views = m.getString("v") + val viewMonth = m.getString("vm") + when (filter) { is MangaListFilter.Search -> { - if (m.getString("s").contains(filter.query, ignoreCase = true) || m.getString("al") - .contains(filter.query, ignoreCase = true) + if (m.getString("s").contains(filter.query, ignoreCase = true) || (m.getJSONArray("al").length() > 0 && m.getJSONArray("al").getString(0).contains(filter.query, ignoreCase = true)) ) { - manga.add( - addManga(href, imgUrl, m), + mangaWithLastUpdateList.add( + MangaWithLastUpdate(addManga(href, imgUrl, m), lastUpdate, views) ) } } is MangaListFilter.Advanced -> { - - if (filter.tags.isNotEmpty()) { - val tagsJon = m.getJSONArray("g").toString() - filter.tags.forEach { - foundTag = false - if (tagsJon.contains(it.key, ignoreCase = true)) { - foundTag = true - } - } - } - - if (filter.tagsExclude.isNotEmpty()) { - val tagsJon = m.getJSONArray("g").toString() - filter.tagsExclude.forEach { - foundTagExclude = false - if (!tagsJon.contains(it.key, ignoreCase = true)) { - foundTagExclude = true - } - } - } - - if (filter.states.isNotEmpty()) { - val stateJson = m.getString("ps") - filter.states.oneOrThrowIfMany().let { - foundState = false - if (stateJson.contains( - when (it) { - MangaState.ONGOING -> "Ongoing" - MangaState.FINISHED -> "Complete" - MangaState.ABANDONED -> "Cancelled" - MangaState.PAUSED -> "Hiatus" - else -> "" - }, - ignoreCase = true, - ) - ) { - foundState = true - } - } + val tags = filter.tags + val tagsExcluded = filter.tagsExclude + val tagsJson = m.getJSONArray("g").toString() + + val tagsMatched = tags.isEmpty() || tags.all { tag -> tagsJson.contains(tag.key, ignoreCase = true) } + val tagsExcludeMatched = tagsExcluded.isEmpty() || tagsExcluded.none { tag -> tagsJson.contains(tag.key, ignoreCase = true) } + val statesMatched = filter.states.isEmpty() || filter.states.any { state -> + m.getString("ps").contains( + when (state) { + MangaState.ONGOING -> "Ongoing" + MangaState.FINISHED -> "Complete" + MangaState.ABANDONED -> "Cancelled" + MangaState.PAUSED -> "Hiatus" + else -> "" + }, + ignoreCase = true, + ) } + if (tagsMatched && tagsExcludeMatched && statesMatched) { + mangaWithLastUpdateList.add( + MangaWithLastUpdate(addManga(href, imgUrl, m), lastUpdate, views) + ) - if (foundTag && foundState && foundTagExclude) { - manga.add(addManga(href, imgUrl, m)) } + sort = true } null -> { - manga.add( - addManga(href, imgUrl, m), + mangaWithLastUpdateList.add( + MangaWithLastUpdate(addManga(href, imgUrl, m), lastUpdate, views) ) } } } - return manga + if(sort){ + when(filter?.sortOrder){ + SortOrder.POPULARITY -> mangaWithLastUpdateList.sortByDescending { it.views } + SortOrder.UPDATED -> mangaWithLastUpdateList.sortByDescending { it.lastUpdate } + SortOrder.ALPHABETICAL -> {} + else -> if (filter != null) { + throw IllegalArgumentException("Unsupported sort order: ${filter.sortOrder}") + } + } + } + return mangaWithLastUpdateList.map { it.manga } } private fun addManga(href: String, imgUrl: String, m: JSONObject): Manga { From 016ced24e0a673a1564bf2bf655a34bb6647ad15 Mon Sep 17 00:00:00 2001 From: Naga <94557604+NagaYZ@users.noreply.github.com> Date: Wed, 13 Mar 2024 00:13:02 +0100 Subject: [PATCH 07/10] added pagination --- .idea/.gitignore | 2 ++ .../parsers/site/nepnep/NepnepParser.kt | 26 ++++++++++++++----- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/.idea/.gitignore b/.idea/.gitignore index 26d33521..8f00030d 100644 --- a/.idea/.gitignore +++ b/.idea/.gitignore @@ -1,3 +1,5 @@ # Default ignored files /shelf/ /workspace.xml +# GitHub Copilot persisted chat sessions +/copilot/chatSessions diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/nepnep/NepnepParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/nepnep/NepnepParser.kt index 5de7975c..610f3b0c 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/nepnep/NepnepParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/nepnep/NepnepParser.kt @@ -3,6 +3,7 @@ package org.koitharu.kotatsu.parsers.site.nepnep import okhttp3.Headers import org.json.JSONArray import org.json.JSONObject +import org.jsoup.nodes.Document import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaParser import org.koitharu.kotatsu.parsers.config.ConfigKey @@ -12,9 +13,12 @@ import org.koitharu.kotatsu.parsers.util.* import org.koitharu.kotatsu.parsers.util.json.getStringOrNull import org.koitharu.kotatsu.parsers.util.json.mapJSONIndexed import org.koitharu.kotatsu.parsers.util.json.toJSONList +import org.koitharu.kotatsu.parsers.util.SoftSuspendLazy +import org.koitharu.kotatsu.parsers.util.SuspendLazy import java.text.SimpleDateFormat import java.util.* + internal abstract class NepnepParser( context: MangaLoaderContext, source: MangaSource, @@ -31,19 +35,22 @@ internal abstract class NepnepParser( .add("User-Agent", UserAgents.CHROME_DESKTOP) .build() + private lateinit var docCache: Document + private var mangaListCache: List = emptyList() + data class MangaWithLastUpdate( val manga: Manga, val lastUpdate: Long, val views: String ) override suspend fun getList(offset: Int, filter: MangaListFilter?): List { - if (offset > 0) { - return emptyList() + val doc = if (::docCache.isInitialized) { + docCache + }else{ + webClient.httpGet("https://$domain/search/").parseHtml() } - val doc = webClient.httpGet("https://$domain/search/").parseHtml() - val json = JSONArray( - doc.selectFirstOrThrow("script:containsData(MainFunction)").data() + val json = JSONArray(doc.selectFirstOrThrow("script:containsData(MainFunction)").data() .substringAfter("vm.Directory = ") .substringBefore("vm.GetIntValue") .trim() @@ -116,7 +123,7 @@ internal abstract class NepnepParser( } } } - return mangaWithLastUpdateList.map { it.manga } + return mangaWithLastUpdateList.map { it.manga }.subList(offset, (offset + 30).coerceAtMost(mangaWithLastUpdateList.size)) } private fun addManga(href: String, imgUrl: String, m: JSONObject): Manga { @@ -136,8 +143,13 @@ internal abstract class NepnepParser( ) } + override suspend fun getAvailableTags(): Set { - val doc = webClient.httpGet("https://$domain/search/").parseHtml() + val doc = if (::docCache.isInitialized) { + docCache + }else{ + webClient.httpGet("https://$domain/search/").parseHtml() + } val tags = doc.selectFirstOrThrow("script:containsData(vm.AvailableFilters)").data() .substringAfter("\"Genre\"") .substringAfter('[') From 39ae6a406cc1258a867950b559e07d73dbd3e4f7 Mon Sep 17 00:00:00 2001 From: Koitharu Date: Wed, 13 Mar 2024 12:40:37 +0200 Subject: [PATCH 08/10] [Grouple] Added Seimanga support --- .../koitharu/kotatsu/parsers/model/Manga.kt | 3 ++- .../parsers/site/ru/grouple/GroupleParser.kt | 24 +++++++++++++++---- .../site/ru/grouple/MintMangaParser.kt | 15 ++++++++---- .../site/ru/grouple/ReadmangaParser.kt | 15 ++++++++---- .../parsers/site/ru/grouple/SeiMangaParser.kt | 21 ++++++++++++++++ .../site/ru/grouple/SelfMangaParser.kt | 8 ++++++- 6 files changed, 70 insertions(+), 16 deletions(-) create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/ru/grouple/SeiMangaParser.kt diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/model/Manga.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/model/Manga.kt index c8bf3b80..39c14b79 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/model/Manga.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/model/Manga.kt @@ -94,6 +94,7 @@ class Manga( largeCoverUrl: String? = this.largeCoverUrl, description: String? = this.description, chapters: List? = this.chapters, + source: MangaSource = this.source, ) = Manga( id = id, title = title, @@ -109,7 +110,7 @@ class Manga( largeCoverUrl = largeCoverUrl, description = description, chapters = chapters, - source = source + source = source, ) override fun equals(other: Any?): Boolean { diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ru/grouple/GroupleParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ru/grouple/GroupleParser.kt index 7fcdab8f..8435e190 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ru/grouple/GroupleParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ru/grouple/GroupleParser.kt @@ -4,6 +4,7 @@ import kotlinx.coroutines.flow.channelFlow import kotlinx.coroutines.flow.first import kotlinx.coroutines.launch import okhttp3.Headers +import okhttp3.HttpUrl import okhttp3.HttpUrl.Companion.toHttpUrl import okhttp3.HttpUrl.Companion.toHttpUrlOrNull import okhttp3.Interceptor @@ -112,7 +113,8 @@ internal abstract class GroupleParser( } override suspend fun getDetails(manga: Manga): Manga { - val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).checkAuthRequired().parseHtml() + val response = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).checkAuthRequired() + val doc = response.parseHtml() val root = doc.body().requireElementById("mangaBox").selectFirstOrThrow("div.leftContent") val dateFormat = SimpleDateFormat("dd.MM.yy", Locale.US) val coverImg = root.selectFirst("div.subject-cover")?.selectFirst("img") @@ -125,7 +127,9 @@ internal abstract class GroupleParser( } else { null } + val newSource = getSource(response.request.url) return manga.copy( + source = newSource, description = root.selectFirst("div.manga-description")?.html(), largeCoverUrl = coverImg?.attr("data-full"), coverUrl = coverImg?.attr("data-thumb") ?: manga.coverUrl, @@ -160,7 +164,7 @@ internal abstract class GroupleParser( url = href, uploadDate = dateFormat.tryParse(tr.selectFirst("td.date")?.text()), scanlator = translators, - source = source, + source = newSource, branch = null, ), ) @@ -177,7 +181,7 @@ internal abstract class GroupleParser( url = link, uploadDate = dateFormat.tryParse(jo.getStringOrNull("dateCreated")), scanlator = null, - source = source, + source = newSource, branch = translations[personId], ) } @@ -187,7 +191,11 @@ internal abstract class GroupleParser( } override suspend fun getPages(chapter: MangaChapter): List { - val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain) + "?mtr=1").checkAuthRequired().parseHtml() + if (chapter.source != source) { // handle redirects between websites + return context.newParserInstance(chapter.source).getPages(chapter) + } + val url = chapter.url.toAbsoluteUrl(domain).toHttpUrl().newBuilder().setQueryParameter("mtr", "1").build() + val doc = webClient.httpGet(url).checkAuthRequired().parseHtml() val scripts = doc.select("script") for (script in scripts) { val data = script.html() @@ -292,6 +300,14 @@ internal abstract class GroupleParser( return root.select("div.tile").mapNotNull(::parseManga) } + protected open fun getSource(url: HttpUrl): MangaSource = when (url.host) { + in SeiMangaParser.domains -> MangaSource.SEIMANGA + in MintMangaParser.domains -> MangaSource.MINTMANGA + in ReadmangaParser.domains -> MangaSource.READMANGA_RU + in SelfMangaParser.domains -> MangaSource.SELFMANGA + else -> source + } + private fun getSortKey(sortOrder: SortOrder) = when (sortOrder) { SortOrder.ALPHABETICAL -> "name" SortOrder.POPULARITY -> "rate" diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ru/grouple/MintMangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ru/grouple/MintMangaParser.kt index ed2a77d6..205225d1 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ru/grouple/MintMangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ru/grouple/MintMangaParser.kt @@ -10,9 +10,14 @@ internal class MintMangaParser( context: MangaLoaderContext, ) : GroupleParser(context, MangaSource.MINTMANGA, 2) { - override val configKeyDomain = ConfigKey.Domain( - "24.mintmanga.one", - "mintmanga.live", - "mintmanga.com", - ) + override val configKeyDomain = ConfigKey.Domain(*domains) + + companion object { + + val domains = arrayOf( + "24.mintmanga.one", + "mintmanga.live", + "mintmanga.com", + ) + } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ru/grouple/ReadmangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ru/grouple/ReadmangaParser.kt index 473f8b2a..423fec3f 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ru/grouple/ReadmangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ru/grouple/ReadmangaParser.kt @@ -10,9 +10,14 @@ internal class ReadmangaParser( context: MangaLoaderContext, ) : GroupleParser(context, MangaSource.READMANGA_RU, 1) { - override val configKeyDomain = ConfigKey.Domain( - "readmanga.live", - "readmanga.io", - "readmanga.me", - ) + override val configKeyDomain = ConfigKey.Domain(*domains) + + companion object { + + val domains = arrayOf( + "readmanga.live", + "readmanga.io", + "readmanga.me", + ) + } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ru/grouple/SeiMangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ru/grouple/SeiMangaParser.kt new file mode 100644 index 00000000..173c699c --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ru/grouple/SeiMangaParser.kt @@ -0,0 +1,21 @@ +package org.koitharu.kotatsu.parsers.site.ru.grouple + +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.config.ConfigKey +import org.koitharu.kotatsu.parsers.model.MangaSource + +@MangaSourceParser("SEIMANGA", "SeiManga", "ru") +internal class SeiMangaParser( + context: MangaLoaderContext, +) : GroupleParser(context, MangaSource.SEIMANGA, 21) { + + override val configKeyDomain = ConfigKey.Domain(*domains) + + companion object { + + val domains = arrayOf( + "seimanga.me", + ) + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ru/grouple/SelfMangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ru/grouple/SelfMangaParser.kt index a37884bd..6bd59bc5 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ru/grouple/SelfMangaParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ru/grouple/SelfMangaParser.kt @@ -11,6 +11,12 @@ internal class SelfMangaParser( context: MangaLoaderContext, ) : GroupleParser(context, MangaSource.SELFMANGA, 3) { - override val configKeyDomain = ConfigKey.Domain("selfmanga.live") + override val configKeyDomain = ConfigKey.Domain(*domains) + companion object { + + val domains = arrayOf( + "selfmanga.live", + ) + } } From 14fc02cb233129c64627ae2ca2e11291b0bb04a1 Mon Sep 17 00:00:00 2001 From: Koitharu Date: Wed, 13 Mar 2024 12:49:23 +0200 Subject: [PATCH 09/10] [ReaperComics] Refactor #620 --- .../kotatsu/parsers/site/en/ReaperComics.kt | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/ReaperComics.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/ReaperComics.kt index 5fa4a412..3b1c2d02 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/ReaperComics.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/ReaperComics.kt @@ -20,8 +20,7 @@ import org.koitharu.kotatsu.parsers.util.* import org.koitharu.kotatsu.parsers.util.json.getStringOrNull import java.text.DateFormat import java.text.SimpleDateFormat -import java.util.Calendar -import java.util.EnumSet +import java.util.* import kotlin.random.Random private const val TOO_MANY_REQUESTS = 429 @@ -35,8 +34,9 @@ internal class ReaperComics(context: MangaLoaderContext) : override val configKeyDomain = ConfigKey.Domain("reaperscans.com") - private val userAgentKey = - ConfigKey.UserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36") + private val userAgentKey = ConfigKey.UserAgent( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36", + ) private val baseHeaders: Headers get() = Headers.Builder().add("User-Agent", config[userAgentKey]).build() @@ -50,8 +50,6 @@ internal class ReaperComics(context: MangaLoaderContext) : private val searchCache = mutableSetOf() // Cache search results private val chapterCache = mutableMapOf() // Cache chapter lists - private val baseUrl = "https://reaperscans.com" - private fun getApiHeaders(): Headers { val userCookie = context.cookieJar.getCookies(domain).find { it.name == "user" @@ -243,11 +241,11 @@ internal class ReaperComics(context: MangaLoaderContext) : val headers = Headers.Builder().add("x-csrf-token", csrfToken).add("x-livewire", "true").build() val responseData = - makeRequest("$baseUrl/livewire/message/$routeName", payload, headers) + makeRequest("https://$domain/livewire/message/$routeName", payload, headers) // response contains state that we need to preserve serverMemo = mergeLeft(serverMemo, responseData.serverMemo) - val chaptersHtml = Jsoup.parse(responseData.effects.html, baseUrl) + val chaptersHtml = Jsoup.parse(responseData.effects.html, "https://$domain") chapters.addAll( chaptersHtml.select(chapterListSelector()).mapChapters { _, li -> val a = li.selectFirstOrThrow("a") @@ -365,16 +363,18 @@ internal class ReaperComics(context: MangaLoaderContext) : ) } } -} -data class LiveWireResponseDto( - val effects: LiveWireEffectsDto, - val serverMemo: JSONObject, -) + private class LiveWireResponseDto( + val effects: LiveWireEffectsDto, + val serverMemo: JSONObject, + ) -data class LiveWireEffectsDto( - val html: String, -) + private class LiveWireEffectsDto( + val html: String, + ) -//!IMPORTANT -private val responseTemplate = """{"fingerprint":{"id":"%s","name":"frontend.comic-chapter-list","locale":"en","path":"%s","method":"GET","v":"acj"},"serverMemo":{"children":[],"errors":[],"htmlHash":"%s","data":{"comic":[],"page":%d,"paginators":{"page":%d}},"dataMeta":{"models":{"comic":{"class":"App\\Models\\Comic","id":"%s","relations":[],"connection":"pgsql","collectionClass":null}}},"checksum":"%s"},"updates":[{"type":"callMethod","payload":{"id":"%s","method":"gotoPage","params":[%d,"page"]}}]}""" + //!IMPORTANT + private val responseTemplate = + """{"fingerprint":{"id":"%s","name":"frontend.comic-chapter-list","locale":"en","path":"%s","method":"GET","v":"acj"},"serverMemo":{"children":[],"errors":[],"htmlHash":"%s","data":{"comic":[],"page":%d,"paginators":{"page":%d}},"dataMeta":{"models":{"comic":{"class":"App\\Models\\Comic","id":"%s","relations":[],"connection":"pgsql","collectionClass":null}}},"checksum":"%s"},"updates":[{"type":"callMethod","payload":{"id":"%s","method":"gotoPage","params":[%d,"page"]}}]}""" + +} From c6d1f1b5258e38c42db83c01e2ff51d896ee4cda Mon Sep 17 00:00:00 2001 From: Koitharu Date: Wed, 13 Mar 2024 12:56:14 +0200 Subject: [PATCH 10/10] [NepNep] Refactor #628 --- .../parsers/site/nepnep/NepnepParser.kt | 57 ++++++++++--------- 1 file changed, 29 insertions(+), 28 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/nepnep/NepnepParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/nepnep/NepnepParser.kt index 610f3b0c..2b41dd23 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/nepnep/NepnepParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/nepnep/NepnepParser.kt @@ -3,7 +3,6 @@ package org.koitharu.kotatsu.parsers.site.nepnep import okhttp3.Headers import org.json.JSONArray import org.json.JSONObject -import org.jsoup.nodes.Document import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaParser import org.koitharu.kotatsu.parsers.config.ConfigKey @@ -13,8 +12,6 @@ import org.koitharu.kotatsu.parsers.util.* import org.koitharu.kotatsu.parsers.util.json.getStringOrNull import org.koitharu.kotatsu.parsers.util.json.mapJSONIndexed import org.koitharu.kotatsu.parsers.util.json.toJSONList -import org.koitharu.kotatsu.parsers.util.SoftSuspendLazy -import org.koitharu.kotatsu.parsers.util.SuspendLazy import java.text.SimpleDateFormat import java.util.* @@ -27,7 +24,8 @@ internal abstract class NepnepParser( override val configKeyDomain = ConfigKey.Domain(domain) - override val availableSortOrders: Set = EnumSet.of(SortOrder.ALPHABETICAL, SortOrder.POPULARITY, SortOrder.UPDATED) + override val availableSortOrders: Set = + EnumSet.of(SortOrder.ALPHABETICAL, SortOrder.POPULARITY, SortOrder.UPDATED) override val availableStates: Set = EnumSet.allOf(MangaState::class.java) override val isTagsExclusionSupported = true @@ -35,22 +33,20 @@ internal abstract class NepnepParser( .add("User-Agent", UserAgents.CHROME_DESKTOP) .build() - private lateinit var docCache: Document - private var mangaListCache: List = emptyList() + private val searchDoc = SoftSuspendLazy { + webClient.httpGet("https://$domain/search/").parseHtml() + } data class MangaWithLastUpdate( val manga: Manga, val lastUpdate: Long, - val views: String + val views: String, ) - override suspend fun getList(offset: Int, filter: MangaListFilter?): List { - val doc = if (::docCache.isInitialized) { - docCache - }else{ - webClient.httpGet("https://$domain/search/").parseHtml() - } - val json = JSONArray(doc.selectFirstOrThrow("script:containsData(MainFunction)").data() + override suspend fun getList(offset: Int, filter: MangaListFilter?): List { + val doc = searchDoc.get() + val json = JSONArray( + doc.selectFirstOrThrow("script:containsData(MainFunction)").data() .substringAfter("vm.Directory = ") .substringBefore("vm.GetIntValue") .trim() @@ -70,10 +66,12 @@ internal abstract class NepnepParser( when (filter) { is MangaListFilter.Search -> { - if (m.getString("s").contains(filter.query, ignoreCase = true) || (m.getJSONArray("al").length() > 0 && m.getJSONArray("al").getString(0).contains(filter.query, ignoreCase = true)) + if (m.getString("s").contains(filter.query, ignoreCase = true) || (m.getJSONArray("al") + .length() > 0 && m.getJSONArray("al").getString(0) + .contains(filter.query, ignoreCase = true)) ) { mangaWithLastUpdateList.add( - MangaWithLastUpdate(addManga(href, imgUrl, m), lastUpdate, views) + MangaWithLastUpdate(addManga(href, imgUrl, m), lastUpdate, views), ) } } @@ -83,8 +81,14 @@ internal abstract class NepnepParser( val tagsExcluded = filter.tagsExclude val tagsJson = m.getJSONArray("g").toString() - val tagsMatched = tags.isEmpty() || tags.all { tag -> tagsJson.contains(tag.key, ignoreCase = true) } - val tagsExcludeMatched = tagsExcluded.isEmpty() || tagsExcluded.none { tag -> tagsJson.contains(tag.key, ignoreCase = true) } + val tagsMatched = + tags.isEmpty() || tags.all { tag -> tagsJson.contains(tag.key, ignoreCase = true) } + val tagsExcludeMatched = tagsExcluded.isEmpty() || tagsExcluded.none { tag -> + tagsJson.contains( + tag.key, + ignoreCase = true, + ) + } val statesMatched = filter.states.isEmpty() || filter.states.any { state -> m.getString("ps").contains( when (state) { @@ -99,7 +103,7 @@ internal abstract class NepnepParser( } if (tagsMatched && tagsExcludeMatched && statesMatched) { mangaWithLastUpdateList.add( - MangaWithLastUpdate(addManga(href, imgUrl, m), lastUpdate, views) + MangaWithLastUpdate(addManga(href, imgUrl, m), lastUpdate, views), ) } @@ -108,13 +112,13 @@ internal abstract class NepnepParser( null -> { mangaWithLastUpdateList.add( - MangaWithLastUpdate(addManga(href, imgUrl, m), lastUpdate, views) + MangaWithLastUpdate(addManga(href, imgUrl, m), lastUpdate, views), ) } } } - if(sort){ - when(filter?.sortOrder){ + if (sort) { + when (filter?.sortOrder) { SortOrder.POPULARITY -> mangaWithLastUpdateList.sortByDescending { it.views } SortOrder.UPDATED -> mangaWithLastUpdateList.sortByDescending { it.lastUpdate } SortOrder.ALPHABETICAL -> {} @@ -123,7 +127,8 @@ internal abstract class NepnepParser( } } } - return mangaWithLastUpdateList.map { it.manga }.subList(offset, (offset + 30).coerceAtMost(mangaWithLastUpdateList.size)) + return mangaWithLastUpdateList.map { it.manga } + .subList(offset, (offset + 30).coerceAtMost(mangaWithLastUpdateList.size)) } private fun addManga(href: String, imgUrl: String, m: JSONObject): Manga { @@ -145,11 +150,7 @@ internal abstract class NepnepParser( override suspend fun getAvailableTags(): Set { - val doc = if (::docCache.isInitialized) { - docCache - }else{ - webClient.httpGet("https://$domain/search/").parseHtml() - } + val doc = searchDoc.get() val tags = doc.selectFirstOrThrow("script:containsData(vm.AvailableFilters)").data() .substringAfter("\"Genre\"") .substringAfter('[')