From 74a20455f25d4cdb4efccaf23ab445180eb86069 Mon Sep 17 00:00:00 2001 From: Koitharu Date: Mon, 24 Jul 2023 10:49:28 +0300 Subject: [PATCH] [MangaLib] Fix pages url --- .../RateLimitInterceptor.kt | 2 +- .../parsers/site/grouple/GroupleParser.kt | 5 +- .../parsers/site/rulib/MangaLibParser.kt | 66 ++++++++++--------- .../parsers/CommonHeadersInterceptor.kt | 2 +- 4 files changed, 41 insertions(+), 34 deletions(-) rename src/main/kotlin/org/koitharu/kotatsu/parsers/network/{ => interceptors}/RateLimitInterceptor.kt (97%) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/network/RateLimitInterceptor.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/network/interceptors/RateLimitInterceptor.kt similarity index 97% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/network/RateLimitInterceptor.kt rename to src/main/kotlin/org/koitharu/kotatsu/parsers/network/interceptors/RateLimitInterceptor.kt index dfc126fb..7165bad3 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/network/RateLimitInterceptor.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/network/interceptors/RateLimitInterceptor.kt @@ -1,4 +1,4 @@ -package org.koitharu.kotatsu.parsers.network +package org.koitharu.kotatsu.parsers.network.interceptors import okhttp3.Interceptor import okhttp3.Response diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt index 4627a8ac..5cb86405 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/grouple/GroupleParser.kt @@ -313,8 +313,9 @@ internal abstract class GroupleParser( } private suspend fun tryHead(url: String): Boolean = runCatchingCancellable { - val response = webClient.httpHead(url) - response.isSuccessful && response.headersContentLength() >= MIN_IMAGE_SIZE + webClient.httpHead(url).use { response -> + response.isSuccessful && response.headersContentLength() >= MIN_IMAGE_SIZE + } }.getOrDefault(false) private fun Response.checkAuthRequired(): Response { diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/rulib/MangaLibParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/rulib/MangaLibParser.kt index 5a7f795a..886f9a18 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/rulib/MangaLibParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/rulib/MangaLibParser.kt @@ -1,6 +1,7 @@ package org.koitharu.kotatsu.parsers.site.rulib import androidx.collection.ArraySet +import kotlinx.coroutines.withTimeoutOrNull import org.json.JSONArray import org.json.JSONObject import org.jsoup.nodes.Document @@ -12,32 +13,14 @@ import org.koitharu.kotatsu.parsers.config.ConfigKey import org.koitharu.kotatsu.parsers.exception.AuthRequiredException import org.koitharu.kotatsu.parsers.exception.NotFoundException import org.koitharu.kotatsu.parsers.exception.ParseException -import org.koitharu.kotatsu.parsers.model.Manga -import org.koitharu.kotatsu.parsers.model.MangaChapter -import org.koitharu.kotatsu.parsers.model.MangaPage -import org.koitharu.kotatsu.parsers.model.MangaSource -import org.koitharu.kotatsu.parsers.model.MangaTag -import org.koitharu.kotatsu.parsers.model.RATING_UNKNOWN -import org.koitharu.kotatsu.parsers.model.SortOrder -import org.koitharu.kotatsu.parsers.util.ChaptersListBuilder -import org.koitharu.kotatsu.parsers.util.attrAsRelativeUrl -import org.koitharu.kotatsu.parsers.util.domain -import org.koitharu.kotatsu.parsers.util.generateUid -import org.koitharu.kotatsu.parsers.util.getCookies -import org.koitharu.kotatsu.parsers.util.host +import org.koitharu.kotatsu.parsers.model.* +import org.koitharu.kotatsu.parsers.util.* import org.koitharu.kotatsu.parsers.util.json.JSONIterator import org.koitharu.kotatsu.parsers.util.json.getStringOrNull import org.koitharu.kotatsu.parsers.util.json.mapJSON -import org.koitharu.kotatsu.parsers.util.mapNotNullToSet -import org.koitharu.kotatsu.parsers.util.parseFailed -import org.koitharu.kotatsu.parsers.util.parseHtml -import org.koitharu.kotatsu.parsers.util.parseJsonArray -import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl -import org.koitharu.kotatsu.parsers.util.toTitleCase -import org.koitharu.kotatsu.parsers.util.tryParse +import org.koitharu.kotatsu.parsers.util.json.values import java.text.SimpleDateFormat -import java.util.EnumSet -import java.util.Locale +import java.util.* internal open class MangaLibParser( context: MangaLoaderContext, @@ -207,17 +190,19 @@ internal open class MangaLibParser( val json = JSONObject( raw.substringAfter("window.__info").substringAfter('=').substringBeforeLast(';'), ) - val domain = json.getJSONObject("servers").run { - getStringOrNull("main") ?: getString( - json.getJSONObject("img").getString("server"), - ) - } - val url = json.getJSONObject("img").getString("url") + val servers = json.getJSONObject("servers") + val img = json.getJSONObject("img") + val defaultServer = servers.getStringOrNull(img.getString("server")) + val baseUrl = img.getString("url") + val pageJson = JSONObject() + pageJson.put("default", defaultServer) + pageJson.put("servers", JSONArray(Iterable { servers.values() })) return pages.mapJSON { x -> - val pageUrl = "$domain/$url${x.getString("u")}" + val pageUrl = concatUrl(baseUrl, x.getString("u")) + pageJson.put("url", pageUrl) MangaPage( id = generateUid(pageUrl), - url = pageUrl, + url = pageJson.toString(), preview = null, source = source, ) @@ -227,6 +212,19 @@ internal open class MangaLibParser( throw ParseException("Script with info not found", fullUrl) } + override suspend fun getPageUrl(page: MangaPage): String { + val json = JSONObject(page.url) + val defaultServer = json.getString("default") + val servers = json.getJSONArray("servers") + val pageUrl = json.getString("url") + return (0 until servers.length()).firstNotNullOfOrNull { i -> + val server = servers.getString(i) + concatUrl(server, pageUrl).takeIf { + tryHeadImage(it) + } + } ?: concatUrl(defaultServer, pageUrl) + } + override suspend fun getTags(): Set { val url = "https://${domain}/manga-list" val doc = webClient.httpGet(url).parseHtml() @@ -265,6 +263,14 @@ internal open class MangaLibParser( return body.selectFirst(".profile-user__username")?.text() ?: body.parseFailed("Cannot find username") } + private suspend fun tryHeadImage(url: String): Boolean = runCatchingCancellable { + withTimeoutOrNull(3_000) { + webClient.httpHead(url).use { response -> + response.isSuccessful && response.mimeType?.startsWith("image/") == true + } + } ?: false + }.getOrDefault(false) + protected open fun isNsfw(doc: Document): Boolean { val modal = doc.body().getElementById("title-caution") if (!modal?.getElementsContainingOwnText("18+").isNullOrEmpty()) { diff --git a/src/test/kotlin/org/koitharu/kotatsu/parsers/CommonHeadersInterceptor.kt b/src/test/kotlin/org/koitharu/kotatsu/parsers/CommonHeadersInterceptor.kt index e6d870fa..17c296a1 100644 --- a/src/test/kotlin/org/koitharu/kotatsu/parsers/CommonHeadersInterceptor.kt +++ b/src/test/kotlin/org/koitharu/kotatsu/parsers/CommonHeadersInterceptor.kt @@ -27,7 +27,7 @@ internal class CommonHeadersInterceptor() : Interceptor { return if (parser is Interceptor) { parser.intercept(ProxyChain(chain, newRequest)) } else { - return chain.proceed(newRequest) + chain.proceed(newRequest) } }