Fix http request processing

master
Koitharu 1 year ago
parent adf5794ad3
commit 0847baf17b
Signed by: Koitharu
GPG Key ID: 676DEE768C17A9D7

@ -304,7 +304,7 @@ internal class ExHentaiParser(
override fun intercept(chain: Interceptor.Chain): Response { override fun intercept(chain: Interceptor.Chain): Response {
val response = chain.proceed(chain.request()) val response = chain.proceed(chain.request())
if (response.headersContentLength() <= 256) { if (response.headersContentLength() <= 256) {
val text = response.peekBody(256).string() val text = response.peekBody(256).use { it.string() }
if (text.contains("IP address has been temporarily banned", ignoreCase = true)) { if (text.contains("IP address has been temporarily banned", ignoreCase = true)) {
val hours = Regex("([0-9]+) hours?").find(text)?.groupValues?.getOrNull(1)?.toLongOrNull() ?: 0 val hours = Regex("([0-9]+) hours?").find(text)?.groupValues?.getOrNull(1)?.toLongOrNull() ?: 0
val minutes = Regex("([0-9]+) minutes?").find(text)?.groupValues?.getOrNull(1)?.toLongOrNull() ?: 0 val minutes = Regex("([0-9]+) minutes?").find(text)?.groupValues?.getOrNull(1)?.toLongOrNull() ?: 0

@ -456,7 +456,7 @@ internal abstract class MangaFireParser(
} }
} }
return@redrawImageResponse result result
} }
} }

@ -2,9 +2,8 @@ package org.koitharu.kotatsu.parsers.site.fr
import kotlinx.coroutines.* import kotlinx.coroutines.*
import okhttp3.Headers import okhttp3.Headers
import okhttp3.Request import okhttp3.HttpUrl
import okhttp3.RequestBody import okhttp3.HttpUrl.Companion.toHttpUrl
import okhttp3.RequestBody.Companion.toRequestBody
import org.json.JSONArray import org.json.JSONArray
import org.jsoup.Jsoup import org.jsoup.Jsoup
import org.jsoup.nodes.Document import org.jsoup.nodes.Document
@ -166,11 +165,11 @@ internal class MangaMana(context: MangaLoaderContext) :
} }
} }
val url = "https://$domain/liste-mangas" val url = "https://$domain/liste-mangas".toHttpUrl()
val token = webClient.httpGet(url).parseHtml().selectFirstOrThrow("meta[name=csrf-token]").attr("content") val token = webClient.httpGet(url).parseHtml().selectFirstOrThrow("meta[name=csrf-token]").attr("content")
val headers = Headers.Builder().add("X-CSRF-TOKEN", token).add("X-Requested-With", "XMLHttpRequest") val headers = Headers.Builder().add("X-CSRF-TOKEN", token).add("X-Requested-With", "XMLHttpRequest")
.add("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8").build() .add("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8").build()
val doc = makeRequest(url, postData.toRequestBody(), headers) val doc = makeRequest(url, postData, headers)
return doc.select("div.p-2 div.col").map { div -> return doc.select("div.p-2 div.col").map { div ->
val href = div.selectFirstOrThrow("a").attrAsRelativeUrl("href") val href = div.selectFirstOrThrow("a").attrAsRelativeUrl("href")
@ -206,13 +205,12 @@ internal class MangaMana(context: MangaLoaderContext) :
} }
private suspend fun makeRequest(url: String, payload: RequestBody, headers: Headers): Document { private suspend fun makeRequest(url: HttpUrl, payload: String, headers: Headers): Document {
var retryCount = 0 var retryCount = 0
val backoffDelay = 2000L // Initial delay (milliseconds) val backoffDelay = 2000L // Initial delay (milliseconds)
val request = Request.Builder().url(url).post(payload).headers(headers).build()
while (true) { while (true) {
try { try {
return Jsoup.parse(context.httpClient.newCall(request).execute().parseJson().getString("html")) return Jsoup.parse(webClient.httpPost(url, payload, headers).parseJson().getString("html"))
} catch (e: Exception) { } catch (e: Exception) {
// Log or handle the exception as needed // Log or handle the exception as needed

@ -2,9 +2,8 @@ package org.koitharu.kotatsu.parsers.site.madara.en
import kotlinx.coroutines.* import kotlinx.coroutines.*
import okhttp3.Headers import okhttp3.Headers
import okhttp3.Request import okhttp3.HttpUrl
import okhttp3.RequestBody import okhttp3.HttpUrl.Companion.toHttpUrl
import okhttp3.RequestBody.Companion.toRequestBody
import org.json.JSONObject import org.json.JSONObject
import org.jsoup.nodes.Document import org.jsoup.nodes.Document
import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaLoaderContext
@ -126,10 +125,10 @@ internal class AdultWebtoon(context: MangaLoaderContext) :
override suspend fun loadChapters(mangaUrl: String, document: Document): List<MangaChapter> { override suspend fun loadChapters(mangaUrl: String, document: Document): List<MangaChapter> {
val mangaId = document.select("div#manga-chapters-holder").attr("data-id") val mangaId = document.select("div#manga-chapters-holder").attr("data-id")
val url = "https://$domain/wp-admin/admin-ajax.php" val url = "https://$domain/wp-admin/admin-ajax.php".toHttpUrl()
val postData = "post_id=$mangaId&action=ajax_chap" val postData = "post_id=$mangaId&action=ajax_chap"
val headers = Headers.Builder().add("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8").build() val headers = Headers.Builder().add("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8").build()
val doc = makeRequest(url, postData.toRequestBody(), headers) val doc = makeRequest(url, postData, headers)
val dateFormat = SimpleDateFormat(datePattern, sourceLocale) val dateFormat = SimpleDateFormat(datePattern, sourceLocale)
return doc.select(selectChapter).mapChapters(reversed = true) { i, li -> return doc.select(selectChapter).mapChapters(reversed = true) { i, li ->
val a = li.selectFirst("a") val a = li.selectFirst("a")
@ -216,14 +215,12 @@ internal class AdultWebtoon(context: MangaLoaderContext) :
return chunked(2).map { it.toInt(16).toByte() }.toByteArray() return chunked(2).map { it.toInt(16).toByte() }.toByteArray()
} }
private suspend fun makeRequest(url: String, payload: RequestBody, headers: Headers): Document { private suspend fun makeRequest(url: HttpUrl, payload: String, headers: Headers): Document {
var retryCount = 0 var retryCount = 0
val backoffDelay = 2000L // Initial delay (milliseconds) val backoffDelay = 2000L // Initial delay (milliseconds)
val request = Request.Builder().url(url).post(payload).headers(headers).build()
while (true) { while (true) {
try { try {
return context.httpClient.newCall(request).execute().parseHtml() return webClient.httpPost(url, payload, headers).parseHtml()
} catch (e: Exception) { } catch (e: Exception) {
// Log or handle the exception as needed // Log or handle the exception as needed
if (++retryCount <= 5) { if (++retryCount <= 5) {

@ -2,9 +2,8 @@ package org.koitharu.kotatsu.parsers.site.madara.en
import kotlinx.coroutines.* import kotlinx.coroutines.*
import okhttp3.Headers import okhttp3.Headers
import okhttp3.Request import okhttp3.HttpUrl
import okhttp3.RequestBody import okhttp3.HttpUrl.Companion.toHttpUrl
import okhttp3.RequestBody.Companion.toRequestBody
import org.jsoup.nodes.Document import org.jsoup.nodes.Document
import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser import org.koitharu.kotatsu.parsers.MangaSourceParser
@ -120,10 +119,10 @@ internal class HentaiManga(context: MangaLoaderContext) :
override suspend fun loadChapters(mangaUrl: String, document: Document): List<MangaChapter> { override suspend fun loadChapters(mangaUrl: String, document: Document): List<MangaChapter> {
val mangaId = document.select("div#manga-chapters-holder").attr("data-id") val mangaId = document.select("div#manga-chapters-holder").attr("data-id")
val url = "https://$domain/wp-admin/admin-ajax.php" val url = "https://$domain/wp-admin/admin-ajax.php".toHttpUrl()
val postData = "post_id=$mangaId&action=ajax_chap" val postData = "post_id=$mangaId&action=ajax_chap"
val headers = Headers.Builder().add("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8").build() val headers = Headers.Builder().add("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8").build()
val doc = makeRequest(url, postData.toRequestBody(), headers) val doc = makeRequest(url, postData, headers)
val dateFormat = SimpleDateFormat(datePattern, sourceLocale) val dateFormat = SimpleDateFormat(datePattern, sourceLocale)
return doc.select(selectChapter).mapChapters(reversed = true) { i, li -> return doc.select(selectChapter).mapChapters(reversed = true) { i, li ->
val a = li.selectFirst("a") val a = li.selectFirst("a")
@ -148,14 +147,12 @@ internal class HentaiManga(context: MangaLoaderContext) :
} }
} }
private suspend fun makeRequest(url: String, payload: RequestBody, headers: Headers): Document { private suspend fun makeRequest(url: HttpUrl, payload: String, headers: Headers): Document {
var retryCount = 0 var retryCount = 0
val backoffDelay = 2000L // Initial delay (milliseconds) val backoffDelay = 2000L // Initial delay (milliseconds)
val request = Request.Builder().url(url).post(payload).headers(headers).build()
while (true) { while (true) {
try { try {
return context.httpClient.newCall(request).execute().parseHtml() return webClient.httpPost(url, payload, headers).parseHtml()
} catch (e: Exception) { } catch (e: Exception) {
// Log or handle the exception as needed // Log or handle the exception as needed
if (++retryCount <= 5) { if (++retryCount <= 5) {

@ -2,9 +2,8 @@ package org.koitharu.kotatsu.parsers.site.madara.en
import kotlinx.coroutines.* import kotlinx.coroutines.*
import okhttp3.Headers import okhttp3.Headers
import okhttp3.Request import okhttp3.HttpUrl
import okhttp3.RequestBody import okhttp3.HttpUrl.Companion.toHttpUrl
import okhttp3.RequestBody.Companion.toRequestBody
import org.jsoup.nodes.Document import org.jsoup.nodes.Document
import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser import org.koitharu.kotatsu.parsers.MangaSourceParser
@ -120,10 +119,10 @@ internal class HentaiWebtoon(context: MangaLoaderContext) :
override suspend fun loadChapters(mangaUrl: String, document: Document): List<MangaChapter> { override suspend fun loadChapters(mangaUrl: String, document: Document): List<MangaChapter> {
val mangaId = document.select("div#manga-chapters-holder").attr("data-id") val mangaId = document.select("div#manga-chapters-holder").attr("data-id")
val url = "https://$domain/wp-admin/admin-ajax.php" val url = "https://$domain/wp-admin/admin-ajax.php".toHttpUrl()
val postData = "post_id=$mangaId&action=ajax_chap" val postData = "post_id=$mangaId&action=ajax_chap"
val headers = Headers.Builder().add("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8").build() val headers = Headers.Builder().add("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8").build()
val doc = makeRequest(url, postData.toRequestBody(), headers) val doc = makeRequest(url, postData, headers)
val dateFormat = SimpleDateFormat(datePattern, sourceLocale) val dateFormat = SimpleDateFormat(datePattern, sourceLocale)
return doc.select(selectChapter).mapChapters(reversed = true) { i, li -> return doc.select(selectChapter).mapChapters(reversed = true) { i, li ->
val a = li.selectFirst("a") val a = li.selectFirst("a")
@ -148,14 +147,12 @@ internal class HentaiWebtoon(context: MangaLoaderContext) :
} }
} }
private suspend fun makeRequest(url: String, payload: RequestBody, headers: Headers): Document { private suspend fun makeRequest(url: HttpUrl, payload: String, headers: Headers): Document {
var retryCount = 0 var retryCount = 0
val backoffDelay = 2000L // Initial delay (milliseconds) val backoffDelay = 2000L // Initial delay (milliseconds)
val request = Request.Builder().url(url).post(payload).headers(headers).build()
while (true) { while (true) {
try { try {
return context.httpClient.newCall(request).execute().parseHtml() return webClient.httpPost(url, payload, headers).parseHtml()
} catch (e: Exception) { } catch (e: Exception) {
// Log or handle the exception as needed // Log or handle the exception as needed
if (++retryCount <= 5) { if (++retryCount <= 5) {

@ -2,9 +2,8 @@ package org.koitharu.kotatsu.parsers.site.madara.en
import kotlinx.coroutines.* import kotlinx.coroutines.*
import okhttp3.Headers import okhttp3.Headers
import okhttp3.Request import okhttp3.HttpUrl
import okhttp3.RequestBody import okhttp3.HttpUrl.Companion.toHttpUrl
import okhttp3.RequestBody.Companion.toRequestBody
import org.jsoup.nodes.Document import org.jsoup.nodes.Document
import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser import org.koitharu.kotatsu.parsers.MangaSourceParser
@ -123,10 +122,10 @@ internal class ManyToon(context: MangaLoaderContext) :
override suspend fun loadChapters(mangaUrl: String, document: Document): List<MangaChapter> { override suspend fun loadChapters(mangaUrl: String, document: Document): List<MangaChapter> {
val mangaId = document.select("div#manga-chapters-holder").attr("data-id") val mangaId = document.select("div#manga-chapters-holder").attr("data-id")
val url = "https://$domain/wp-admin/admin-ajax.php" val url = "https://$domain/wp-admin/admin-ajax.php".toHttpUrl()
val postData = "post_id=$mangaId&action=ajax_chap" val postData = "post_id=$mangaId&action=ajax_chap"
val headers = Headers.Builder().add("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8").build() val headers = Headers.Builder().add("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8").build()
val doc = makeRequest(url, postData.toRequestBody(), headers) val doc = makeRequest(url, postData, headers)
val dateFormat = SimpleDateFormat(datePattern, sourceLocale) val dateFormat = SimpleDateFormat(datePattern, sourceLocale)
return doc.select(selectChapter).mapChapters(reversed = true) { i, li -> return doc.select(selectChapter).mapChapters(reversed = true) { i, li ->
val a = li.selectFirst("a") val a = li.selectFirst("a")
@ -151,14 +150,12 @@ internal class ManyToon(context: MangaLoaderContext) :
} }
} }
private suspend fun makeRequest(url: String, payload: RequestBody, headers: Headers): Document { private suspend fun makeRequest(url: HttpUrl, payload: String, headers: Headers): Document {
var retryCount = 0 var retryCount = 0
val backoffDelay = 2000L // Initial delay (milliseconds) val backoffDelay = 2000L // Initial delay (milliseconds)
val request = Request.Builder().url(url).post(payload).headers(headers).build()
while (true) { while (true) {
try { try {
return context.httpClient.newCall(request).execute().parseHtml() webClient.httpPost(url, payload, headers).parseHtml()
} catch (e: Exception) { } catch (e: Exception) {
// Log or handle the exception as needed // Log or handle the exception as needed
if (++retryCount <= 5) { if (++retryCount <= 5) {

@ -209,6 +209,5 @@ internal abstract class MangAdventureParser(
private val Manga.slug: String private val Manga.slug: String
get() = url.substring(8, url.length - 1) get() = url.substring(8, url.length - 1)
protected suspend fun HttpUrl.Builder.get() = protected suspend fun HttpUrl.Builder.get() = webClient.httpGet(build()).parseJson()
webClient.httpGet(build()).body?.string()?.let(::JSONObject)
} }

@ -2,8 +2,6 @@ package org.koitharu.kotatsu.parsers.site.mangareader.en
import androidx.collection.ArrayMap import androidx.collection.ArrayMap
import kotlinx.coroutines.sync.withLock import kotlinx.coroutines.sync.withLock
import okhttp3.FormBody
import okhttp3.Request
import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.model.* import org.koitharu.kotatsu.parsers.model.*
@ -43,13 +41,8 @@ internal class RizzComic(context: MangaLoaderContext) :
private val slugRegex = Regex("""[^a-z0-9]+""") private val slugRegex = Regex("""[^a-z0-9]+""")
private val searchMangaSelector = ".utao .uta .imgu, .listupd .bs .bsx, .listo .bs .bsx" private val searchMangaSelector = ".utao .uta .imgu, .listupd .bs .bsx, .listo .bs .bsx"
private suspend fun getRandomPart(): String { private suspend fun getRandomPart(): String {
val request = Request.Builder() val response = webClient.httpGet("https://$domain$listUrl").parseHtml()
.url("https://$domain$listUrl") val url = response
.get()
.build()
val response = context.httpClient.newCall(request).await()
val url = response.parseHtml()
.selectFirst(searchMangaSelector)!! .selectFirst(searchMangaSelector)!!
.select("a").attr("href") .select("a").attr("href")
@ -74,9 +67,7 @@ internal class RizzComic(context: MangaLoaderContext) :
!filter.query.isNullOrEmpty() -> { !filter.query.isNullOrEmpty() -> {
url = "https://$domain$searchUrl" url = "https://$domain$searchUrl"
if (filter.query != "") { if (filter.query != "") {
FormBody.Builder() mapOf("search_value" to filter.query.trim())
.add("search_value", filter.query.trim())
.build()
} else { } else {
null null
} }
@ -87,28 +78,22 @@ internal class RizzComic(context: MangaLoaderContext) :
val genres = filter.tags.map { it.key } val genres = filter.tags.map { it.key }
val formBuilder = FormBody.Builder() val form = ArrayMap<String, String>()
.add("StatusValue", state) form["StatusValue"] = state
.add("TypeValue", "all") form["TypeValue"] = "all"
.add("OrderValue", order.toPayloadValue()) form["OrderValue"] = order.toPayloadValue()
genres.forEach { genre -> genres.forEach { genre ->
formBuilder.add("genres_checked[]", genre) form["genres_checked[]"] = genre
} }
formBuilder.build() form
} }
} }
val request = Request.Builder() val response = if (payload != null) {
.url(url) webClient.httpPost(url, payload)
.apply { } else {
if (payload != null) { webClient.httpGet(url)
post(payload) }.parseJsonArray()
} else {
get()
}
}
.build()
val response = context.httpClient.newCall(request).execute().parseJsonArray()
return response.mapJSON { j -> return response.mapJSON { j ->
val title = j.getString("title") val title = j.getString("title")
val urlManga = "https://$domain$listUrl/${randomPartCache.get()}-" + title.trim().lowercase() val urlManga = "https://$domain$listUrl/${randomPartCache.get()}-" + title.trim().lowercase()

@ -1,5 +1,6 @@
package org.koitharu.kotatsu.parsers.site.vi package org.koitharu.kotatsu.parsers.site.vi
import okhttp3.internal.closeQuietly
import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.config.ConfigKey import org.koitharu.kotatsu.parsers.config.ConfigKey
@ -140,7 +141,7 @@ internal class DuaLeoTruyen(context: MangaLoaderContext) :
"truyen" to comicsId, "truyen" to comicsId,
"chap" to chapterId, "chap" to chapterId,
), ),
) ).closeQuietly()
} }
return doc.select(".content_view_chap img").mapIndexed { i, img -> return doc.select(".content_view_chap img").mapIndexed { i, img ->

Loading…
Cancel
Save