diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/network/RateLimitInterceptor.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/network/RateLimitInterceptor.kt new file mode 100644 index 000000000..dfc126fbc --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/network/RateLimitInterceptor.kt @@ -0,0 +1,79 @@ +package org.koitharu.kotatsu.parsers.network + +import okhttp3.Interceptor +import okhttp3.Response +import okhttp3.internal.notifyAll +import okio.IOException +import java.util.concurrent.Semaphore +import java.util.concurrent.TimeUnit + +// TODO rewrite this +class RateLimitInterceptor : Interceptor { + + private val requestQueue = ArrayDeque(10) + private val rateLimitMillis = TimeUnit.SECONDS.toMillis(60L) + private val fairLock = Semaphore(1, true) + + override fun intercept(chain: Interceptor.Chain): Response { + val call = chain.call() + val request = chain.request() + + try { + fairLock.acquire() + } catch (e: InterruptedException) { + throw IOException(e) + } + + val requestQueue = this.requestQueue + val timestamp: Long + + try { + synchronized(requestQueue) { + while (requestQueue.size >= 10) { + val periodStart = System.currentTimeMillis() - rateLimitMillis + var hasRemovedExpired = false + while (requestQueue.isEmpty().not() && requestQueue.first() <= periodStart) { + requestQueue.removeFirst() + hasRemovedExpired = true + } + if (call.isCanceled()) { + throw IOException("Canceled") + } else if (hasRemovedExpired) { + break + } else { + try { + requestQueue.wait(requestQueue.first() - periodStart) + } catch (_: InterruptedException) { + continue + } + } + } + + timestamp = System.currentTimeMillis() + requestQueue.addLast(timestamp) + } + } finally { + fairLock.release() + } + + val response = chain.proceed(request) + if (response.networkResponse == null) { + synchronized(requestQueue) { + if (requestQueue.isEmpty() || timestamp < requestQueue.first()) return@synchronized + val iterator = requestQueue.iterator() + while (iterator.hasNext()) { + if (iterator.next() == timestamp) { + iterator.remove() + break + } + } + requestQueue.notifyAll() + } + } + + return response + } + + @Suppress("PLATFORM_CLASS_MAPPED_TO_KOTLIN", "NOTHING_TO_INLINE") + private inline fun Any.wait(timeout: Long) = (this as Object).wait(timeout) +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/TuMangaOnlineParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/TuMangaOnlineParser.kt index 4445841fe..d8fe1a9ad 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/TuMangaOnlineParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/TuMangaOnlineParser.kt @@ -1,143 +1,43 @@ package org.koitharu.kotatsu.parsers.site -import android.os.SystemClock -import okhttp3.FormBody import okhttp3.Headers -import okhttp3.Interceptor -import okhttp3.Request -import okhttp3.Response +import okhttp3.HttpUrl.Companion.toHttpUrl import org.jsoup.nodes.Document import org.jsoup.nodes.Element -import org.koitharu.kotatsu.core.util.ext.ensureSuccess import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaSourceParser import org.koitharu.kotatsu.parsers.PagedMangaParser import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.model.Manga -import org.koitharu.kotatsu.parsers.model.MangaChapter -import org.koitharu.kotatsu.parsers.model.MangaPage -import org.koitharu.kotatsu.parsers.model.MangaSource -import org.koitharu.kotatsu.parsers.model.MangaState -import org.koitharu.kotatsu.parsers.model.MangaTag -import org.koitharu.kotatsu.parsers.model.RATING_UNKNOWN -import org.koitharu.kotatsu.parsers.model.SortOrder -import org.koitharu.kotatsu.parsers.network.WebClient -import org.koitharu.kotatsu.parsers.util.attrAsAbsoluteUrlOrNull -import org.koitharu.kotatsu.parsers.util.attrAsRelativeUrl -import org.koitharu.kotatsu.parsers.util.attrAsRelativeUrlOrNull -import org.koitharu.kotatsu.parsers.util.await -import org.koitharu.kotatsu.parsers.util.domain -import org.koitharu.kotatsu.parsers.util.generateUid -import org.koitharu.kotatsu.parsers.util.host -import org.koitharu.kotatsu.parsers.util.mapChapters -import org.koitharu.kotatsu.parsers.util.mapNotNullToSet -import org.koitharu.kotatsu.parsers.util.parseHtml -import org.koitharu.kotatsu.parsers.util.selectFirstOrThrow -import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl -import org.koitharu.kotatsu.parsers.util.tryParse -import java.io.IOException +import org.koitharu.kotatsu.parsers.model.* +import org.koitharu.kotatsu.parsers.util.* import java.text.SimpleDateFormat -import java.util.EnumSet -import java.util.concurrent.Semaphore -import java.util.concurrent.TimeUnit +import java.util.* @MangaSourceParser("TUMANGAONLINE", "TuMangaOnline", "es") -class TuMangaOnlineParser(context: MangaLoaderContext) : PagedMangaParser ( +class TuMangaOnlineParser(context: MangaLoaderContext) : PagedMangaParser( context, source = MangaSource.TUMANGAONLINE, pageSize = 24, -), Interceptor { +) { - override val configKeyDomain: ConfigKey.Domain - get() = ConfigKey.Domain("lectortmo.com") + override val configKeyDomain = ConfigKey.Domain("lectortmo.com") private val chapterDateFormat = SimpleDateFormat("yyyy-MM-dd", sourceLocale) - override val sortOrders: Set - get() = EnumSet.of( - SortOrder.NEWEST, - SortOrder.POPULARITY, - ) - - private val requestQueue = ArrayDeque(permits) - private val rateLimitMillis = unit.toMillis(period) - private val fairLock = Semaphore(1, true) - - override fun intercept(chain: Interceptor.Chain): Response { - val call = chain.call() - if (call.isCanceled()) throw IOException("Canceled") - - val request = chain.request() - when (domain) { - request.url.host -> {} - else -> return chain.proceed(request) - } - - try { - fairLock.acquire() - } catch (e: InterruptedException) { - throw IOException(e) - } - - val requestQueue = this.requestQueue - val timestamp: Long - - try { - synchronized(requestQueue) { - while (requestQueue.size >= permits) { - val periodStart = SystemClock.elapsedRealtime() - rateLimitMillis - var hasRemovedExpired = false - while (requestQueue.isEmpty().not() && requestQueue.first() <= periodStart) { - requestQueue.removeFirst() - hasRemovedExpired = true - } - if (call.isCanceled()) { - throw IOException("Canceled") - } else if (hasRemovedExpired) { - break - } else { - try { - (requestQueue as Object).wait(requestQueue.first() - periodStart) - } catch (_: InterruptedException) { - continue - } - } - } - - timestamp = SystemClock.elapsedRealtime() - requestQueue.addLast(timestamp) - } - } finally { - fairLock.release() - } - - val response = chain.proceed(request) - if (response.networkResponse == null) { - synchronized(requestQueue) { - if (requestQueue.isEmpty() || timestamp < requestQueue.first()) return@synchronized - val iterator = requestQueue.iterator() - while (iterator.hasNext()) { - if (iterator.next() == timestamp) { - iterator.remove() - break - } - } - (requestQueue as Object).notifyAll() - } - } - - return response - } + override val sortOrders = EnumSet.of( + SortOrder.NEWEST, + SortOrder.POPULARITY, + ) override suspend fun getListPage( page: Int, query: String?, tags: Set?, - sortOrder: SortOrder + sortOrder: SortOrder, ): List { val url = buildString { append("/library") - if(query.isNullOrEmpty()){ + if (query.isNullOrEmpty()) { append("?order_item=") if (sortOrder == SortOrder.POPULARITY) { append("likes_count") @@ -148,7 +48,7 @@ class TuMangaOnlineParser(context: MangaLoaderContext) : PagedMangaParser ( append("&order_dir=desc") append("&filter_by=title") if (tags != null) { - for(tag in tags){ + for (tag in tags) { append("&genders[]=${tag.key}") } } @@ -162,17 +62,15 @@ class TuMangaOnlineParser(context: MangaLoaderContext) : PagedMangaParser ( val doc = webClient.httpGet(url, headers).parseHtml() val items = doc.body().select("div.element") return items.mapNotNull { item -> - val href = item.selectFirst("a")?.attrAsRelativeUrlOrNull("href")?.substringAfter(' ') ?: return@mapNotNull null + val href = + item.selectFirst("a")?.attrAsRelativeUrlOrNull("href")?.substringAfter(' ') ?: return@mapNotNull null Manga( id = generateUid(href), title = item.selectFirst("h4.text-truncate")?.text() ?: return@mapNotNull null, coverUrl = item.select("style").toString().substringAfter("('").substringBeforeLast("')"), altTitle = null, author = null, - rating = item.selectFirst("span.score") - ?.text() - ?.toFloatOrNull() - ?.div(10F) ?: RATING_UNKNOWN, + rating = item.selectFirst("span.score")?.text()?.toFloatOrNull()?.div(10F) ?: RATING_UNKNOWN, url = href, isNsfw = item.select("i").hasClass("fas fa-heartbeat fa-2x"), tags = emptySet(), @@ -192,7 +90,7 @@ class TuMangaOnlineParser(context: MangaLoaderContext) : PagedMangaParser ( largeCoverUrl = contents.selectFirst(".book-thumbnail")?.attrAsAbsoluteUrlOrNull("src"), state = parseStatus(contents.select("span.book-status").text().orEmpty()), author = contents.selectFirst("h5.card-title")?.attr("title")?.substringAfter(", "), - chapters = if(doc.select("div.chapters").isEmpty()){ + chapters = if (doc.select("div.chapters").isEmpty()) { doc.select(oneShotChapterListSelector()).mapChapters(reversed = true) { i, item -> oneShotChapterFromElement(item) } @@ -204,14 +102,14 @@ class TuMangaOnlineParser(context: MangaLoaderContext) : PagedMangaParser ( scanelement.forEach { chapters.add(regularChapterFromElement(it, chaptername, i)) } } chapters - } + }, ) } private fun oneShotChapterListSelector() = "div.chapter-list-element > ul.list-group li.list-group-item" + private fun oneShotChapterFromElement(element: Element): MangaChapter { - val href = element.selectFirstOrThrow("div.row > .text-right > a") - .attrAsRelativeUrl("href") + val href = element.selectFirstOrThrow("div.row > .text-right > a").attrAsRelativeUrl("href") return MangaChapter( id = generateUid(href), name = "One Shot", @@ -225,9 +123,9 @@ class TuMangaOnlineParser(context: MangaLoaderContext) : PagedMangaParser ( } private fun regularChapterListSelector() = "div.chapters > ul.list-group li.p-0.list-group-item" + private fun regularChapterFromElement(element: Element, chName: String, number: Int): MangaChapter { - val href = element.selectFirstOrThrow("div.row > .text-right > a") - .attrAsRelativeUrl("href") + val href = element.selectFirstOrThrow("div.row > .text-right > a").attrAsRelativeUrl("href") return MangaChapter( id = generateUid(href), name = chName, @@ -255,7 +153,7 @@ class TuMangaOnlineParser(context: MangaLoaderContext) : PagedMangaParser ( doc = webClient.httpGet(newUrl, headers).parseHtml() } - return doc.select("div.viewer-container img:not(noscript img)").map{ + return doc.select("div.viewer-container img:not(noscript img)").map { val href = if (it.hasAttr("data-src")) { it.attr("abs:data-src") } else { @@ -274,23 +172,20 @@ class TuMangaOnlineParser(context: MangaLoaderContext) : PagedMangaParser ( val script1 = document.selectFirst("script:containsData(uniqid)") val script2 = document.selectFirst("script:containsData(window.location.replace)") - val redirectHeaders = Headers.Builder() - .set("Referer", document.baseUri()) - .build() + val redirectHeaders = Headers.Builder().set("Referer", document.baseUri()).build() if (script1 != null) { val data = script1.data() - val regexParams = """\{uniqid:'(.+)',cascade:(.+)\}""".toRegex() + val regexParams = """\{uniqid:'(.+)',cascade:(.+)}""".toRegex() val regexAction = """form\.action\s?=\s?'(.+)'""".toRegex() val params = regexParams.find(data)!! - val action = regexAction.find(data)!!.groupValues[1] - - val formBody = FormBody.Builder() - .add("uniqid", params.groupValues[1]) - .add("cascade", params.groupValues[2]) - .build() + val action = regexAction.find(data)!!.groupValues[1].toHttpUrl() - return redirectToReadingPage(webClient.httpPost(action,redirectHeaders,formBody).parseHtml()) + val formBody = mapOf( + "uniqid" to params.groupValues[1], + "cascade" to params.groupValues[2], + ) + return redirectToReadingPage(webClient.httpPost(action, formBody, redirectHeaders).parseHtml()) } if (script2 != null) { @@ -311,7 +206,7 @@ class TuMangaOnlineParser(context: MangaLoaderContext) : PagedMangaParser ( MangaTag( title = element.select("label").text(), key = element.select("input").attr("value"), - source = source + source = source, ) } } @@ -321,19 +216,4 @@ class TuMangaOnlineParser(context: MangaLoaderContext) : PagedMangaParser ( status.contains("Finalizado") -> MangaState.FINISHED else -> null } - - private suspend fun WebClient.httpPost(url: String, headers: Headers, body: FormBody): Response { - val client = context.httpClient - val request = Request.Builder() - .post(body) - .headers(headers) - .url(url) - return client.newCall(request.build()).await().ensureSuccess() - } - - companion object { - private const val permits = 10 - private const val period = 60L - private val unit = TimeUnit.SECONDS - } } diff --git a/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaSources.kt b/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaSources.kt index ef660faa0..095f16826 100644 --- a/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaSources.kt +++ b/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaSources.kt @@ -3,5 +3,5 @@ package org.koitharu.kotatsu.parsers import org.junit.jupiter.params.provider.EnumSource import org.koitharu.kotatsu.parsers.model.MangaSource -@EnumSource(MangaSource::class, names = ["LOCAL", "DUMMY"], mode = EnumSource.Mode.EXCLUDE) +@EnumSource(MangaSource::class, names = ["TUMANGAONLINE"], mode = EnumSource.Mode.INCLUDE) internal annotation class MangaSources