KotatsuApp
/
kotatsu-parsers
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[TuMangaOnline] Fixes

Browse Source
pull/206/head
Koitharu 3 years ago
parent a8c086b340
commit 3874fe31d5
No known key found for this signature in database
GPG Key ID: 8E861F8CE6E7CE27
3 changed files with 113 additions and 154 deletions
Show Stats Download Patch File Download Diff File

79
src/main/kotlin/org/koitharu/kotatsu/parsers/network/RateLimitInterceptor.kt
Unescape Escape View File

@ -0,0 +1,79 @@
package org.koitharu.kotatsu.parsers.network
import okhttp3.Interceptor
import okhttp3.Response
import okhttp3.internal.notifyAll
import okio.IOException
import java.util.concurrent.Semaphore
import java.util.concurrent.TimeUnit
// TODO rewrite this
class RateLimitInterceptor : Interceptor {
private val requestQueue = ArrayDeque<Long>(10)
private val rateLimitMillis = TimeUnit.SECONDS.toMillis(60L)
private val fairLock = Semaphore(1, true)
override fun intercept(chain: Interceptor.Chain): Response {
val call = chain.call()
val request = chain.request()
try {
fairLock.acquire()
} catch (e: InterruptedException) {
throw IOException(e)
}
val requestQueue = this.requestQueue
val timestamp: Long
try {
synchronized(requestQueue) {
while (requestQueue.size >= 10) {
val periodStart = System.currentTimeMillis() - rateLimitMillis
var hasRemovedExpired = false
while (requestQueue.isEmpty().not() && requestQueue.first() <= periodStart) {
requestQueue.removeFirst()
hasRemovedExpired = true
}
if (call.isCanceled()) {
throw IOException("Canceled")
} else if (hasRemovedExpired) {
break
} else {
try {
requestQueue.wait(requestQueue.first() - periodStart)
} catch (_: InterruptedException) {
continue
}
}
}
timestamp = System.currentTimeMillis()
requestQueue.addLast(timestamp)
}
} finally {
fairLock.release()
}
val response = chain.proceed(request)
if (response.networkResponse == null) {
synchronized(requestQueue) {
if (requestQueue.isEmpty() || timestamp < requestQueue.first()) return@synchronized
val iterator = requestQueue.iterator()
while (iterator.hasNext()) {
if (iterator.next() == timestamp) {
iterator.remove()
break
}
}
requestQueue.notifyAll()
}
}
return response
}
@Suppress("PLATFORM_CLASS_MAPPED_TO_KOTLIN", "NOTHING_TO_INLINE")
private inline fun Any.wait(timeout: Long) = (this as Object).wait(timeout)
}

186
src/main/kotlin/org/koitharu/kotatsu/parsers/site/TuMangaOnlineParser.kt
Unescape Escape View File

@ -1,143 +1,43 @@
package org.koitharu.kotatsu.parsers.site
import android.os.SystemClock
import okhttp3.FormBody
import okhttp3.Headers
import okhttp3.Interceptor
import okhttp3.Request
import okhttp3.Response
import okhttp3.HttpUrl.Companion.toHttpUrl
import org.jsoup.nodes.Document
import org.jsoup.nodes.Element
import org.koitharu.kotatsu.core.util.ext.ensureSuccess
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.PagedMangaParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.model.Manga
import org.koitharu.kotatsu.parsers.model.MangaChapter
import org.koitharu.kotatsu.parsers.model.MangaPage
import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.model.MangaState
import org.koitharu.kotatsu.parsers.model.MangaTag
import org.koitharu.kotatsu.parsers.model.RATING_UNKNOWN
import org.koitharu.kotatsu.parsers.model.SortOrder
import org.koitharu.kotatsu.parsers.network.WebClient
import org.koitharu.kotatsu.parsers.util.attrAsAbsoluteUrlOrNull
import org.koitharu.kotatsu.parsers.util.attrAsRelativeUrl
import org.koitharu.kotatsu.parsers.util.attrAsRelativeUrlOrNull
import org.koitharu.kotatsu.parsers.util.await
import org.koitharu.kotatsu.parsers.util.domain
import org.koitharu.kotatsu.parsers.util.generateUid
import org.koitharu.kotatsu.parsers.util.host
import org.koitharu.kotatsu.parsers.util.mapChapters
import org.koitharu.kotatsu.parsers.util.mapNotNullToSet
import org.koitharu.kotatsu.parsers.util.parseHtml
import org.koitharu.kotatsu.parsers.util.selectFirstOrThrow
import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl
import org.koitharu.kotatsu.parsers.util.tryParse
import java.io.IOException
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import java.text.SimpleDateFormat
import java.util.EnumSet
import java.util.concurrent.Semaphore
import java.util.concurrent.TimeUnit
import java.util.*
@MangaSourceParser("TUMANGAONLINE", "TuMangaOnline", "es")
class TuMangaOnlineParser(context: MangaLoaderContext) : PagedMangaParser (
class TuMangaOnlineParser(context: MangaLoaderContext) : PagedMangaParser(
context,
source = MangaSource.TUMANGAONLINE,
pageSize = 24,
), Interceptor {
) {
override val configKeyDomain: ConfigKey.Domain
get() = ConfigKey.Domain("lectortmo.com")
override val configKeyDomain = ConfigKey.Domain("lectortmo.com")
private val chapterDateFormat = SimpleDateFormat("yyyy-MM-dd", sourceLocale)
override val sortOrders: Set<SortOrder>
get() = EnumSet.of(
SortOrder.NEWEST,
SortOrder.POPULARITY,
)
private val requestQueue = ArrayDeque<Long>(permits)
private val rateLimitMillis = unit.toMillis(period)
private val fairLock = Semaphore(1, true)
override fun intercept(chain: Interceptor.Chain): Response {
val call = chain.call()
if (call.isCanceled()) throw IOException("Canceled")
val request = chain.request()
when (domain) {
request.url.host -> {}
else -> return chain.proceed(request)
}
try {
fairLock.acquire()
} catch (e: InterruptedException) {
throw IOException(e)
}
val requestQueue = this.requestQueue
val timestamp: Long
try {
synchronized(requestQueue) {
while (requestQueue.size >= permits) {
val periodStart = SystemClock.elapsedRealtime() - rateLimitMillis
var hasRemovedExpired = false
while (requestQueue.isEmpty().not() && requestQueue.first() <= periodStart) {
requestQueue.removeFirst()
hasRemovedExpired = true
}
if (call.isCanceled()) {
throw IOException("Canceled")
} else if (hasRemovedExpired) {
break
} else {
try {
(requestQueue as Object).wait(requestQueue.first() - periodStart)
} catch (_: InterruptedException) {
continue
}
}
}
timestamp = SystemClock.elapsedRealtime()
requestQueue.addLast(timestamp)
}
} finally {
fairLock.release()
}
val response = chain.proceed(request)
if (response.networkResponse == null) {
synchronized(requestQueue) {
if (requestQueue.isEmpty() || timestamp < requestQueue.first()) return@synchronized
val iterator = requestQueue.iterator()
while (iterator.hasNext()) {
if (iterator.next() == timestamp) {
iterator.remove()
break
}
}
(requestQueue as Object).notifyAll()
}
}
return response
}
override val sortOrders = EnumSet.of(
SortOrder.NEWEST,
SortOrder.POPULARITY,
)
override suspend fun getListPage(
page: Int,
query: String?,
tags: Set<MangaTag>?,
sortOrder: SortOrder
sortOrder: SortOrder,
): List<Manga> {
val url = buildString {
append("/library")
if(query.isNullOrEmpty()){
if (query.isNullOrEmpty()) {
append("?order_item=")
if (sortOrder == SortOrder.POPULARITY) {
append("likes_count")
@ -148,7 +48,7 @@ class TuMangaOnlineParser(context: MangaLoaderContext) : PagedMangaParser (
append("&order_dir=desc")
append("&filter_by=title")
if (tags != null) {
for(tag in tags){
for (tag in tags) {
append("&genders[]=${tag.key}")
}
}
@ -162,17 +62,15 @@ class TuMangaOnlineParser(context: MangaLoaderContext) : PagedMangaParser (
val doc = webClient.httpGet(url, headers).parseHtml()
val items = doc.body().select("div.element")
return items.mapNotNull { item ->
val href = item.selectFirst("a")?.attrAsRelativeUrlOrNull("href")?.substringAfter(' ') ?: return@mapNotNull null
val href =
item.selectFirst("a")?.attrAsRelativeUrlOrNull("href")?.substringAfter(' ') ?: return@mapNotNull null
Manga(
id = generateUid(href),
title = item.selectFirst("h4.text-truncate")?.text() ?: return@mapNotNull null,
coverUrl = item.select("style").toString().substringAfter("('").substringBeforeLast("')"),
altTitle = null,
author = null,
rating = item.selectFirst("span.score")
?.text()
?.toFloatOrNull()
?.div(10F) ?: RATING_UNKNOWN,
rating = item.selectFirst("span.score")?.text()?.toFloatOrNull()?.div(10F) ?: RATING_UNKNOWN,
url = href,
isNsfw = item.select("i").hasClass("fas fa-heartbeat fa-2x"),
tags = emptySet(),
@ -192,7 +90,7 @@ class TuMangaOnlineParser(context: MangaLoaderContext) : PagedMangaParser (
largeCoverUrl = contents.selectFirst(".book-thumbnail")?.attrAsAbsoluteUrlOrNull("src"),
state = parseStatus(contents.select("span.book-status").text().orEmpty()),
author = contents.selectFirst("h5.card-title")?.attr("title")?.substringAfter(", "),
chapters = if(doc.select("div.chapters").isEmpty()){
chapters = if (doc.select("div.chapters").isEmpty()) {
doc.select(oneShotChapterListSelector()).mapChapters(reversed = true) { i, item ->
oneShotChapterFromElement(item)
}
@ -204,14 +102,14 @@ class TuMangaOnlineParser(context: MangaLoaderContext) : PagedMangaParser (
scanelement.forEach { chapters.add(regularChapterFromElement(it, chaptername, i)) }
}
chapters
}
},
)
}
private fun oneShotChapterListSelector() = "div.chapter-list-element > ul.list-group li.list-group-item"
private fun oneShotChapterFromElement(element: Element): MangaChapter {
val href = element.selectFirstOrThrow("div.row > .text-right > a")
.attrAsRelativeUrl("href")
val href = element.selectFirstOrThrow("div.row > .text-right > a").attrAsRelativeUrl("href")
return MangaChapter(
id = generateUid(href),
name = "One Shot",
@ -225,9 +123,9 @@ class TuMangaOnlineParser(context: MangaLoaderContext) : PagedMangaParser (
}
private fun regularChapterListSelector() = "div.chapters > ul.list-group li.p-0.list-group-item"
private fun regularChapterFromElement(element: Element, chName: String, number: Int): MangaChapter {
val href = element.selectFirstOrThrow("div.row > .text-right > a")
.attrAsRelativeUrl("href")
val href = element.selectFirstOrThrow("div.row > .text-right > a").attrAsRelativeUrl("href")
return MangaChapter(
id = generateUid(href),
name = chName,
@ -255,7 +153,7 @@ class TuMangaOnlineParser(context: MangaLoaderContext) : PagedMangaParser (
doc = webClient.httpGet(newUrl, headers).parseHtml()
}
return doc.select("div.viewer-container img:not(noscript img)").map{
return doc.select("div.viewer-container img:not(noscript img)").map {
val href = if (it.hasAttr("data-src")) {
it.attr("abs:data-src")
} else {
@ -274,23 +172,20 @@ class TuMangaOnlineParser(context: MangaLoaderContext) : PagedMangaParser (
val script1 = document.selectFirst("script:containsData(uniqid)")
val script2 = document.selectFirst("script:containsData(window.location.replace)")
val redirectHeaders = Headers.Builder()
.set("Referer", document.baseUri())
.build()
val redirectHeaders = Headers.Builder().set("Referer", document.baseUri()).build()
if (script1 != null) {
val data = script1.data()
val regexParams = """\{uniqid:'(.+)',cascade:(.+)\}""".toRegex()
val regexParams = """\{uniqid:'(.+)',cascade:(.+)}""".toRegex()
val regexAction = """form\.action\s?=\s?'(.+)'""".toRegex()
val params = regexParams.find(data)!!
val action = regexAction.find(data)!!.groupValues[1]
val formBody = FormBody.Builder()
.add("uniqid", params.groupValues[1])
.add("cascade", params.groupValues[2])
.build()
val action = regexAction.find(data)!!.groupValues[1].toHttpUrl()
return redirectToReadingPage(webClient.httpPost(action,redirectHeaders,formBody).parseHtml())
val formBody = mapOf(
"uniqid" to params.groupValues[1],
"cascade" to params.groupValues[2],
)
return redirectToReadingPage(webClient.httpPost(action, formBody, redirectHeaders).parseHtml())
}
if (script2 != null) {
@ -311,7 +206,7 @@ class TuMangaOnlineParser(context: MangaLoaderContext) : PagedMangaParser (
MangaTag(
title = element.select("label").text(),
key = element.select("input").attr("value"),
source = source
source = source,
)
}
}
@ -321,19 +216,4 @@ class TuMangaOnlineParser(context: MangaLoaderContext) : PagedMangaParser (
status.contains("Finalizado") -> MangaState.FINISHED
else -> null
}
private suspend fun WebClient.httpPost(url: String, headers: Headers, body: FormBody): Response {
val client = context.httpClient
val request = Request.Builder()
.post(body)
.headers(headers)
.url(url)
return client.newCall(request.build()).await().ensureSuccess()
}
companion object {
private const val permits = 10
private const val period = 60L
private val unit = TimeUnit.SECONDS
}
}

2
src/test/kotlin/org/koitharu/kotatsu/parsers/MangaSources.kt
Unescape Escape View File

@ -3,5 +3,5 @@ package org.koitharu.kotatsu.parsers
import org.junit.jupiter.params.provider.EnumSource
import org.koitharu.kotatsu.parsers.model.MangaSource
@EnumSource(MangaSource::class, names = ["LOCAL", "DUMMY"], mode = EnumSource.Mode.EXCLUDE)
@EnumSource(MangaSource::class, names = ["TUMANGAONLINE"], mode = EnumSource.Mode.INCLUDE)
internal annotation class MangaSources

Write Preview
Loading…
Cancel
Save