[TuMangaOnline] Fixes

src/main/kotlin/org/koitharu/kotatsu/parsers/network/RateLimitInterceptor.kt

@@ -0,0 +1,79 @@
+package org.koitharu.kotatsu.parsers.network
+
+import okhttp3.Interceptor
+import okhttp3.Response
+import okhttp3.internal.notifyAll
+import okio.IOException
+import java.util.concurrent.Semaphore
+import java.util.concurrent.TimeUnit
+
+// TODO rewrite this
+class RateLimitInterceptor : Interceptor {
+
+	private val requestQueue = ArrayDeque<Long>(10)
+	private val rateLimitMillis = TimeUnit.SECONDS.toMillis(60L)
+	private val fairLock = Semaphore(1, true)
+
+	override fun intercept(chain: Interceptor.Chain): Response {
+		val call = chain.call()
+		val request = chain.request()
+
+		try {
+			fairLock.acquire()
+		} catch (e: InterruptedException) {
+			throw IOException(e)
+		}
+
+		val requestQueue = this.requestQueue
+		val timestamp: Long
+
+		try {
+			synchronized(requestQueue) {
+				while (requestQueue.size >= 10) {
+					val periodStart = System.currentTimeMillis() - rateLimitMillis
+					var hasRemovedExpired = false
+					while (requestQueue.isEmpty().not() && requestQueue.first() <= periodStart) {
+						requestQueue.removeFirst()
+						hasRemovedExpired = true
+					}
+					if (call.isCanceled()) {
+						throw IOException("Canceled")
+					} else if (hasRemovedExpired) {
+						break
+					} else {
+						try {
+							requestQueue.wait(requestQueue.first() - periodStart)
+						} catch (_: InterruptedException) {
+							continue
+						}
+					}
+				}
+
+				timestamp = System.currentTimeMillis()
+				requestQueue.addLast(timestamp)
+			}
+		} finally {
+			fairLock.release()
+		}
+
+		val response = chain.proceed(request)
+		if (response.networkResponse == null) {
+			synchronized(requestQueue) {
+				if (requestQueue.isEmpty() || timestamp < requestQueue.first()) return@synchronized
+				val iterator = requestQueue.iterator()
+				while (iterator.hasNext()) {
+					if (iterator.next() == timestamp) {
+						iterator.remove()
+						break
+					}
+				}
+				requestQueue.notifyAll()
+			}
+		}
+
+		return response
+	}
+
+	@Suppress("PLATFORM_CLASS_MAPPED_TO_KOTLIN", "NOTHING_TO_INLINE")
+	private inline fun Any.wait(timeout: Long) = (this as Object).wait(timeout)
+}

src/main/kotlin/org/koitharu/kotatsu/parsers/site/TuMangaOnlineParser.kt

@@ -1,143 +1,43 @@
 package org.koitharu.kotatsu.parsers.site
 
-import android.os.SystemClock
-import okhttp3.FormBody
 import okhttp3.Headers
-import okhttp3.Interceptor
+import okhttp3.HttpUrl.Companion.toHttpUrl
-import okhttp3.Request
-import okhttp3.Response
 import org.jsoup.nodes.Document
 import org.jsoup.nodes.Element
-import org.koitharu.kotatsu.core.util.ext.ensureSuccess
 import org.koitharu.kotatsu.parsers.MangaLoaderContext
 import org.koitharu.kotatsu.parsers.MangaSourceParser
 import org.koitharu.kotatsu.parsers.PagedMangaParser
 import org.koitharu.kotatsu.parsers.config.ConfigKey
-import org.koitharu.kotatsu.parsers.model.Manga
+import org.koitharu.kotatsu.parsers.model.*
-import org.koitharu.kotatsu.parsers.model.MangaChapter
+import org.koitharu.kotatsu.parsers.util.*
-import org.koitharu.kotatsu.parsers.model.MangaPage
-import org.koitharu.kotatsu.parsers.model.MangaSource
-import org.koitharu.kotatsu.parsers.model.MangaState
-import org.koitharu.kotatsu.parsers.model.MangaTag
-import org.koitharu.kotatsu.parsers.model.RATING_UNKNOWN
-import org.koitharu.kotatsu.parsers.model.SortOrder
-import org.koitharu.kotatsu.parsers.network.WebClient
-import org.koitharu.kotatsu.parsers.util.attrAsAbsoluteUrlOrNull
-import org.koitharu.kotatsu.parsers.util.attrAsRelativeUrl
-import org.koitharu.kotatsu.parsers.util.attrAsRelativeUrlOrNull
-import org.koitharu.kotatsu.parsers.util.await
-import org.koitharu.kotatsu.parsers.util.domain
-import org.koitharu.kotatsu.parsers.util.generateUid
-import org.koitharu.kotatsu.parsers.util.host
-import org.koitharu.kotatsu.parsers.util.mapChapters
-import org.koitharu.kotatsu.parsers.util.mapNotNullToSet
-import org.koitharu.kotatsu.parsers.util.parseHtml
-import org.koitharu.kotatsu.parsers.util.selectFirstOrThrow
-import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl
-import org.koitharu.kotatsu.parsers.util.tryParse
-import java.io.IOException
 import java.text.SimpleDateFormat
-import java.util.EnumSet
+import java.util.*
-import java.util.concurrent.Semaphore
-import java.util.concurrent.TimeUnit
 
 @MangaSourceParser("TUMANGAONLINE", "TuMangaOnline", "es")
-class TuMangaOnlineParser(context: MangaLoaderContext) : PagedMangaParser (
+class TuMangaOnlineParser(context: MangaLoaderContext) : PagedMangaParser(
 	context,
 	source = MangaSource.TUMANGAONLINE,
 	pageSize = 24,
-), Interceptor {
+) {
 
-	override val configKeyDomain: ConfigKey.Domain
+	override val configKeyDomain = ConfigKey.Domain("lectortmo.com")
-		get() = ConfigKey.Domain("lectortmo.com")
 
 	private val chapterDateFormat = SimpleDateFormat("yyyy-MM-dd", sourceLocale)
 
-	override val sortOrders: Set<SortOrder>
+	override val sortOrders = EnumSet.of(
-		get() = EnumSet.of(
+		SortOrder.NEWEST,
-			SortOrder.NEWEST,
+		SortOrder.POPULARITY,
-			SortOrder.POPULARITY,
+	)
-		)
-
-	private val requestQueue = ArrayDeque<Long>(permits)
-	private val rateLimitMillis = unit.toMillis(period)
-	private val fairLock = Semaphore(1, true)
-
-	override fun intercept(chain: Interceptor.Chain): Response {
-		val call = chain.call()
-		if (call.isCanceled()) throw IOException("Canceled")
-
-		val request = chain.request()
-		when (domain) {
-			request.url.host -> {}
-			else -> return chain.proceed(request)
-		}
-
-		try {
-			fairLock.acquire()
-		} catch (e: InterruptedException) {
-			throw IOException(e)
-		}
-
-		val requestQueue = this.requestQueue
-		val timestamp: Long
-
-		try {
-			synchronized(requestQueue) {
-				while (requestQueue.size >= permits) {
-					val periodStart = SystemClock.elapsedRealtime() - rateLimitMillis
-					var hasRemovedExpired = false
-					while (requestQueue.isEmpty().not() && requestQueue.first() <= periodStart) {
-						requestQueue.removeFirst()
-						hasRemovedExpired = true
-					}
-					if (call.isCanceled()) {
-						throw IOException("Canceled")
-					} else if (hasRemovedExpired) {
-						break
-					} else {
-						try {
-							(requestQueue as Object).wait(requestQueue.first() - periodStart)
-						} catch (_: InterruptedException) {
-							continue
-						}
-					}
-				}
-
-				timestamp = SystemClock.elapsedRealtime()
-				requestQueue.addLast(timestamp)
-			}
-		} finally {
-			fairLock.release()
-		}
-
-		val response = chain.proceed(request)
-		if (response.networkResponse == null) {
-			synchronized(requestQueue) {
-				if (requestQueue.isEmpty() || timestamp < requestQueue.first()) return@synchronized
-				val iterator = requestQueue.iterator()
-				while (iterator.hasNext()) {
-					if (iterator.next() == timestamp) {
-						iterator.remove()
-						break
-					}
-				}
-				(requestQueue as Object).notifyAll()
-			}
-		}
-
-		return response
-	}
 
 	override suspend fun getListPage(
 		page: Int,
 		query: String?,
 		tags: Set<MangaTag>?,
-		sortOrder: SortOrder
+		sortOrder: SortOrder,
 	): List<Manga> {
 		val url = buildString {
 			append("/library")
-			if(query.isNullOrEmpty()){
+			if (query.isNullOrEmpty()) {
 				append("?order_item=")
 				if (sortOrder == SortOrder.POPULARITY) {
 					append("likes_count")
@@ -148,7 +48,7 @@ class TuMangaOnlineParser(context: MangaLoaderContext) : PagedMangaParser (
 				append("&order_dir=desc")
 				append("&filter_by=title")
 				if (tags != null) {
-					for(tag in tags){
+					for (tag in tags) {
 						append("&genders[]=${tag.key}")
 					}
 				}
@@ -162,17 +62,15 @@ class TuMangaOnlineParser(context: MangaLoaderContext) : PagedMangaParser (
 		val doc = webClient.httpGet(url, headers).parseHtml()
 		val items = doc.body().select("div.element")
 		return items.mapNotNull { item ->
-			val href = item.selectFirst("a")?.attrAsRelativeUrlOrNull("href")?.substringAfter(' ') ?: return@mapNotNull null
+			val href =
+				item.selectFirst("a")?.attrAsRelativeUrlOrNull("href")?.substringAfter(' ') ?: return@mapNotNull null
 			Manga(
 				id = generateUid(href),
 				title = item.selectFirst("h4.text-truncate")?.text() ?: return@mapNotNull null,
 				coverUrl = item.select("style").toString().substringAfter("('").substringBeforeLast("')"),
 				altTitle = null,
 				author = null,
-				rating = item.selectFirst("span.score")
+				rating = item.selectFirst("span.score")?.text()?.toFloatOrNull()?.div(10F) ?: RATING_UNKNOWN,
-					?.text()
-					?.toFloatOrNull()
-					?.div(10F) ?: RATING_UNKNOWN,
 				url = href,
 				isNsfw = item.select("i").hasClass("fas fa-heartbeat fa-2x"),
 				tags = emptySet(),
@@ -192,7 +90,7 @@ class TuMangaOnlineParser(context: MangaLoaderContext) : PagedMangaParser (
 			largeCoverUrl = contents.selectFirst(".book-thumbnail")?.attrAsAbsoluteUrlOrNull("src"),
 			state = parseStatus(contents.select("span.book-status").text().orEmpty()),
 			author = contents.selectFirst("h5.card-title")?.attr("title")?.substringAfter(", "),
-			chapters = if(doc.select("div.chapters").isEmpty()){
+			chapters = if (doc.select("div.chapters").isEmpty()) {
 				doc.select(oneShotChapterListSelector()).mapChapters(reversed = true) { i, item ->
 					oneShotChapterFromElement(item)
 				}
@@ -204,14 +102,14 @@ class TuMangaOnlineParser(context: MangaLoaderContext) : PagedMangaParser (
 					scanelement.forEach { chapters.add(regularChapterFromElement(it, chaptername, i)) }
 				}
 				chapters
-			}
+			},
 		)
 	}
 
 	private fun oneShotChapterListSelector() = "div.chapter-list-element > ul.list-group li.list-group-item"
+
 	private fun oneShotChapterFromElement(element: Element): MangaChapter {
-		val href = element.selectFirstOrThrow("div.row > .text-right > a")
+		val href = element.selectFirstOrThrow("div.row > .text-right > a").attrAsRelativeUrl("href")
-			.attrAsRelativeUrl("href")
 		return MangaChapter(
 			id = generateUid(href),
 			name = "One Shot",
@@ -225,9 +123,9 @@ class TuMangaOnlineParser(context: MangaLoaderContext) : PagedMangaParser (
 	}
 
 	private fun regularChapterListSelector() = "div.chapters > ul.list-group li.p-0.list-group-item"
+
 	private fun regularChapterFromElement(element: Element, chName: String, number: Int): MangaChapter {
-		val href = element.selectFirstOrThrow("div.row > .text-right > a")
+		val href = element.selectFirstOrThrow("div.row > .text-right > a").attrAsRelativeUrl("href")
-			.attrAsRelativeUrl("href")
 		return MangaChapter(
 			id = generateUid(href),
 			name = chName,
@@ -255,7 +153,7 @@ class TuMangaOnlineParser(context: MangaLoaderContext) : PagedMangaParser (
 			doc = webClient.httpGet(newUrl, headers).parseHtml()
 		}
 
-		return doc.select("div.viewer-container img:not(noscript img)").map{
+		return doc.select("div.viewer-container img:not(noscript img)").map {
 			val href = if (it.hasAttr("data-src")) {
 				it.attr("abs:data-src")
 			} else {
@@ -274,23 +172,20 @@ class TuMangaOnlineParser(context: MangaLoaderContext) : PagedMangaParser (
 		val script1 = document.selectFirst("script:containsData(uniqid)")
 		val script2 = document.selectFirst("script:containsData(window.location.replace)")
 
-		val redirectHeaders = Headers.Builder()
+		val redirectHeaders = Headers.Builder().set("Referer", document.baseUri()).build()
-			.set("Referer", document.baseUri())
-			.build()
 
 		if (script1 != null) {
 			val data = script1.data()
-			val regexParams = """\{uniqid:'(.+)',cascade:(.+)\}""".toRegex()
+			val regexParams = """\{uniqid:'(.+)',cascade:(.+)}""".toRegex()
 			val regexAction = """form\.action\s?=\s?'(.+)'""".toRegex()
 			val params = regexParams.find(data)!!
-			val action = regexAction.find(data)!!.groupValues[1]
+			val action = regexAction.find(data)!!.groupValues[1].toHttpUrl()
-
-			val formBody = FormBody.Builder()
-				.add("uniqid", params.groupValues[1])
-				.add("cascade", params.groupValues[2])
-				.build()
 
-			return redirectToReadingPage(webClient.httpPost(action,redirectHeaders,formBody).parseHtml())
+			val formBody = mapOf(
+				"uniqid" to params.groupValues[1],
+				"cascade" to params.groupValues[2],
+			)
+			return redirectToReadingPage(webClient.httpPost(action, formBody, redirectHeaders).parseHtml())
 		}
 
 		if (script2 != null) {
@@ -311,7 +206,7 @@ class TuMangaOnlineParser(context: MangaLoaderContext) : PagedMangaParser (
 			MangaTag(
 				title = element.select("label").text(),
 				key = element.select("input").attr("value"),
-				source = source
+				source = source,
 			)
 		}
 	}
@@ -321,19 +216,4 @@ class TuMangaOnlineParser(context: MangaLoaderContext) : PagedMangaParser (
 		status.contains("Finalizado") -> MangaState.FINISHED
 		else -> null
 	}
-
-	private suspend fun WebClient.httpPost(url: String, headers: Headers, body: FormBody): Response {
-		val client = context.httpClient
-		val request = Request.Builder()
-			.post(body)
-			.headers(headers)
-			.url(url)
-		return client.newCall(request.build()).await().ensureSuccess()
-	}
-
-	companion object {
-		private const val permits = 10
-		private const val period = 60L
-		private val unit = TimeUnit.SECONDS
-	}
 }

src/test/kotlin/org/koitharu/kotatsu/parsers/MangaSources.kt

@@ -3,5 +3,5 @@ package org.koitharu.kotatsu.parsers
 import org.junit.jupiter.params.provider.EnumSource
 import org.koitharu.kotatsu.parsers.model.MangaSource
 
-@EnumSource(MangaSource::class, names = ["LOCAL", "DUMMY"], mode = EnumSource.Mode.EXCLUDE)
+@EnumSource(MangaSource::class, names = ["TUMANGAONLINE"], mode = EnumSource.Mode.INCLUDE)
 internal annotation class MangaSources