Add TMO
seew3l
3 years ago
parent
69af4fe586
commit
adf6fea4ac
@ -0,0 +1,392 @@
|
||||
package org.koitharu.kotatsu.parsers.site
|
||||
|
||||
import android.os.SystemClock
|
||||
import okhttp3.FormBody
|
||||
import okhttp3.Headers
|
||||
import okhttp3.Interceptor
|
||||
import okhttp3.OkHttpClient
|
||||
import okhttp3.Request
|
||||
import okhttp3.Response
|
||||
import org.json.JSONObject
|
||||
import org.jsoup.nodes.Document
|
||||
import org.jsoup.nodes.Element
|
||||
import org.koitharu.kotatsu.parsers.MangaLoaderContext
|
||||
import org.koitharu.kotatsu.parsers.MangaSourceParser
|
||||
import org.koitharu.kotatsu.parsers.PagedMangaParser
|
||||
import org.koitharu.kotatsu.parsers.config.ConfigKey
|
||||
import org.koitharu.kotatsu.parsers.model.Manga
|
||||
import org.koitharu.kotatsu.parsers.model.MangaChapter
|
||||
import org.koitharu.kotatsu.parsers.model.MangaPage
|
||||
import org.koitharu.kotatsu.parsers.model.MangaSource
|
||||
import org.koitharu.kotatsu.parsers.model.MangaState
|
||||
import org.koitharu.kotatsu.parsers.model.MangaTag
|
||||
import org.koitharu.kotatsu.parsers.model.RATING_UNKNOWN
|
||||
import org.koitharu.kotatsu.parsers.model.SortOrder
|
||||
import org.koitharu.kotatsu.parsers.util.attrAsAbsoluteUrlOrNull
|
||||
import org.koitharu.kotatsu.parsers.util.attrAsRelativeUrl
|
||||
import org.koitharu.kotatsu.parsers.util.domain
|
||||
import org.koitharu.kotatsu.parsers.util.generateUid
|
||||
import org.koitharu.kotatsu.parsers.util.host
|
||||
import org.koitharu.kotatsu.parsers.util.parseFailed
|
||||
import org.koitharu.kotatsu.parsers.util.parseHtml
|
||||
import org.koitharu.kotatsu.parsers.util.selectFirstOrThrow
|
||||
import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl
|
||||
import java.io.IOException
|
||||
import java.text.SimpleDateFormat
|
||||
import java.util.EnumSet
|
||||
import java.util.Locale
|
||||
import java.util.concurrent.Semaphore
|
||||
import java.util.concurrent.TimeUnit
|
||||
|
||||
/**
|
||||
* This parser is just for parser development, it should not be used in releases
|
||||
*/
|
||||
@MangaSourceParser("TUMANGAONLINE", "TuMangaOnline", "es")
|
||||
class TuMangaOnlineParser(context: MangaLoaderContext) : PagedMangaParser (
|
||||
context,
|
||||
source = MangaSource.TUMANGAONLINE,
|
||||
pageSize = 24,
|
||||
), Interceptor {
|
||||
|
||||
private val client = OkHttpClient.Builder().build()
|
||||
|
||||
override val headers: Headers
|
||||
get() = super.headers.newBuilder()
|
||||
.set("Referer","https://$domain/")
|
||||
.build()
|
||||
|
||||
override val configKeyDomain: ConfigKey.Domain
|
||||
get() = ConfigKey.Domain("lectortmo.com")
|
||||
|
||||
override val sortOrders: Set<SortOrder>
|
||||
get() = EnumSet.of(
|
||||
SortOrder.NEWEST,
|
||||
SortOrder.POPULARITY,
|
||||
)
|
||||
|
||||
private val host = "lectortmo.com"
|
||||
private val permits = 10
|
||||
private val period = 60L
|
||||
private val unit = TimeUnit.SECONDS
|
||||
|
||||
private val requestQueue = ArrayDeque<Long>(permits)
|
||||
private val rateLimitMillis = unit.toMillis(period)
|
||||
private val fairLock = Semaphore(1, true)
|
||||
|
||||
// RateLimit
|
||||
override fun intercept(chain: Interceptor.Chain): Response {
|
||||
val call = chain.call()
|
||||
if (call.isCanceled()) throw IOException("Canceled")
|
||||
|
||||
val request = chain.request()
|
||||
when (host) {
|
||||
request.url.host -> {}
|
||||
else -> return chain.proceed(request)
|
||||
}
|
||||
|
||||
try {
|
||||
fairLock.acquire()
|
||||
} catch (e: InterruptedException) {
|
||||
throw IOException(e)
|
||||
}
|
||||
|
||||
val requestQueue = this.requestQueue
|
||||
val timestamp: Long
|
||||
|
||||
try {
|
||||
synchronized(requestQueue) {
|
||||
while (requestQueue.size >= permits) { // queue is full, remove expired entries
|
||||
val periodStart = SystemClock.elapsedRealtime() - rateLimitMillis
|
||||
var hasRemovedExpired = false
|
||||
while (requestQueue.isEmpty().not() && requestQueue.first() <= periodStart) {
|
||||
requestQueue.removeFirst()
|
||||
hasRemovedExpired = true
|
||||
}
|
||||
if (call.isCanceled()) {
|
||||
throw IOException("Canceled")
|
||||
} else if (hasRemovedExpired) {
|
||||
break
|
||||
} else {
|
||||
try {
|
||||
(requestQueue as Object).wait(requestQueue.first() - periodStart)
|
||||
} catch (_: InterruptedException) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
timestamp = SystemClock.elapsedRealtime()
|
||||
requestQueue.addLast(timestamp)
|
||||
}
|
||||
} finally {
|
||||
fairLock.release()
|
||||
}
|
||||
|
||||
val response = chain.proceed(request)
|
||||
if (response.networkResponse == null) {
|
||||
synchronized(requestQueue) {
|
||||
if (requestQueue.isEmpty() || timestamp < requestQueue.first()) return@synchronized
|
||||
val iterator = requestQueue.iterator()
|
||||
while (iterator.hasNext()) {
|
||||
if (iterator.next() == timestamp) {
|
||||
iterator.remove()
|
||||
break
|
||||
}
|
||||
}
|
||||
(requestQueue as Object).notifyAll()
|
||||
}
|
||||
}
|
||||
|
||||
return response
|
||||
}
|
||||
|
||||
override suspend fun getListPage(
|
||||
page: Int,
|
||||
query: String?,
|
||||
tags: Set<MangaTag>?,
|
||||
sortOrder: SortOrder
|
||||
): List<Manga> {
|
||||
val url = buildString {
|
||||
append("/library")
|
||||
if(query.isNullOrEmpty()){
|
||||
append("?order_item=")
|
||||
if (sortOrder == SortOrder.POPULARITY) {
|
||||
append("likes_count")
|
||||
}
|
||||
if (sortOrder == SortOrder.NEWEST) {
|
||||
append("creation")
|
||||
}
|
||||
append("&order_dir=desc")
|
||||
append("&filter_by=title")
|
||||
if (tags != null) {
|
||||
for(tag in tags){
|
||||
append("&genders[]=${tag.key}")
|
||||
}
|
||||
}
|
||||
} else {
|
||||
append("?title=$query")
|
||||
}
|
||||
append("&_pg=1")
|
||||
append("&page=$page")
|
||||
}.toAbsoluteUrl(domain)
|
||||
|
||||
val doc = webClient.httpGet(url, headers).parseHtml()
|
||||
val items = doc.body().select("div.element")
|
||||
return items.mapNotNull { item ->
|
||||
val href = item.selectFirst("a")?.attrAsRelativeUrl("href")?.substringAfter(" ") ?: return@mapNotNull null
|
||||
Manga(
|
||||
id = generateUid(href),
|
||||
title = item.selectFirst("h4.text-truncate")?.text() ?: return@mapNotNull null,
|
||||
coverUrl = item.select("style").toString().substringAfter("('").substringBeforeLast("')"),
|
||||
altTitle = null,
|
||||
author = null,
|
||||
rating = item.selectFirst("span.score")
|
||||
?.text()
|
||||
?.toFloatOrNull()
|
||||
?.div(10F) ?: RATING_UNKNOWN,
|
||||
url = href,
|
||||
isNsfw = item.select("i").hasClass("fas fa-heartbeat fa-2x"),
|
||||
tags = emptySet(),
|
||||
state = null,
|
||||
publicUrl = href.toAbsoluteUrl(doc.host ?: domain),
|
||||
source = source,
|
||||
)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
override suspend fun getDetails(manga: Manga): Manga {
|
||||
val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml()
|
||||
val contents = doc.body().selectFirstOrThrow("section.element-header-content")
|
||||
return manga.copy(
|
||||
description = contents.selectFirst("p.element-description")?.html(),
|
||||
largeCoverUrl = contents.selectFirst(".book-thumbnail")?.attrAsAbsoluteUrlOrNull("src"),
|
||||
state = parseStatus(contents.select("span.book-status").text().orEmpty()),
|
||||
author = contents.selectFirst("h5.card-title")?.attr("title")?.substringAfter(", "),
|
||||
chapters = if(doc.select("div.chapters").isEmpty()){
|
||||
doc.select(oneShotChapterListSelector()).reversed().map { oneShotChapterFromElement(it) }
|
||||
} else {
|
||||
val chapters = mutableListOf<MangaChapter>()
|
||||
doc.select(regularChapterListSelector()).reversed().forEachIndexed{ i, chapelement ->
|
||||
val chaptername = chapelement.select("div.col-10.text-truncate").text().replace(" ", " ").trim()
|
||||
val scanelement = chapelement.select("ul.chapter-list > li")
|
||||
scanelement.forEach { chapters.add(regularChapterFromElement(it, chaptername, i)) }
|
||||
}
|
||||
chapters
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
private fun oneShotChapterListSelector() = "div.chapter-list-element > ul.list-group li.list-group-item"
|
||||
private fun oneShotChapterFromElement(element: Element): MangaChapter {
|
||||
val href = element.selectFirst("div.row > .text-right > a")
|
||||
?.attrAsRelativeUrl("href") ?: element.parseFailed()
|
||||
return MangaChapter(
|
||||
id = generateUid(href),
|
||||
name = "One Shot",
|
||||
number = 1,
|
||||
url = href,
|
||||
scanlator = element.select("div.col-md-6.text-truncate").text(),
|
||||
branch = null,
|
||||
uploadDate = element.select("span.badge.badge-primary.p-2").first()?.text()
|
||||
?.let { parseChapterDate(it) }
|
||||
?: 0,
|
||||
source = source,
|
||||
)
|
||||
}
|
||||
|
||||
private fun regularChapterListSelector() = "div.chapters > ul.list-group li.p-0.list-group-item"
|
||||
private fun regularChapterFromElement(element: Element, chName: String, number: Int): MangaChapter {
|
||||
val href = element.selectFirst("div.row > .text-right > a")
|
||||
?.attrAsRelativeUrl("href") ?: element.parseFailed()
|
||||
return MangaChapter(
|
||||
id = generateUid(href),
|
||||
name = chName,
|
||||
number = number + 1,
|
||||
url = href,
|
||||
scanlator = element.select("div.col-md-6.text-truncate").text(),
|
||||
branch = null,
|
||||
uploadDate = element.select("span.badge.badge-primary.p-2").first()?.text()
|
||||
?.let { parseChapterDate(it) }
|
||||
?: 0,
|
||||
source = source,
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
|
||||
val redirectDoc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml()
|
||||
var doc = redirectToReadingPage(redirectDoc)
|
||||
val currentUrl = doc.location()
|
||||
val newUrl = if (!currentUrl.contains("cascade")) {
|
||||
currentUrl.substringBefore("paginated") + "cascade"
|
||||
} else {
|
||||
currentUrl
|
||||
}
|
||||
|
||||
if (currentUrl != newUrl) {
|
||||
doc = webClient.httpGet(newUrl, headers).parseHtml()
|
||||
}
|
||||
|
||||
return doc.select("div.viewer-container img:not(noscript img)").map{
|
||||
val href = if (it.hasAttr("data-src")) {
|
||||
it.attr("abs:data-src")
|
||||
} else {
|
||||
it.attr("abs:src")
|
||||
}
|
||||
MangaPage(
|
||||
id = generateUid(href),
|
||||
url = href,
|
||||
preview = null,
|
||||
source = source,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// Some chapters uses JavaScript to redirect to read page
|
||||
private suspend fun redirectToReadingPage(document: Document): Document {
|
||||
val script1 = document.selectFirst("script:containsData(uniqid)")
|
||||
val script2 = document.selectFirst("script:containsData(window.location.replace)")
|
||||
|
||||
val redirectHeaders = Headers.Builder()
|
||||
.set("Referer", document.baseUri())
|
||||
.build()
|
||||
|
||||
if (script1 != null) {
|
||||
val data = script1.data()
|
||||
val regexParams = """\{uniqid:'(.+)',cascade:(.+)\}""".toRegex()
|
||||
val regexAction = """form\.action\s?=\s?'(.+)'""".toRegex()
|
||||
val params = regexParams.find(data)!!
|
||||
val action = regexAction.find(data)!!.groupValues[1]
|
||||
|
||||
val body = JSONObject()
|
||||
body.put("uniqid",params.groupValues[1])
|
||||
body.put("cascade",params.groupValues[2])
|
||||
|
||||
val formBody = FormBody.Builder()
|
||||
.add("uniqid", params.groupValues[1])
|
||||
.add("cascade", params.groupValues[2])
|
||||
.build()
|
||||
|
||||
val postRequest = Request.Builder()
|
||||
.url(action)
|
||||
.headers(redirectHeaders)
|
||||
.post(formBody)
|
||||
.build()
|
||||
|
||||
return redirectToReadingPage(client.newCall(postRequest).execute().parseHtml())
|
||||
}
|
||||
|
||||
if (script2 != null) {
|
||||
val data = script2.data()
|
||||
val regexRedirect = """window\.location\.replace\('(.+)'\)""".toRegex()
|
||||
val url = regexRedirect.find(data)!!.groupValues[1]
|
||||
|
||||
return redirectToReadingPage(webClient.httpGet(url, redirectHeaders).parseHtml())
|
||||
}
|
||||
|
||||
return document
|
||||
}
|
||||
|
||||
override suspend fun getTags() = setOf<MangaTag>(
|
||||
MangaTag("Acción", "1", source),
|
||||
MangaTag("Aventura", "2", source),
|
||||
MangaTag("Comedia", "3", source),
|
||||
MangaTag("Drama", "4", source),
|
||||
MangaTag("Recuentos de la vida", "5", source),
|
||||
MangaTag("Ecchi", "6", source),
|
||||
MangaTag("Fantasia", "7", source),
|
||||
MangaTag("Magia", "8", source),
|
||||
MangaTag("Sobrenatural", "9", source),
|
||||
MangaTag("Horror", "10", source),
|
||||
MangaTag("Misterio", "11", source),
|
||||
MangaTag("Psicológico", "12", source),
|
||||
MangaTag("Romance", "13", source),
|
||||
MangaTag("Ciencia Ficción", "14", source),
|
||||
MangaTag("Thriller", "15", source),
|
||||
MangaTag("Deporte", "16", source),
|
||||
MangaTag("Girls Love", "17", source),
|
||||
MangaTag("Boys Love", "18", source),
|
||||
MangaTag("Harem", "19", source),
|
||||
MangaTag("Mecha", "20", source),
|
||||
MangaTag("Supervivencia", "21", source),
|
||||
MangaTag("Reencarnación", "22", source),
|
||||
MangaTag("Gore", "23", source),
|
||||
MangaTag("Apocalíptico", "24", source),
|
||||
MangaTag("Tragedia", "25", source),
|
||||
MangaTag("Vida Escolar", "26", source),
|
||||
MangaTag("Historia", "27", source),
|
||||
MangaTag("Militar", "28", source),
|
||||
MangaTag("Policiaco", "29", source),
|
||||
MangaTag("Crimen", "30", source),
|
||||
MangaTag("Superpoderes", "31", source),
|
||||
MangaTag("Vampiros", "32", source),
|
||||
MangaTag("Artes Marciales", "33", source),
|
||||
MangaTag("Samurái", "34", source),
|
||||
MangaTag("Género Bender", "35", source),
|
||||
MangaTag("Realidad Virtual", "36", source),
|
||||
MangaTag("Ciberpunk", "37", source),
|
||||
MangaTag("Musica", "38", source),
|
||||
MangaTag("Parodia", "39", source),
|
||||
MangaTag("Animación", "40", source),
|
||||
MangaTag("Demonios", "41", source),
|
||||
MangaTag("Familia", "42", source),
|
||||
MangaTag("Extranjero", "43", source),
|
||||
MangaTag("Niños", "44", source),
|
||||
MangaTag("Realidad", "45", source),
|
||||
MangaTag("Telenovela", "46", source),
|
||||
MangaTag("Guerra", "47", source),
|
||||
MangaTag("Oeste", "48", source),
|
||||
)
|
||||
|
||||
|
||||
private fun parseStatus(status: String) = when {
|
||||
status.contains("Publicándose") -> MangaState.ONGOING
|
||||
status.contains("Finalizado") -> MangaState.FINISHED
|
||||
else -> null
|
||||
}
|
||||
|
||||
private fun parseChapterDate(date: String): Long =
|
||||
SimpleDateFormat("yyyy-MM-dd", Locale.getDefault()).parse(date)?.time ?: 0
|
||||
|
||||
}
|
||||
Loading…
Reference in New Issue