KotatsuApp
/
kotatsu-parsers
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[LineWebtoons] Refactoring

Browse Source
pull/286/head
Koitharu 3 years ago
parent 9103f9d473
commit 400a90464e
Signed by: Koitharu
GPG Key ID: 676DEE768C17A9D7
1 changed files with 94 additions and 202 deletions
Show Stats Download Patch File Download Diff File

296
src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/LineWebtoonsParser.kt
Unescape Escape View File

@ -1,27 +1,32 @@
package org.koitharu.kotatsu.parsers.site.en
import kotlinx.coroutines.async
import kotlinx.coroutines.coroutineScope
import okhttp3.Headers
import okhttp3.HttpUrl
import okhttp3.HttpUrl.Companion.toHttpUrl
import org.json.JSONObject
import org.koitharu.kotatsu.parsers.MangaLoaderContext
import org.koitharu.kotatsu.parsers.MangaParser
import org.koitharu.kotatsu.parsers.MangaSourceParser
import org.koitharu.kotatsu.parsers.config.ConfigKey
import org.koitharu.kotatsu.parsers.exception.NotFoundException
import org.koitharu.kotatsu.parsers.exception.ParseException
import org.koitharu.kotatsu.parsers.model.*
import org.koitharu.kotatsu.parsers.util.*
import org.koitharu.kotatsu.parsers.util.json.getStringOrNull
import org.koitharu.kotatsu.parsers.util.json.mapJSON
import org.koitharu.kotatsu.parsers.util.json.mapJSONIndexed
import org.koitharu.kotatsu.parsers.util.json.mapJSONToSet
import org.koitharu.kotatsu.parsers.util.json.toJSONList
import java.net.URLEncoder
import org.koitharu.kotatsu.parsers.util.json.*
import java.util.*
import javax.crypto.Mac
import javax.crypto.spec.SecretKeySpec
internal abstract class LineWebtoonsParser(context: MangaLoaderContext, source: MangaSource) : MangaParser(context, source) {
private val signer = WebtoonsUrlSigner("gUtPzJFZch4ZyAGviiyH94P99lQ3pFdRTwpJWDlSGFfwgpr6ses5ALOxWHOIT7R1")
internal abstract class LineWebtoonsParser(
context: MangaLoaderContext,
source: MangaSource,
) : MangaParser(context, source) {
private val signer by lazy {
WebtoonsUrlSigner("gUtPzJFZch4ZyAGviiyH94P99lQ3pFdRTwpJWDlSGFfwgpr6ses5ALOxWHOIT7R1")
}
// we don't __really__ support changing this domain because:
// 1. I don't think other websites have this exact API
@ -29,8 +34,7 @@ internal abstract class LineWebtoonsParser(context: MangaLoaderContext, source:
// 3. we rely on the HTTP client setting the referer header to webtoons.com
//
// This effectively means that changing the domain will break the source. Yikes
override val configKeyDomain
get() = ConfigKey.Domain("webtoons.com")
override val configKeyDomain = ConfigKey.Domain("webtoons.com")
private val apiDomain = "global.apis.naver.com"
private val staticDomain = "webtoon-phinf.pstatic.net"
@ -48,7 +52,7 @@ internal abstract class LineWebtoonsParser(context: MangaLoaderContext, source:
.build()
override suspend fun getPageUrl(page: MangaPage): String {
return page.url
return page.url.toAbsoluteUrl(staticDomain)
}
// some language tags do not map perfectly to the ones used by the API
@ -60,7 +64,9 @@ internal abstract class LineWebtoonsParser(context: MangaLoaderContext, source:
}
private suspend fun getChapters(titleNo: Long): List<MangaChapter> {
val firstResult = makeRequest("/lineWebtoon/webtoon/challengeEpisodeList.json?v=2&titleNo=$titleNo&startIndex=0&pageSize=30")
val firstResult = makeRequest(
url = "/lineWebtoon/webtoon/challengeEpisodeList.json?v=2&titleNo=$titleNo&startIndex=0&pageSize=30",
)
val totalEpisodeCount = firstResult
.getJSONObject("episodeList")
@ -73,20 +79,21 @@ internal abstract class LineWebtoonsParser(context: MangaLoaderContext, source:
.toMutableList()
while (episodes.count() < totalEpisodeCount) {
val page = makeRequest("/lineWebtoon/webtoon/challengeEpisodeList.json?v=2&titleNo=$titleNo&startIndex=${episodes.count()}&pageSize=30")
.getJSONObject("episodeList")
val page = makeRequest(
url = "/lineWebtoon/webtoon/challengeEpisodeList.json?v=2&titleNo=$titleNo&startIndex=${episodes.count()}&pageSize=30",
).getJSONObject("episodeList")
.getJSONArray("episode")
.toJSONList()
episodes.addAll(page)
}
return episodes.mapIndexed { i, jo ->
return episodes.mapChapters { i, jo ->
MangaChapter(
id = generateUid("$titleNo-$i"),
name = jo.getString("episodeTitle"),
number = jo.getInt("episodeSeq"),
url = "$titleNo-${jo.getString("episodeNo")}",
url = "$titleNo-${jo.get("episodeNo")}",
uploadDate = jo.getLong("modifyYmdt"),
branch = null,
scanlator = null,
@ -95,10 +102,11 @@ internal abstract class LineWebtoonsParser(context: MangaLoaderContext, source:
}.sortedBy { it.number }
}
override suspend fun getDetails(manga: Manga): Manga {
override suspend fun getDetails(manga: Manga): Manga = coroutineScope {
val titleNo = manga.url.toLong()
val chaptersDeferred = async { getChapters(titleNo) }
return makeRequest("/lineWebtoon/webtoon/challengeTitleInfo.json?v=2&titleNo=${titleNo}")
makeRequest("/lineWebtoon/webtoon/challengeTitleInfo.json?v=2&titleNo=${titleNo}")
.getJSONObject("titleInfo")
.let { jo ->
Manga(
@ -107,18 +115,16 @@ internal abstract class LineWebtoonsParser(context: MangaLoaderContext, source:
altTitle = null,
url = "$titleNo",
publicUrl = "https://$domain/$languageCode/canvas/a/list?title_no=${titleNo}",
rating = jo.getDouble("starScoreAverage").toFloat() / 10f,
isNsfw = jo.getBoolean("ageGradeNotice"),
coverUrl = "https://$staticDomain${jo.getString("thumbnail")}",
largeCoverUrl = if (jo.has("thumbnailVertical")) {
"https://$staticDomain${jo.getString("thumbnailVertical")}"
} else { null },
rating = jo.getFloatOrDefault("starScoreAverage", -10f) / 10f,
isNsfw = jo.getBooleanOrDefault("ageGradeNotice", isNsfwSource),
coverUrl = jo.getString("thumbnail").toAbsoluteUrl(staticDomain),
largeCoverUrl = jo.getStringOrNull("thumbnailVertical")?.toAbsoluteUrl(staticDomain),
tags = setOf(parseTag(jo.getJSONObject("genreInfo"))),
author = jo.getString("writingAuthorName"),
author = jo.getStringOrNull("writingAuthorName"),
description = jo.getString("synopsis"),
// I don't think the API provides this info
state = null,
chapters = getChapters(titleNo),
chapters = chaptersDeferred.await(),
source = source,
)
}
@ -136,9 +142,7 @@ internal abstract class LineWebtoonsParser(context: MangaLoaderContext, source:
SortOrder.UPDATED -> "UPDATE"
SortOrder.POPULARITY -> "READ_COUNT"
SortOrder.RATING -> "LIKEIT"
else -> {
throw IllegalArgumentException("Unsupported sort order: $sortOrder")
}
else -> throw IllegalArgumentException("Unsupported sort order: $sortOrder")
}
val manga = if (query != null) {
@ -146,7 +150,7 @@ internal abstract class LineWebtoonsParser(context: MangaLoaderContext, source:
throw IllegalArgumentException("This source does not support search with tags")
}
makeRequest("/lineWebtoon/webtoon/searchChallenge?query=${query.urlEncoded()}&startIndex=${offset+1}&pageSize=20")
makeRequest("/lineWebtoon/webtoon/searchChallenge?query=${query.urlEncoded()}&startIndex=${offset + 1}&pageSize=20")
.getJSONObject("challengeSearch")
.getJSONArray("titleList")
.mapJSON { jo ->
@ -156,21 +160,22 @@ internal abstract class LineWebtoonsParser(context: MangaLoaderContext, source:
id = generateUid(titleNo),
title = jo.getString("title"),
altTitle = null,
url = "$titleNo",
url = titleNo.toString(),
publicUrl = "https://$domain/$languageCode/canvas/a/list?title_no=$titleNo",
rating = RATING_UNKNOWN,
isNsfw = false,
coverUrl = "https://$staticDomain${jo.getString("thumbnail")}",
isNsfw = isNsfwSource,
coverUrl = jo.getString("thumbnail").toAbsoluteUrl(staticDomain),
largeCoverUrl = null,
tags = setOf(),
author = jo.getString("writingAuthorName"),
tags = emptySet(),
author = jo.getStringOrNull("writingAuthorName"),
description = null,
state = null,
source = source,
)
}
} else {
val result = makeRequest("/lineWebtoon/webtoon/challengeGenreTitleList.json?genre=$genre&sortOrder=$sortOrderStr&startIndex=${offset+1}&pageSize=20")
val result =
makeRequest("/lineWebtoon/webtoon/challengeGenreTitleList.json?genre=$genre&sortOrder=$sortOrderStr&startIndex=${offset + 1}&pageSize=20")
val genres = result.getJSONObject("genreList")
.getJSONArray("challengeGenres")
@ -187,14 +192,14 @@ internal abstract class LineWebtoonsParser(context: MangaLoaderContext, source:
id = generateUid(titleNo),
title = jo.getString("title"),
altTitle = null,
url = "$titleNo",
url = titleNo.toString(),
publicUrl = "https://$domain/$languageCode/canvas/a/list?title_no=$titleNo",
rating = jo.getDouble("starScoreAverage").toFloat() / 10f,
isNsfw = jo.getBoolean("ageGradeNotice"),
coverUrl = "https://$staticDomain${jo.getString("thumbnail")}",
rating = jo.getFloatOrDefault("starScoreAverage", -10f) / 10f,
isNsfw = jo.getBooleanOrDefault("ageGradeNotice", isNsfwSource),
coverUrl = jo.getString("thumbnail").toAbsoluteUrl(staticDomain),
largeCoverUrl = jo.getStringOrNull("thumbnailVertical")?.toAbsoluteUrl(staticDomain),
tags = setOf(genres[jo.getString("representGenre")]!!),
author = jo.getString("writingAuthorName"),
tags = setOfNotNull(genres[jo.getString("representGenre")]),
author = jo.getStringOrNull("writingAuthorName"),
description = jo.getString("synopsis"),
// I don't think the API provides this info
state = null,
@ -212,10 +217,10 @@ internal abstract class LineWebtoonsParser(context: MangaLoaderContext, source:
return makeRequest("/lineWebtoon/webtoon/challengeEpisodeInfo.json?v=2&titleNo=$titleNo&episodeNo=$episodeNo")
.getJSONObject("episodeInfo")
.getJSONArray("imageInfo")
.mapJSONIndexed() { i, jo ->
.mapJSONIndexed { i, jo ->
MangaPage(
id = generateUid("$titleNo-$episodeNo-$i"),
url = "https://$staticDomain${jo.getString("url")}",
url = jo.getString("url"),
preview = null,
source = source,
)
@ -239,188 +244,75 @@ internal abstract class LineWebtoonsParser(context: MangaLoaderContext, source:
private suspend fun makeRequest(url: String): JSONObject {
val resp = webClient.httpGet(finalizeUrl(url))
val message = resp.parseJson().getJSONObject("message")
if (resp.isSuccessful) {
return message.getJSONObject("result")
} else {
// TODO: handle 404 and some other error codes
val code = message.getInt("code")
val errorMessage = message.getString("message")
throw ParseException("Api error (code=$code): $errorMessage", url)
val message: JSONObject? = resp.parseJson().optJSONObject("message")
return when (resp.code) {
in 200..299 -> checkNotNull(message).getJSONObject("result")
404 -> throw NotFoundException(message?.getStringOrNull("message").orEmpty(), url)
else -> {
val code = message?.getIntOrDefault("code", 0)
val errorMessage = message?.getStringOrNull("message")
throw ParseException("Api error (code=$code): $errorMessage", url)
}
}
}
private fun finalizeUrl(url: String): String {
val absoluteUrl = url.toAbsoluteUrl(apiDomain)
val parsedUrl = absoluteUrl.toHttpUrl()
val hasVersion = parsedUrl.queryParameter("v") != null
val hasQuery = parsedUrl.query != null
val urlWithParams = absoluteUrl + if (hasQuery) {
"&"
} else {
"?"
} + "serviceZone=GLOBAL&" + if (!hasVersion) {
"v=1"
} else { "" } + "&language=$languageCode&locale=$languageCode&platform=APP_ANDROID"
return signer.makeEncryptUrl(urlWithParams)
private fun finalizeUrl(url: String): HttpUrl {
val httpUrl = url.toAbsoluteUrl(apiDomain).toHttpUrl()
val builder = httpUrl.newBuilder()
.addQueryParameter("serviceZone", "GLOBAL")
if (httpUrl.queryParameter("v") == null) {
builder.addQueryParameter("v", "1")
}
builder.addQueryParameter("language", languageCode)
.addQueryParameter("locale", "languageCode")
.addQueryParameter("platform", "APP_ANDROID")
signer.makeEncryptUrl(builder)
return builder.build()
}
@MangaSourceParser("LINEWEBTOONS_EN", "Line Webtoons English", "en", type = ContentType.MANGA)
class English(context: MangaLoaderContext) : LineWebtoonsParser(context, MangaSource.LINEWEBTOONS_EN)
@MangaSourceParser("LINEWEBTOONS_ZH", "Line Webtoons Chinese", "zh", type = ContentType.MANGA)
class Chinese(context: MangaLoaderContext) : LineWebtoonsParser(context, MangaSource.LINEWEBTOONS_ZH)
@MangaSourceParser("LINEWEBTOONS_TH", "Line Webtoons Thai", "th", type = ContentType.MANGA)
class Thai(context: MangaLoaderContext) : LineWebtoonsParser(context, MangaSource.LINEWEBTOONS_TH)
@MangaSourceParser("LINEWEBTOONS_ID", "Line Webtoons Indonesian", "id", type = ContentType.MANGA)
class Indonesian(context: MangaLoaderContext) : LineWebtoonsParser(context, MangaSource.LINEWEBTOONS_ID)
@MangaSourceParser("LINEWEBTOONS_ES", "Line Webtoons Spanish", "es", type = ContentType.MANGA)
class Spanish(context: MangaLoaderContext) : LineWebtoonsParser(context, MangaSource.LINEWEBTOONS_ES)
@MangaSourceParser("LINEWEBTOONS_FR", "Line Webtoons French", "fr", type = ContentType.MANGA)
class French(context: MangaLoaderContext) : LineWebtoonsParser(context, MangaSource.LINEWEBTOONS_FR)
@MangaSourceParser("LINEWEBTOONS_DE", "Line Webtoons German", "de", type = ContentType.MANGA)
class German(context: MangaLoaderContext) : LineWebtoonsParser(context, MangaSource.LINEWEBTOONS_DE)
}
private inner class WebtoonsUrlSigner(private val secret: String) {
private class WebtoonsUrlSigner(val secret: String) {
private val mac = Mac.getInstance("HmacSHA1").apply {
this.init(SecretKeySpec(secret.encodeToByteArray(), "HmacSHA1"))
}
private fun getMessage(url: String, msgpad: String): String {
return url.substring(0, 0xFF.coerceAtMost(url.length)) + msgpad
}
private fun getMessageDigest(s: String): String {
var signedMessage: ByteArray
synchronized(mac) { signedMessage = mac.doFinal(s.toByteArray()) }
// we don't use the context.encodeBase64 here because it adds newlines and doesn't add padding
// we, however, need padding and no newlines
return encodeBase64(signedMessage)
}
fun makeEncryptUrl(s: String): String {
return makeEncryptUrlCore(
s,
java.lang.String.valueOf(Calendar.getInstance().timeInMillis),
)
}
private fun makeEncryptUrlCore(url: String, msgpad: String): String {
val digest = URLEncoder.encode(getMessageDigest(getMessage(url, msgpad)), "utf-8")
return url + if (url.contains("?")) {
"&"
} else {
"?"
} + "msgpad=$msgpad&md=$digest"
}
}
private val mac = Mac.getInstance("HmacSHA1").apply {
this.init(SecretKeySpec(secret.encodeToByteArray(), "HmacSHA1"))
}
private fun getMessage(url: String, msgpad: String): String {
return url.substring(0, 0xFF.coerceAtMost(url.length)) + msgpad
}
private val INT_TO_BASE64: CharArray = charArrayOf(
'A',
'B',
'C',
'D',
'E',
'F',
'G',
'H',
'I',
'J',
'K',
'L',
'M',
'N',
'O',
'P',
'Q',
'R',
'S',
'T',
'U',
'V',
'W',
'X',
'Y',
'Z',
'a',
'b',
'c',
'd',
'e',
'f',
'g',
'h',
'i',
'j',
'k',
'l',
'm',
'n',
'o',
'p',
'q',
'r',
's',
't',
'u',
'v',
'w',
'x',
'y',
'z',
'0',
'1',
'2',
'3',
'4',
'5',
'6',
'7',
'8',
'9',
'+',
'/',
)
private fun encodeBase64(arr: ByteArray): String {
val groupsCount = arr.size / 3
val extraCount = arr.size - groupsCount * 3
val sb = StringBuffer((arr.size + 2) / 3 * 4)
val code = INT_TO_BASE64
var groupIndex = 0
var position = 0
while (groupIndex < groupsCount) {
val v4 = arr[position].toInt() and 0xFF
val v5 = arr[position + 1].toInt() and 0xFF
val v6 = arr[position + 2].toInt() and 0xFF
sb.append(code[v4 shr 2])
sb.append(code[v4 shl 4 and 0x3F or (v5 shr 4)])
sb.append(code[v5 shl 2 and 0x3F or (v6 shr 6)])
sb.append(code[v6 and 0x3F])
++groupIndex
position += 3
}
private fun getMessageDigest(s: String): String {
val signedMessage = synchronized(mac) { mac.doFinal(s.toByteArray()) }
return context.encodeBase64(signedMessage)
}
if (extraCount != 0) {
val v1 = arr[position].toInt() and 0xFF
sb.append(code[v1 shr 2])
if (extraCount == 1) {
sb.append(code[v1 shl 4 and 0x3F])
sb.append("==")
} else {
val v2 = arr[position + 1].toInt() and 0xFF
sb.append(code[v1 shl 4 and 0x3F or (v2 shr 4)])
sb.append(code[v2 shl 2 and 0x3F])
sb.append('=')
fun makeEncryptUrl(urlBuilder: HttpUrl.Builder) {
val msgpad = Calendar.getInstance().timeInMillis.toString()
val digest = getMessageDigest(getMessage(urlBuilder.build().toString(), msgpad))
urlBuilder
.addQueryParameter("msgpad", msgpad)
.addQueryParameter("md", digest)
// .addEncodedQueryParameter("md", digest.urlEncoded())
}
}
return sb.toString()
}

Write Preview
Loading…
Cancel
Save