diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/LineWebtoonsParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/LineWebtoonsParser.kt index b80c28da..f60a07de 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/LineWebtoonsParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/LineWebtoonsParser.kt @@ -1,27 +1,32 @@ package org.koitharu.kotatsu.parsers.site.en +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope import okhttp3.Headers +import okhttp3.HttpUrl import okhttp3.HttpUrl.Companion.toHttpUrl import org.json.JSONObject import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaParser import org.koitharu.kotatsu.parsers.MangaSourceParser import org.koitharu.kotatsu.parsers.config.ConfigKey +import org.koitharu.kotatsu.parsers.exception.NotFoundException import org.koitharu.kotatsu.parsers.exception.ParseException import org.koitharu.kotatsu.parsers.model.* import org.koitharu.kotatsu.parsers.util.* -import org.koitharu.kotatsu.parsers.util.json.getStringOrNull -import org.koitharu.kotatsu.parsers.util.json.mapJSON -import org.koitharu.kotatsu.parsers.util.json.mapJSONIndexed -import org.koitharu.kotatsu.parsers.util.json.mapJSONToSet -import org.koitharu.kotatsu.parsers.util.json.toJSONList -import java.net.URLEncoder +import org.koitharu.kotatsu.parsers.util.json.* import java.util.* import javax.crypto.Mac import javax.crypto.spec.SecretKeySpec -internal abstract class LineWebtoonsParser(context: MangaLoaderContext, source: MangaSource) : MangaParser(context, source) { - private val signer = WebtoonsUrlSigner("gUtPzJFZch4ZyAGviiyH94P99lQ3pFdRTwpJWDlSGFfwgpr6ses5ALOxWHOIT7R1") +internal abstract class LineWebtoonsParser( + context: MangaLoaderContext, + source: MangaSource, +) : MangaParser(context, source) { + + private val signer by lazy { + WebtoonsUrlSigner("gUtPzJFZch4ZyAGviiyH94P99lQ3pFdRTwpJWDlSGFfwgpr6ses5ALOxWHOIT7R1") + } // we don't __really__ support changing this domain because: // 1. I don't think other websites have this exact API @@ -29,8 +34,7 @@ internal abstract class LineWebtoonsParser(context: MangaLoaderContext, source: // 3. we rely on the HTTP client setting the referer header to webtoons.com // // This effectively means that changing the domain will break the source. Yikes - override val configKeyDomain - get() = ConfigKey.Domain("webtoons.com") + override val configKeyDomain = ConfigKey.Domain("webtoons.com") private val apiDomain = "global.apis.naver.com" private val staticDomain = "webtoon-phinf.pstatic.net" @@ -48,7 +52,7 @@ internal abstract class LineWebtoonsParser(context: MangaLoaderContext, source: .build() override suspend fun getPageUrl(page: MangaPage): String { - return page.url + return page.url.toAbsoluteUrl(staticDomain) } // some language tags do not map perfectly to the ones used by the API @@ -60,7 +64,9 @@ internal abstract class LineWebtoonsParser(context: MangaLoaderContext, source: } private suspend fun getChapters(titleNo: Long): List { - val firstResult = makeRequest("/lineWebtoon/webtoon/challengeEpisodeList.json?v=2&titleNo=$titleNo&startIndex=0&pageSize=30") + val firstResult = makeRequest( + url = "/lineWebtoon/webtoon/challengeEpisodeList.json?v=2&titleNo=$titleNo&startIndex=0&pageSize=30", + ) val totalEpisodeCount = firstResult .getJSONObject("episodeList") @@ -73,20 +79,21 @@ internal abstract class LineWebtoonsParser(context: MangaLoaderContext, source: .toMutableList() while (episodes.count() < totalEpisodeCount) { - val page = makeRequest("/lineWebtoon/webtoon/challengeEpisodeList.json?v=2&titleNo=$titleNo&startIndex=${episodes.count()}&pageSize=30") - .getJSONObject("episodeList") + val page = makeRequest( + url = "/lineWebtoon/webtoon/challengeEpisodeList.json?v=2&titleNo=$titleNo&startIndex=${episodes.count()}&pageSize=30", + ).getJSONObject("episodeList") .getJSONArray("episode") .toJSONList() episodes.addAll(page) } - return episodes.mapIndexed { i, jo -> + return episodes.mapChapters { i, jo -> MangaChapter( id = generateUid("$titleNo-$i"), name = jo.getString("episodeTitle"), number = jo.getInt("episodeSeq"), - url = "$titleNo-${jo.getString("episodeNo")}", + url = "$titleNo-${jo.get("episodeNo")}", uploadDate = jo.getLong("modifyYmdt"), branch = null, scanlator = null, @@ -95,10 +102,11 @@ internal abstract class LineWebtoonsParser(context: MangaLoaderContext, source: }.sortedBy { it.number } } - override suspend fun getDetails(manga: Manga): Manga { + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { val titleNo = manga.url.toLong() + val chaptersDeferred = async { getChapters(titleNo) } - return makeRequest("/lineWebtoon/webtoon/challengeTitleInfo.json?v=2&titleNo=${titleNo}") + makeRequest("/lineWebtoon/webtoon/challengeTitleInfo.json?v=2&titleNo=${titleNo}") .getJSONObject("titleInfo") .let { jo -> Manga( @@ -107,18 +115,16 @@ internal abstract class LineWebtoonsParser(context: MangaLoaderContext, source: altTitle = null, url = "$titleNo", publicUrl = "https://$domain/$languageCode/canvas/a/list?title_no=${titleNo}", - rating = jo.getDouble("starScoreAverage").toFloat() / 10f, - isNsfw = jo.getBoolean("ageGradeNotice"), - coverUrl = "https://$staticDomain${jo.getString("thumbnail")}", - largeCoverUrl = if (jo.has("thumbnailVertical")) { - "https://$staticDomain${jo.getString("thumbnailVertical")}" - } else { null }, + rating = jo.getFloatOrDefault("starScoreAverage", -10f) / 10f, + isNsfw = jo.getBooleanOrDefault("ageGradeNotice", isNsfwSource), + coverUrl = jo.getString("thumbnail").toAbsoluteUrl(staticDomain), + largeCoverUrl = jo.getStringOrNull("thumbnailVertical")?.toAbsoluteUrl(staticDomain), tags = setOf(parseTag(jo.getJSONObject("genreInfo"))), - author = jo.getString("writingAuthorName"), + author = jo.getStringOrNull("writingAuthorName"), description = jo.getString("synopsis"), // I don't think the API provides this info state = null, - chapters = getChapters(titleNo), + chapters = chaptersDeferred.await(), source = source, ) } @@ -136,9 +142,7 @@ internal abstract class LineWebtoonsParser(context: MangaLoaderContext, source: SortOrder.UPDATED -> "UPDATE" SortOrder.POPULARITY -> "READ_COUNT" SortOrder.RATING -> "LIKEIT" - else -> { - throw IllegalArgumentException("Unsupported sort order: $sortOrder") - } + else -> throw IllegalArgumentException("Unsupported sort order: $sortOrder") } val manga = if (query != null) { @@ -146,7 +150,7 @@ internal abstract class LineWebtoonsParser(context: MangaLoaderContext, source: throw IllegalArgumentException("This source does not support search with tags") } - makeRequest("/lineWebtoon/webtoon/searchChallenge?query=${query.urlEncoded()}&startIndex=${offset+1}&pageSize=20") + makeRequest("/lineWebtoon/webtoon/searchChallenge?query=${query.urlEncoded()}&startIndex=${offset + 1}&pageSize=20") .getJSONObject("challengeSearch") .getJSONArray("titleList") .mapJSON { jo -> @@ -156,21 +160,22 @@ internal abstract class LineWebtoonsParser(context: MangaLoaderContext, source: id = generateUid(titleNo), title = jo.getString("title"), altTitle = null, - url = "$titleNo", + url = titleNo.toString(), publicUrl = "https://$domain/$languageCode/canvas/a/list?title_no=$titleNo", rating = RATING_UNKNOWN, - isNsfw = false, - coverUrl = "https://$staticDomain${jo.getString("thumbnail")}", + isNsfw = isNsfwSource, + coverUrl = jo.getString("thumbnail").toAbsoluteUrl(staticDomain), largeCoverUrl = null, - tags = setOf(), - author = jo.getString("writingAuthorName"), + tags = emptySet(), + author = jo.getStringOrNull("writingAuthorName"), description = null, state = null, source = source, ) } } else { - val result = makeRequest("/lineWebtoon/webtoon/challengeGenreTitleList.json?genre=$genre&sortOrder=$sortOrderStr&startIndex=${offset+1}&pageSize=20") + val result = + makeRequest("/lineWebtoon/webtoon/challengeGenreTitleList.json?genre=$genre&sortOrder=$sortOrderStr&startIndex=${offset + 1}&pageSize=20") val genres = result.getJSONObject("genreList") .getJSONArray("challengeGenres") @@ -187,14 +192,14 @@ internal abstract class LineWebtoonsParser(context: MangaLoaderContext, source: id = generateUid(titleNo), title = jo.getString("title"), altTitle = null, - url = "$titleNo", + url = titleNo.toString(), publicUrl = "https://$domain/$languageCode/canvas/a/list?title_no=$titleNo", - rating = jo.getDouble("starScoreAverage").toFloat() / 10f, - isNsfw = jo.getBoolean("ageGradeNotice"), - coverUrl = "https://$staticDomain${jo.getString("thumbnail")}", + rating = jo.getFloatOrDefault("starScoreAverage", -10f) / 10f, + isNsfw = jo.getBooleanOrDefault("ageGradeNotice", isNsfwSource), + coverUrl = jo.getString("thumbnail").toAbsoluteUrl(staticDomain), largeCoverUrl = jo.getStringOrNull("thumbnailVertical")?.toAbsoluteUrl(staticDomain), - tags = setOf(genres[jo.getString("representGenre")]!!), - author = jo.getString("writingAuthorName"), + tags = setOfNotNull(genres[jo.getString("representGenre")]), + author = jo.getStringOrNull("writingAuthorName"), description = jo.getString("synopsis"), // I don't think the API provides this info state = null, @@ -212,10 +217,10 @@ internal abstract class LineWebtoonsParser(context: MangaLoaderContext, source: return makeRequest("/lineWebtoon/webtoon/challengeEpisodeInfo.json?v=2&titleNo=$titleNo&episodeNo=$episodeNo") .getJSONObject("episodeInfo") .getJSONArray("imageInfo") - .mapJSONIndexed() { i, jo -> + .mapJSONIndexed { i, jo -> MangaPage( id = generateUid("$titleNo-$episodeNo-$i"), - url = "https://$staticDomain${jo.getString("url")}", + url = jo.getString("url"), preview = null, source = source, ) @@ -239,188 +244,75 @@ internal abstract class LineWebtoonsParser(context: MangaLoaderContext, source: private suspend fun makeRequest(url: String): JSONObject { val resp = webClient.httpGet(finalizeUrl(url)) - val message = resp.parseJson().getJSONObject("message") - if (resp.isSuccessful) { - return message.getJSONObject("result") - } else { - // TODO: handle 404 and some other error codes - val code = message.getInt("code") - val errorMessage = message.getString("message") - throw ParseException("Api error (code=$code): $errorMessage", url) + val message: JSONObject? = resp.parseJson().optJSONObject("message") + return when (resp.code) { + in 200..299 -> checkNotNull(message).getJSONObject("result") + 404 -> throw NotFoundException(message?.getStringOrNull("message").orEmpty(), url) + else -> { + val code = message?.getIntOrDefault("code", 0) + val errorMessage = message?.getStringOrNull("message") + throw ParseException("Api error (code=$code): $errorMessage", url) + } } } - private fun finalizeUrl(url: String): String { - val absoluteUrl = url.toAbsoluteUrl(apiDomain) - val parsedUrl = absoluteUrl.toHttpUrl() - val hasVersion = parsedUrl.queryParameter("v") != null - val hasQuery = parsedUrl.query != null - - val urlWithParams = absoluteUrl + if (hasQuery) { - "&" - } else { - "?" - } + "serviceZone=GLOBAL&" + if (!hasVersion) { - "v=1" - } else { "" } + "&language=$languageCode&locale=$languageCode&platform=APP_ANDROID" - - return signer.makeEncryptUrl(urlWithParams) + private fun finalizeUrl(url: String): HttpUrl { + val httpUrl = url.toAbsoluteUrl(apiDomain).toHttpUrl() + val builder = httpUrl.newBuilder() + .addQueryParameter("serviceZone", "GLOBAL") + if (httpUrl.queryParameter("v") == null) { + builder.addQueryParameter("v", "1") + } + builder.addQueryParameter("language", languageCode) + .addQueryParameter("locale", "languageCode") + .addQueryParameter("platform", "APP_ANDROID") + signer.makeEncryptUrl(builder) + return builder.build() } @MangaSourceParser("LINEWEBTOONS_EN", "Line Webtoons English", "en", type = ContentType.MANGA) class English(context: MangaLoaderContext) : LineWebtoonsParser(context, MangaSource.LINEWEBTOONS_EN) + @MangaSourceParser("LINEWEBTOONS_ZH", "Line Webtoons Chinese", "zh", type = ContentType.MANGA) class Chinese(context: MangaLoaderContext) : LineWebtoonsParser(context, MangaSource.LINEWEBTOONS_ZH) + @MangaSourceParser("LINEWEBTOONS_TH", "Line Webtoons Thai", "th", type = ContentType.MANGA) class Thai(context: MangaLoaderContext) : LineWebtoonsParser(context, MangaSource.LINEWEBTOONS_TH) + @MangaSourceParser("LINEWEBTOONS_ID", "Line Webtoons Indonesian", "id", type = ContentType.MANGA) class Indonesian(context: MangaLoaderContext) : LineWebtoonsParser(context, MangaSource.LINEWEBTOONS_ID) + @MangaSourceParser("LINEWEBTOONS_ES", "Line Webtoons Spanish", "es", type = ContentType.MANGA) class Spanish(context: MangaLoaderContext) : LineWebtoonsParser(context, MangaSource.LINEWEBTOONS_ES) + @MangaSourceParser("LINEWEBTOONS_FR", "Line Webtoons French", "fr", type = ContentType.MANGA) class French(context: MangaLoaderContext) : LineWebtoonsParser(context, MangaSource.LINEWEBTOONS_FR) + @MangaSourceParser("LINEWEBTOONS_DE", "Line Webtoons German", "de", type = ContentType.MANGA) class German(context: MangaLoaderContext) : LineWebtoonsParser(context, MangaSource.LINEWEBTOONS_DE) -} - + private inner class WebtoonsUrlSigner(private val secret: String) { -private class WebtoonsUrlSigner(val secret: String) { - private val mac = Mac.getInstance("HmacSHA1").apply { - this.init(SecretKeySpec(secret.encodeToByteArray(), "HmacSHA1")) - } - - private fun getMessage(url: String, msgpad: String): String { - return url.substring(0, 0xFF.coerceAtMost(url.length)) + msgpad - } - - private fun getMessageDigest(s: String): String { - var signedMessage: ByteArray - synchronized(mac) { signedMessage = mac.doFinal(s.toByteArray()) } - - // we don't use the context.encodeBase64 here because it adds newlines and doesn't add padding - // we, however, need padding and no newlines - return encodeBase64(signedMessage) - } - - - fun makeEncryptUrl(s: String): String { - return makeEncryptUrlCore( - s, - java.lang.String.valueOf(Calendar.getInstance().timeInMillis), - ) - } - - private fun makeEncryptUrlCore(url: String, msgpad: String): String { - val digest = URLEncoder.encode(getMessageDigest(getMessage(url, msgpad)), "utf-8") - return url + if (url.contains("?")) { - "&" - } else { - "?" - } + "msgpad=$msgpad&md=$digest" - } -} + private val mac = Mac.getInstance("HmacSHA1").apply { + this.init(SecretKeySpec(secret.encodeToByteArray(), "HmacSHA1")) + } + private fun getMessage(url: String, msgpad: String): String { + return url.substring(0, 0xFF.coerceAtMost(url.length)) + msgpad + } -private val INT_TO_BASE64: CharArray = charArrayOf( - 'A', - 'B', - 'C', - 'D', - 'E', - 'F', - 'G', - 'H', - 'I', - 'J', - 'K', - 'L', - 'M', - 'N', - 'O', - 'P', - 'Q', - 'R', - 'S', - 'T', - 'U', - 'V', - 'W', - 'X', - 'Y', - 'Z', - 'a', - 'b', - 'c', - 'd', - 'e', - 'f', - 'g', - 'h', - 'i', - 'j', - 'k', - 'l', - 'm', - 'n', - 'o', - 'p', - 'q', - 'r', - 's', - 't', - 'u', - 'v', - 'w', - 'x', - 'y', - 'z', - '0', - '1', - '2', - '3', - '4', - '5', - '6', - '7', - '8', - '9', - '+', - '/', -) - -private fun encodeBase64(arr: ByteArray): String { - val groupsCount = arr.size / 3 - val extraCount = arr.size - groupsCount * 3 - - val sb = StringBuffer((arr.size + 2) / 3 * 4) - val code = INT_TO_BASE64 - var groupIndex = 0 - var position = 0 - while (groupIndex < groupsCount) { - val v4 = arr[position].toInt() and 0xFF - val v5 = arr[position + 1].toInt() and 0xFF - val v6 = arr[position + 2].toInt() and 0xFF - sb.append(code[v4 shr 2]) - sb.append(code[v4 shl 4 and 0x3F or (v5 shr 4)]) - sb.append(code[v5 shl 2 and 0x3F or (v6 shr 6)]) - sb.append(code[v6 and 0x3F]) - ++groupIndex - position += 3 - } + private fun getMessageDigest(s: String): String { + val signedMessage = synchronized(mac) { mac.doFinal(s.toByteArray()) } + return context.encodeBase64(signedMessage) + } - if (extraCount != 0) { - val v1 = arr[position].toInt() and 0xFF - sb.append(code[v1 shr 2]) - if (extraCount == 1) { - sb.append(code[v1 shl 4 and 0x3F]) - sb.append("==") - } else { - val v2 = arr[position + 1].toInt() and 0xFF - sb.append(code[v1 shl 4 and 0x3F or (v2 shr 4)]) - sb.append(code[v2 shl 2 and 0x3F]) - sb.append('=') + fun makeEncryptUrl(urlBuilder: HttpUrl.Builder) { + val msgpad = Calendar.getInstance().timeInMillis.toString() + val digest = getMessageDigest(getMessage(urlBuilder.build().toString(), msgpad)) + urlBuilder + .addQueryParameter("msgpad", msgpad) + .addQueryParameter("md", digest) +// .addEncodedQueryParameter("md", digest.urlEncoded()) } } - return sb.toString() }