[LineWebtoons] Refactoring

src/main/kotlin/org/koitharu/kotatsu/parsers/site/en/LineWebtoonsParser.kt

@@ -1,27 +1,32 @@
 package org.koitharu.kotatsu.parsers.site.en
 
+import kotlinx.coroutines.async
+import kotlinx.coroutines.coroutineScope
 import okhttp3.Headers
+import okhttp3.HttpUrl
 import okhttp3.HttpUrl.Companion.toHttpUrl
 import org.json.JSONObject
 import org.koitharu.kotatsu.parsers.MangaLoaderContext
 import org.koitharu.kotatsu.parsers.MangaParser
 import org.koitharu.kotatsu.parsers.MangaSourceParser
 import org.koitharu.kotatsu.parsers.config.ConfigKey
+import org.koitharu.kotatsu.parsers.exception.NotFoundException
 import org.koitharu.kotatsu.parsers.exception.ParseException
 import org.koitharu.kotatsu.parsers.model.*
 import org.koitharu.kotatsu.parsers.util.*
-import org.koitharu.kotatsu.parsers.util.json.getStringOrNull
+import org.koitharu.kotatsu.parsers.util.json.*
-import org.koitharu.kotatsu.parsers.util.json.mapJSON
-import org.koitharu.kotatsu.parsers.util.json.mapJSONIndexed
-import org.koitharu.kotatsu.parsers.util.json.mapJSONToSet
-import org.koitharu.kotatsu.parsers.util.json.toJSONList
-import java.net.URLEncoder
 import java.util.*
 import javax.crypto.Mac
 import javax.crypto.spec.SecretKeySpec
 
-internal abstract class LineWebtoonsParser(context: MangaLoaderContext, source: MangaSource) : MangaParser(context, source)  {
+internal abstract class LineWebtoonsParser(
-	private val signer = WebtoonsUrlSigner("gUtPzJFZch4ZyAGviiyH94P99lQ3pFdRTwpJWDlSGFfwgpr6ses5ALOxWHOIT7R1")
+	context: MangaLoaderContext,
+	source: MangaSource,
+) : MangaParser(context, source) {
+
+	private val signer by lazy {
+		WebtoonsUrlSigner("gUtPzJFZch4ZyAGviiyH94P99lQ3pFdRTwpJWDlSGFfwgpr6ses5ALOxWHOIT7R1")
+	}
 
 	// we don't __really__ support changing this domain because:
 	// 1. I don't think other websites have this exact API
@@ -29,8 +34,7 @@ internal abstract class LineWebtoonsParser(context: MangaLoaderContext, source:
 	// 3. we rely on the HTTP client setting the referer header to webtoons.com
 	//
 	// This effectively means that changing the domain will break the source. Yikes
-	override val configKeyDomain
+	override val configKeyDomain = ConfigKey.Domain("webtoons.com")
-		get() = ConfigKey.Domain("webtoons.com")
 
 	private val apiDomain = "global.apis.naver.com"
 	private val staticDomain = "webtoon-phinf.pstatic.net"
@@ -48,7 +52,7 @@ internal abstract class LineWebtoonsParser(context: MangaLoaderContext, source:
 			.build()
 
 	override suspend fun getPageUrl(page: MangaPage): String {
-		return page.url
+		return page.url.toAbsoluteUrl(staticDomain)
 	}
 
 	// some language tags do not map perfectly to the ones used by the API
@@ -60,7 +64,9 @@ internal abstract class LineWebtoonsParser(context: MangaLoaderContext, source:
 		}
 
 	private suspend fun getChapters(titleNo: Long): List<MangaChapter> {
-		val firstResult = makeRequest("/lineWebtoon/webtoon/challengeEpisodeList.json?v=2&titleNo=$titleNo&startIndex=0&pageSize=30")
+		val firstResult = makeRequest(
+			url = "/lineWebtoon/webtoon/challengeEpisodeList.json?v=2&titleNo=$titleNo&startIndex=0&pageSize=30",
+		)
 
 		val totalEpisodeCount = firstResult
 			.getJSONObject("episodeList")
@@ -73,20 +79,21 @@ internal abstract class LineWebtoonsParser(context: MangaLoaderContext, source:
 			.toMutableList()
 
 		while (episodes.count() < totalEpisodeCount) {
-			val page = makeRequest("/lineWebtoon/webtoon/challengeEpisodeList.json?v=2&titleNo=$titleNo&startIndex=${episodes.count()}&pageSize=30")
+			val page = makeRequest(
-				.getJSONObject("episodeList")
+				url = "/lineWebtoon/webtoon/challengeEpisodeList.json?v=2&titleNo=$titleNo&startIndex=${episodes.count()}&pageSize=30",
+			).getJSONObject("episodeList")
 				.getJSONArray("episode")
 				.toJSONList()
 
 			episodes.addAll(page)
 		}
 
-		return episodes.mapIndexed { i, jo ->
+		return episodes.mapChapters { i, jo ->
 			MangaChapter(
 				id = generateUid("$titleNo-$i"),
 				name = jo.getString("episodeTitle"),
 				number = jo.getInt("episodeSeq"),
-				url = "$titleNo-${jo.getString("episodeNo")}",
+				url = "$titleNo-${jo.get("episodeNo")}",
 				uploadDate = jo.getLong("modifyYmdt"),
 				branch = null,
 				scanlator = null,
@@ -95,10 +102,11 @@ internal abstract class LineWebtoonsParser(context: MangaLoaderContext, source:
 		}.sortedBy { it.number }
 	}
 
-	override suspend fun getDetails(manga: Manga): Manga {
+	override suspend fun getDetails(manga: Manga): Manga = coroutineScope {
 		val titleNo = manga.url.toLong()
+		val chaptersDeferred = async { getChapters(titleNo) }
 
-		return makeRequest("/lineWebtoon/webtoon/challengeTitleInfo.json?v=2&titleNo=${titleNo}")
+		makeRequest("/lineWebtoon/webtoon/challengeTitleInfo.json?v=2&titleNo=${titleNo}")
 			.getJSONObject("titleInfo")
 			.let { jo ->
 				Manga(
@@ -107,18 +115,16 @@ internal abstract class LineWebtoonsParser(context: MangaLoaderContext, source:
 					altTitle = null,
 					url = "$titleNo",
 					publicUrl = "https://$domain/$languageCode/canvas/a/list?title_no=${titleNo}",
-					rating = jo.getDouble("starScoreAverage").toFloat() / 10f,
+					rating = jo.getFloatOrDefault("starScoreAverage", -10f) / 10f,
-					isNsfw = jo.getBoolean("ageGradeNotice"),
+					isNsfw = jo.getBooleanOrDefault("ageGradeNotice", isNsfwSource),
-					coverUrl = "https://$staticDomain${jo.getString("thumbnail")}",
+					coverUrl = jo.getString("thumbnail").toAbsoluteUrl(staticDomain),
-					largeCoverUrl = if (jo.has("thumbnailVertical")) {
+					largeCoverUrl = jo.getStringOrNull("thumbnailVertical")?.toAbsoluteUrl(staticDomain),
-						"https://$staticDomain${jo.getString("thumbnailVertical")}"
-					} else { null },
 					tags = setOf(parseTag(jo.getJSONObject("genreInfo"))),
-					author = jo.getString("writingAuthorName"),
+					author = jo.getStringOrNull("writingAuthorName"),
 					description = jo.getString("synopsis"),
 					// I don't think the API provides this info
 					state = null,
-					chapters = getChapters(titleNo),
+					chapters = chaptersDeferred.await(),
 					source = source,
 				)
 			}
@@ -136,9 +142,7 @@ internal abstract class LineWebtoonsParser(context: MangaLoaderContext, source:
 			SortOrder.UPDATED -> "UPDATE"
 			SortOrder.POPULARITY -> "READ_COUNT"
 			SortOrder.RATING -> "LIKEIT"
-			else -> {
+			else -> throw IllegalArgumentException("Unsupported sort order: $sortOrder")
-				throw IllegalArgumentException("Unsupported sort order: $sortOrder")
-			}
 		}
 
 		val manga = if (query != null) {
@@ -146,7 +150,7 @@ internal abstract class LineWebtoonsParser(context: MangaLoaderContext, source:
 				throw IllegalArgumentException("This source does not support search with tags")
 			}
 
-			makeRequest("/lineWebtoon/webtoon/searchChallenge?query=${query.urlEncoded()}&startIndex=${offset+1}&pageSize=20")
+			makeRequest("/lineWebtoon/webtoon/searchChallenge?query=${query.urlEncoded()}&startIndex=${offset + 1}&pageSize=20")
 				.getJSONObject("challengeSearch")
 				.getJSONArray("titleList")
 				.mapJSON { jo ->
@@ -156,21 +160,22 @@ internal abstract class LineWebtoonsParser(context: MangaLoaderContext, source:
 						id = generateUid(titleNo),
 						title = jo.getString("title"),
 						altTitle = null,
-						url = "$titleNo",
+						url = titleNo.toString(),
 						publicUrl = "https://$domain/$languageCode/canvas/a/list?title_no=$titleNo",
 						rating = RATING_UNKNOWN,
-						isNsfw = false,
+						isNsfw = isNsfwSource,
-						coverUrl = "https://$staticDomain${jo.getString("thumbnail")}",
+						coverUrl = jo.getString("thumbnail").toAbsoluteUrl(staticDomain),
 						largeCoverUrl = null,
-						tags = setOf(),
+						tags = emptySet(),
-						author = jo.getString("writingAuthorName"),
+						author = jo.getStringOrNull("writingAuthorName"),
 						description = null,
 						state = null,
 						source = source,
 					)
 				}
 		} else {
-			val result = makeRequest("/lineWebtoon/webtoon/challengeGenreTitleList.json?genre=$genre&sortOrder=$sortOrderStr&startIndex=${offset+1}&pageSize=20")
+			val result =
+				makeRequest("/lineWebtoon/webtoon/challengeGenreTitleList.json?genre=$genre&sortOrder=$sortOrderStr&startIndex=${offset + 1}&pageSize=20")
 
 			val genres = result.getJSONObject("genreList")
 				.getJSONArray("challengeGenres")
@@ -187,14 +192,14 @@ internal abstract class LineWebtoonsParser(context: MangaLoaderContext, source:
 						id = generateUid(titleNo),
 						title = jo.getString("title"),
 						altTitle = null,
-						url = "$titleNo",
+						url = titleNo.toString(),
 						publicUrl = "https://$domain/$languageCode/canvas/a/list?title_no=$titleNo",
-						rating = jo.getDouble("starScoreAverage").toFloat() / 10f,
+						rating = jo.getFloatOrDefault("starScoreAverage", -10f) / 10f,
-						isNsfw = jo.getBoolean("ageGradeNotice"),
+						isNsfw = jo.getBooleanOrDefault("ageGradeNotice", isNsfwSource),
-						coverUrl = "https://$staticDomain${jo.getString("thumbnail")}",
+						coverUrl = jo.getString("thumbnail").toAbsoluteUrl(staticDomain),
 						largeCoverUrl = jo.getStringOrNull("thumbnailVertical")?.toAbsoluteUrl(staticDomain),
-						tags = setOf(genres[jo.getString("representGenre")]!!),
+						tags = setOfNotNull(genres[jo.getString("representGenre")]),
-						author = jo.getString("writingAuthorName"),
+						author = jo.getStringOrNull("writingAuthorName"),
 						description = jo.getString("synopsis"),
 						// I don't think the API provides this info
 						state = null,
@@ -212,10 +217,10 @@ internal abstract class LineWebtoonsParser(context: MangaLoaderContext, source:
 		return makeRequest("/lineWebtoon/webtoon/challengeEpisodeInfo.json?v=2&titleNo=$titleNo&episodeNo=$episodeNo")
 			.getJSONObject("episodeInfo")
 			.getJSONArray("imageInfo")
-			.mapJSONIndexed() { i, jo ->
+			.mapJSONIndexed { i, jo ->
 				MangaPage(
 					id = generateUid("$titleNo-$episodeNo-$i"),
-					url = "https://$staticDomain${jo.getString("url")}",
+					url = jo.getString("url"),
 					preview = null,
 					source = source,
 				)
@@ -239,188 +244,75 @@ internal abstract class LineWebtoonsParser(context: MangaLoaderContext, source:
 
 	private suspend fun makeRequest(url: String): JSONObject {
 		val resp = webClient.httpGet(finalizeUrl(url))
-		val message = resp.parseJson().getJSONObject("message")
+		val message: JSONObject? = resp.parseJson().optJSONObject("message")
-		if (resp.isSuccessful) {
+		return when (resp.code) {
-			return message.getJSONObject("result")
+			in 200..299 -> checkNotNull(message).getJSONObject("result")
-		} else {
+			404 -> throw NotFoundException(message?.getStringOrNull("message").orEmpty(), url)
-			// TODO: handle 404 and some other error codes
+			else -> {
-			val code = message.getInt("code")
+				val code = message?.getIntOrDefault("code", 0)
-			val errorMessage = message.getString("message")
+				val errorMessage = message?.getStringOrNull("message")
-			throw ParseException("Api error (code=$code): $errorMessage", url)
+				throw ParseException("Api error (code=$code): $errorMessage", url)
+			}
 		}
 	}
 
-	private fun finalizeUrl(url: String): String {
+	private fun finalizeUrl(url: String): HttpUrl {
-		val absoluteUrl = url.toAbsoluteUrl(apiDomain)
+		val httpUrl = url.toAbsoluteUrl(apiDomain).toHttpUrl()
-		val parsedUrl = absoluteUrl.toHttpUrl()
+		val builder = httpUrl.newBuilder()
-		val hasVersion = parsedUrl.queryParameter("v") != null
+			.addQueryParameter("serviceZone", "GLOBAL")
-		val hasQuery = parsedUrl.query != null
+		if (httpUrl.queryParameter("v") == null) {
-
+			builder.addQueryParameter("v", "1")
-		val urlWithParams = absoluteUrl + if (hasQuery) {
+		}
-			"&"
+		builder.addQueryParameter("language", languageCode)
-		} else {
+			.addQueryParameter("locale", "languageCode")
-			"?"
+			.addQueryParameter("platform", "APP_ANDROID")
-		} + "serviceZone=GLOBAL&" + if (!hasVersion) {
+		signer.makeEncryptUrl(builder)
-			"v=1"
+		return builder.build()
-		} else { "" } + "&language=$languageCode&locale=$languageCode&platform=APP_ANDROID"
-
-		return signer.makeEncryptUrl(urlWithParams)
 	}
 
 	@MangaSourceParser("LINEWEBTOONS_EN", "Line Webtoons English", "en", type = ContentType.MANGA)
 	class English(context: MangaLoaderContext) : LineWebtoonsParser(context, MangaSource.LINEWEBTOONS_EN)
+
 	@MangaSourceParser("LINEWEBTOONS_ZH", "Line Webtoons Chinese", "zh", type = ContentType.MANGA)
 	class Chinese(context: MangaLoaderContext) : LineWebtoonsParser(context, MangaSource.LINEWEBTOONS_ZH)
+
 	@MangaSourceParser("LINEWEBTOONS_TH", "Line Webtoons Thai", "th", type = ContentType.MANGA)
 	class Thai(context: MangaLoaderContext) : LineWebtoonsParser(context, MangaSource.LINEWEBTOONS_TH)
+
 	@MangaSourceParser("LINEWEBTOONS_ID", "Line Webtoons Indonesian", "id", type = ContentType.MANGA)
 	class Indonesian(context: MangaLoaderContext) : LineWebtoonsParser(context, MangaSource.LINEWEBTOONS_ID)
+
 	@MangaSourceParser("LINEWEBTOONS_ES", "Line Webtoons Spanish", "es", type = ContentType.MANGA)
 	class Spanish(context: MangaLoaderContext) : LineWebtoonsParser(context, MangaSource.LINEWEBTOONS_ES)
+
 	@MangaSourceParser("LINEWEBTOONS_FR", "Line Webtoons French", "fr", type = ContentType.MANGA)
 	class French(context: MangaLoaderContext) : LineWebtoonsParser(context, MangaSource.LINEWEBTOONS_FR)
+
 	@MangaSourceParser("LINEWEBTOONS_DE", "Line Webtoons German", "de", type = ContentType.MANGA)
 	class German(context: MangaLoaderContext) : LineWebtoonsParser(context, MangaSource.LINEWEBTOONS_DE)
 
-}
+	private inner class WebtoonsUrlSigner(private val secret: String) {
-
 
-private class WebtoonsUrlSigner(val secret: String) {
+		private val mac = Mac.getInstance("HmacSHA1").apply {
-	private val mac = Mac.getInstance("HmacSHA1").apply {
+			this.init(SecretKeySpec(secret.encodeToByteArray(), "HmacSHA1"))
-		this.init(SecretKeySpec(secret.encodeToByteArray(), "HmacSHA1"))
+		}
-	}
-
-	private fun getMessage(url: String, msgpad: String): String {
-		return url.substring(0, 0xFF.coerceAtMost(url.length)) + msgpad
-	}
-
-	private fun getMessageDigest(s: String): String {
-		var signedMessage: ByteArray
-		synchronized(mac) { signedMessage = mac.doFinal(s.toByteArray()) }
-
-		// we don't use the context.encodeBase64 here because it adds newlines and doesn't add padding
-		// we, however, need padding and no newlines
-		return encodeBase64(signedMessage)
-	}
-
-
-	fun makeEncryptUrl(s: String): String {
-		return makeEncryptUrlCore(
-			s,
-			java.lang.String.valueOf(Calendar.getInstance().timeInMillis),
-		)
-	}
-
-	private fun makeEncryptUrlCore(url: String, msgpad: String): String {
-		val digest = URLEncoder.encode(getMessageDigest(getMessage(url, msgpad)), "utf-8")
-		return url + if (url.contains("?")) {
-			"&"
-		} else {
-			"?"
-		} + "msgpad=$msgpad&md=$digest"
-	}
-}
 
+		private fun getMessage(url: String, msgpad: String): String {
+			return url.substring(0, 0xFF.coerceAtMost(url.length)) + msgpad
+		}
 
-private val INT_TO_BASE64: CharArray = charArrayOf(
+		private fun getMessageDigest(s: String): String {
-	'A',
+			val signedMessage = synchronized(mac) { mac.doFinal(s.toByteArray()) }
-	'B',
+			return context.encodeBase64(signedMessage)
-	'C',
+		}
-	'D',
-	'E',
-	'F',
-	'G',
-	'H',
-	'I',
-	'J',
-	'K',
-	'L',
-	'M',
-	'N',
-	'O',
-	'P',
-	'Q',
-	'R',
-	'S',
-	'T',
-	'U',
-	'V',
-	'W',
-	'X',
-	'Y',
-	'Z',
-	'a',
-	'b',
-	'c',
-	'd',
-	'e',
-	'f',
-	'g',
-	'h',
-	'i',
-	'j',
-	'k',
-	'l',
-	'm',
-	'n',
-	'o',
-	'p',
-	'q',
-	'r',
-	's',
-	't',
-	'u',
-	'v',
-	'w',
-	'x',
-	'y',
-	'z',
-	'0',
-	'1',
-	'2',
-	'3',
-	'4',
-	'5',
-	'6',
-	'7',
-	'8',
-	'9',
-	'+',
-	'/',
-)
-
-private fun encodeBase64(arr: ByteArray): String {
-	val groupsCount = arr.size / 3
-	val extraCount = arr.size - groupsCount * 3
-
-	val sb = StringBuffer((arr.size + 2) / 3 * 4)
-	val code = INT_TO_BASE64
-	var groupIndex = 0
-	var position = 0
-	while (groupIndex < groupsCount) {
-		val v4 = arr[position].toInt() and 0xFF
-		val v5 = arr[position + 1].toInt() and 0xFF
-		val v6 = arr[position + 2].toInt() and 0xFF
-		sb.append(code[v4 shr 2])
-		sb.append(code[v4 shl 4 and 0x3F or (v5 shr 4)])
-		sb.append(code[v5 shl 2 and 0x3F or (v6 shr 6)])
-		sb.append(code[v6 and 0x3F])
-		++groupIndex
-		position += 3
-	}
 
-	if (extraCount != 0) {
+		fun makeEncryptUrl(urlBuilder: HttpUrl.Builder) {
-		val v1 = arr[position].toInt() and 0xFF
+			val msgpad = Calendar.getInstance().timeInMillis.toString()
-		sb.append(code[v1 shr 2])
+			val digest = getMessageDigest(getMessage(urlBuilder.build().toString(), msgpad))
-		if (extraCount == 1) {
+			urlBuilder
-			sb.append(code[v1 shl 4 and 0x3F])
+				.addQueryParameter("msgpad", msgpad)
-			sb.append("==")
+				.addQueryParameter("md", digest)
-		} else {
+//				.addEncodedQueryParameter("md", digest.urlEncoded())
-			val v2 = arr[position + 1].toInt() and 0xFF
-			sb.append(code[v1 shl 4 and 0x3F or (v2 shr 4)])
-			sb.append(code[v2 shl 2 and 0x3F])
-			sb.append('=')
 		}
 	}
-	return sb.toString()
 }