From 1330feaea5b4c3337f42a15a6ae0f3d7695f0ff1 Mon Sep 17 00:00:00 2001 From: Naga <94557604+NagaYZ@users.noreply.github.com> Date: Sat, 24 Feb 2024 23:32:37 +0100 Subject: [PATCH 01/18] added webtoon original english support --- .../koitharu/kotatsu/parsers/model/Manga.kt | 19 +- .../parsers/site/all/WebtoonsParser.kt | 334 ++++++++++++++++++ 2 files changed, 351 insertions(+), 2 deletions(-) create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/model/Manga.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/model/Manga.kt index afda6dc86..76f030f01 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/model/Manga.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/model/Manga.kt @@ -67,7 +67,14 @@ class Manga( * Manga source */ @JvmField val source: MangaSource, -) { + + @JvmField val date: Long? = null, + + @JvmField val readCount: Long? = null, + + @JvmField val likeCount: Long? = null, + + ) { /** * Return if manga has a specified rating @@ -110,6 +117,9 @@ class Manga( description = description, chapters = chapters, source = source, + date = date, + readCount = readCount, + likeCount = likeCount, ) override fun equals(other: Any?): Boolean { @@ -133,7 +143,9 @@ class Manga( if (description != other.description) return false if (chapters != other.chapters) return false if (source != other.source) return false - + if (date != other.date) return false + if (readCount != other.readCount) return false + if (likeCount != other.likeCount) return false return true } @@ -153,6 +165,9 @@ class Manga( result = 31 * result + (description?.hashCode() ?: 0) result = 31 * result + (chapters?.hashCode() ?: 0) result = 31 * result + source.hashCode() + result = 31 * result + (date?.hashCode() ?: 0) + result = 31 * result + (readCount?.hashCode() ?: 0) + result = 31 * result + (likeCount?.hashCode() ?: 0) return result } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt new file mode 100644 index 000000000..870f6a066 --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt @@ -0,0 +1,334 @@ +package org.koitharu.kotatsu.parsers.site.all + +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope +import okhttp3.Headers +import okhttp3.HttpUrl +import okhttp3.HttpUrl.Companion.toHttpUrl +import org.json.JSONObject +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaParser +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.config.ConfigKey +import org.koitharu.kotatsu.parsers.exception.NotFoundException +import org.koitharu.kotatsu.parsers.exception.ParseException +import org.koitharu.kotatsu.parsers.model.* +import org.koitharu.kotatsu.parsers.util.* +import org.koitharu.kotatsu.parsers.util.json.* +import java.util.* +import java.util.logging.Logger.global +import javax.crypto.Mac +import javax.crypto.spec.SecretKeySpec + +internal abstract class WebtoonsParser( + context: MangaLoaderContext, + source: MangaSource, +) : MangaParser(context, source) { + + override val isMultipleTagsSupported = false + + private val signer by lazy { + WebtoonsUrlSigner("gUtPzJFZch4ZyAGviiyH94P99lQ3pFdRTwpJWDlSGFfwgpr6ses5ALOxWHOIT7R1") + } + + // we don't __really__ support changing this domain because: + // 1. I don't think other websites have this exact API + // 2. most communication is done with other domains (hosting API and static content), which are not configurable + // 3. we rely on the HTTP client setting the referer header to webtoons.com + // + // This effectively means that changing the domain will break the source. Yikes + override val configKeyDomain = ConfigKey.Domain("webtoons.com") + + private val apiDomain = "global.apis.naver.com" + private val staticDomain = "webtoon-phinf.pstatic.net" + + override val availableSortOrders: Set = EnumSet.of( + SortOrder.POPULARITY, // views + SortOrder.RATING, // star rating + //SortOrder.LIKE, // likes + SortOrder.UPDATED + ) + override val headers: Headers + get() = Headers.Builder() + .add("User-Agent", "nApps (Android 12;; linewebtoon; 3.1.0)") + .build() + + override suspend fun getPageUrl(page: MangaPage): String { + return page.url.toAbsoluteUrl(staticDomain) + } + + // some language tags do not map perfectly to the ones used by the API + private val languageCode: String + get() = when (val tag = sourceLocale.toLanguageTag()) { + "in" -> "id" + "zh" -> "zh-hant" + else -> tag + } + + private suspend fun getChapters(titleNo: Long): List { + val firstResult = makeRequest( + url = "/lineWebtoon/webtoon/episodeList.json?v=5&titleNo=$titleNo&startIndex=0&pageSize=30", + ) + + val totalEpisodeCount = firstResult + .getJSONObject("episodeList") + .getInt("totalServiceEpisodeCount") + + val episodes = firstResult + .getJSONObject("episodeList") + .getJSONArray("episode") + .toJSONList() + .toMutableList() + + while (episodes.count() < totalEpisodeCount) { + val page = makeRequest( + url = "/lineWebtoon/webtoon/episodeList.json?v=5&titleNo=$titleNo&startIndex=${episodes.count()}&pageSize=30", + ).getJSONObject("episodeList") + .getJSONArray("episode") + .toJSONList() + + episodes.addAll(page) + } + + return episodes.mapChapters { i, jo -> + MangaChapter( + id = generateUid("$titleNo-$i"), + name = jo.getString("episodeTitle"), + number = jo.getInt("episodeSeq"), + url = "$titleNo-${jo.get("episodeNo")}", + uploadDate = jo.getLong("modifyYmdt"), + branch = null, + scanlator = null, + source = source, + ) + }.sortedBy { it.number } + } + + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { + val titleNo = manga.url.toLong() + val chaptersDeferred = async { getChapters(titleNo) } + + makeRequest("/lineWebtoon/webtoon/titleInfo.json?titleNo=${titleNo}&anyServiceStatus=false") + .getJSONObject("titleInfo") + .let { jo -> + Manga( + id = generateUid(titleNo), + title = jo.getString("title"), + altTitle = null, + url = "$titleNo", + publicUrl = "https://$domain/$languageCode/originals/a/list?title_no=${titleNo}", + rating = jo.getFloatOrDefault("starScoreAverage", -10f) / 10f, + isNsfw = jo.getBooleanOrDefault("ageGradeNotice", isNsfwSource), + coverUrl = jo.getString("thumbnail").toAbsoluteUrl(staticDomain), + largeCoverUrl = jo.getStringOrNull("thumbnailVertical")?.toAbsoluteUrl(staticDomain), + tags = setOf(parseTag(jo.getJSONObject("genreInfo"))), + author = jo.getStringOrNull("writingAuthorName"), + description = jo.getString("synopsis"), + // I don't think the API provides this info + state = null, + chapters = chaptersDeferred.await(), + source = source, + ) + } + } + + override suspend fun getList(offset: Int, filter: MangaListFilter?): List { + + val manga = + when (filter) { + is MangaListFilter.Search -> { + makeRequest("/lineWebtoon/webtoon/searchWebtoon?query=${filter.query.urlEncoded()}&startIndex=${offset + 1}&pageSize=20") + .getJSONObject("webtoonSearch") + .getJSONArray("titleList") + .mapJSON { jo -> + val titleNo = jo.getLong("titleNo") + Manga( + id = generateUid(titleNo), + title = jo.getString("title"), + altTitle = null, + url = titleNo.toString(), + publicUrl = "https://$domain/$languageCode/originals/a/list?title_no=$titleNo", + rating = RATING_UNKNOWN, + isNsfw = isNsfwSource, + coverUrl = jo.getString("thumbnail").toAbsoluteUrl(staticDomain), + largeCoverUrl = null, + tags = emptySet(), + author = jo.getStringOrNull("writingAuthorName"), + description = null, + state = null, + source = source, + ) + } + } + + is MangaListFilter.Advanced -> { + val genre = filter.tags.oneOrThrowIfMany()?.key ?: "ALL" + + val genres = makeRequest("/lineWebtoon/webtoon/genreList.json") + .getJSONObject("genreList") + .getJSONArray("genres") + .mapJSON { jo -> parseTag(jo) } + .associateBy { tag -> tag.key } + + val result = makeRequest("/lineWebtoon/webtoon/titleList.json") + .getJSONObject("titleList") + .getJSONArray("titles") + .mapJSON { jo -> + val titleNo = jo.getLong("titleNo") + Manga( + id = generateUid(titleNo), + title = jo.getString("title"), + altTitle = null, + url = titleNo.toString(), + publicUrl = "https://$domain/$languageCode/originals/a/list?title_no=$titleNo", + rating = jo.getFloatOrDefault("starScoreAverage", -10f) / 10f, + isNsfw = jo.getBooleanOrDefault("ageGradeNotice", isNsfwSource), + coverUrl = jo.getString("thumbnail").toAbsoluteUrl(staticDomain), + largeCoverUrl = jo.getStringOrNull("thumbnailVertical")?.toAbsoluteUrl(staticDomain), + tags = setOfNotNull(genres[jo.getString("representGenre")]), + author = jo.getStringOrNull("writingAuthorName"), + description = jo.getString("synopsis"), + // I don't think the API provides this info + state = null, + source = source, + date = jo.getLong("lastEpisodeRegisterYmdt"), + readCount = jo.getLong("readCount"), + likeCount = jo.getLong("likeitCount") + ) + } + + val sortedResult = when (filter.sortOrder) { + SortOrder.UPDATED -> result.sortedBy { it.date } + SortOrder.POPULARITY -> result.sortedBy { it.readCount } + SortOrder.RATING -> result.sortedBy { it.rating } + //SortOrder.LIKE -> result.sortedBy { it.likeCount } + else -> throw IllegalArgumentException("Unsupported sort order: ${filter.sortOrder}") + } + + if (genre != "ALL") { + sortedResult.filter { it.tags.contains(genres[genre]) } + } else { + sortedResult + } + } + + null -> { + makeRequest("/lineWebtoon/webtoon/titleList.json") + .getJSONObject("titleList") + .getJSONArray("titles") + .mapJSON { jo -> + val titleNo = jo.getLong("titleNo") + Manga( + id = generateUid(titleNo), + title = jo.getString("title"), + altTitle = null, + url = titleNo.toString(), + publicUrl = "https://$domain/$languageCode/originals/a/list?title_no=$titleNo", + rating = jo.getFloatOrDefault("starScoreAverage", -10f) / 10f, + isNsfw = jo.getBooleanOrDefault("ageGradeNotice", isNsfwSource), + coverUrl = jo.getString("thumbnail").toAbsoluteUrl(staticDomain), + largeCoverUrl = jo.getStringOrNull("thumbnailVertical")?.toAbsoluteUrl(staticDomain), + tags = emptySet(), + author = jo.getStringOrNull("writingAuthorName"), + description = jo.getString("synopsis"), + // I don't think the API provides this info + state = null, + source = source, + date = jo.getLong("lastEpisodeRegisterYmdt"), + readCount = jo.getLong("readCount"), + likeCount = jo.getLong("likeitCount") + ) + } + } + } + + return manga + } + + override suspend fun getPages(chapter: MangaChapter): List { + val (titleNo, episodeNo) = requireNotNull(chapter.url.splitTwoParts('-')) + return makeRequest("/lineWebtoon/webtoon/episodeInfo.json?v=4&titleNo=$titleNo&episodeNo=$episodeNo") + .getJSONObject("episodeInfo") + .getJSONArray("imageInfo") + .mapJSONIndexed { i, jo -> + MangaPage( + id = generateUid("$titleNo-$episodeNo-$i"), + url = jo.getString("url"), + preview = null, + source = source, + ) + } + } + + private fun parseTag(jo: JSONObject): MangaTag { + return MangaTag( + title = jo.getString("name"), + key = jo.getString("code"), + source = source, + ) + } + + override suspend fun getAvailableTags(): Set { + return makeRequest("/lineWebtoon/webtoon/genreList.json") + .getJSONObject("genreList") + .getJSONArray("genres") + .mapJSONToSet { jo -> parseTag(jo) } + } + + private suspend fun makeRequest(url: String): JSONObject { + val resp = webClient.httpGet(finalizeUrl(url)) + val message: JSONObject? = resp.parseJson().optJSONObject("message") + return when (resp.code) { + in 200..299 -> checkNotNull(message).getJSONObject("result") + 404 -> throw NotFoundException(message?.getStringOrNull("message").orEmpty(), url) + else -> { + val code = message?.getIntOrDefault("code", 0) + val errorMessage = message?.getStringOrNull("message") + throw ParseException("Api error (code=$code): $errorMessage", url) + } + } + } + + private fun finalizeUrl(url: String): HttpUrl { + val httpUrl = url.toAbsoluteUrl(apiDomain).toHttpUrl() + val builder = httpUrl.newBuilder() + .addQueryParameter("serviceZone", "GLOBAL") + if (httpUrl.queryParameter("v") == null) { + builder.addQueryParameter("v", "1") + } + builder.addQueryParameter("language", languageCode) + .addQueryParameter("locale", "languageCode") + .addQueryParameter("platform", "APP_ANDROID") + signer.makeEncryptUrl(builder) + return builder.build() + } + + @MangaSourceParser("WEBTOONS_EN", "Webtoons English", "en", type = ContentType.MANGA) + class English(context: MangaLoaderContext) : WebtoonsParser(context, MangaSource.WEBTOONS_EN) + + + private inner class WebtoonsUrlSigner(private val secret: String) { + + private val mac = Mac.getInstance("HmacSHA1").apply { + this.init(SecretKeySpec(secret.encodeToByteArray(), "HmacSHA1")) + } + + private fun getMessage(url: String, msgpad: String): String { + return url.substring(0, 0xFF.coerceAtMost(url.length)) + msgpad + } + + private fun getMessageDigest(s: String): String { + val signedMessage = synchronized(mac) { mac.doFinal(s.toByteArray()) } + return context.encodeBase64(signedMessage) + } + + fun makeEncryptUrl(urlBuilder: HttpUrl.Builder) { + val msgPad = Calendar.getInstance().timeInMillis.toString() + val digest = getMessageDigest(getMessage(urlBuilder.build().toString(), msgPad)) + urlBuilder + .addQueryParameter("msgpad", msgPad) + .addQueryParameter("md", digest) +// .addEncodedQueryParameter("md", digest.urlEncoded()) + } + } +} From a7ed4fbcc34f7f760329131af66b6d3ea28a14b0 Mon Sep 17 00:00:00 2001 From: Naga <94557604+NagaYZ@users.noreply.github.com> Date: Sat, 24 Feb 2024 23:33:10 +0100 Subject: [PATCH 02/18] fix import --- .../org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt index 870f6a066..c62fa8fa7 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt @@ -16,7 +16,6 @@ import org.koitharu.kotatsu.parsers.model.* import org.koitharu.kotatsu.parsers.util.* import org.koitharu.kotatsu.parsers.util.json.* import java.util.* -import java.util.logging.Logger.global import javax.crypto.Mac import javax.crypto.spec.SecretKeySpec From ada1b7b54c873dc45f79dfe462e6362386665ed6 Mon Sep 17 00:00:00 2001 From: Naga <94557604+NagaYZ@users.noreply.github.com> Date: Sun, 25 Feb 2024 00:38:17 +0100 Subject: [PATCH 03/18] fixed sort descending --- .../org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt index c62fa8fa7..75040594f 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt @@ -198,8 +198,8 @@ internal abstract class WebtoonsParser( val sortedResult = when (filter.sortOrder) { SortOrder.UPDATED -> result.sortedBy { it.date } - SortOrder.POPULARITY -> result.sortedBy { it.readCount } - SortOrder.RATING -> result.sortedBy { it.rating } + SortOrder.POPULARITY -> result.sortedByDescending { it.readCount } + SortOrder.RATING -> result.sortedByDescending { it.rating } //SortOrder.LIKE -> result.sortedBy { it.likeCount } else -> throw IllegalArgumentException("Unsupported sort order: ${filter.sortOrder}") } From 3c08119ccc5a14cdaeca612c07823825139fe050 Mon Sep 17 00:00:00 2001 From: Naga <94557604+NagaYZ@users.noreply.github.com> Date: Sun, 25 Feb 2024 03:12:12 +0100 Subject: [PATCH 04/18] added caching for title list to speed up loading --- .../parsers/site/all/WebtoonsParser.kt | 111 ++++++++---------- 1 file changed, 52 insertions(+), 59 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt index 75040594f..ab90edf15 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt @@ -131,12 +131,60 @@ internal abstract class WebtoonsParser( } } + private val allTitleCache = SuspendLazy { + makeRequest("/lineWebtoon/webtoon/titleList.json?") + .getJSONObject("titleList") + .getJSONArray("titles") + .toJSONList() + } + + private val allGenreCache = SuspendLazy { + makeRequest("/lineWebtoon/webtoon/genreList.json") + .getJSONObject("genreList") + .getJSONArray("genres") + .mapJSON { jo -> parseTag(jo) } + .associateBy { tag -> tag.key } + } + + private suspend fun getAllGenreList(): Map{ + return allGenreCache.get() + } + + private suspend fun getAllTitleList(): List { + val genres = getAllGenreList() + + return allTitleCache.get().map { jo -> + val titleNo = jo.getLong("titleNo") + Manga( + id = generateUid(titleNo), + url = titleNo.toString(), + publicUrl = "https://$domain/$languageCode/originals/a/list?title_no=$titleNo", + title = jo.getString("title"), + coverUrl = jo.getString("thumbnail").toAbsoluteUrl(staticDomain), + altTitle = null, + author = jo.getStringOrNull("writingAuthorName"), + isNsfw = jo.getBooleanOrDefault("ageGradeNotice", isNsfwSource), + rating = jo.getFloatOrDefault("starScoreAverage", -10f) / 10f, + tags = setOfNotNull(genres[jo.getString("representGenre")]), + description = jo.getString("synopsis"), + state = null, + source = source, + date = jo.getLong("lastEpisodeRegisterYmdt"), + readCount = jo.getLong("readCount"), + likeCount = jo.getLong("likeitCount") + ) + } + } + override suspend fun getList(offset: Int, filter: MangaListFilter?): List { + if (offset > 0) { + return emptyList() + } val manga = when (filter) { is MangaListFilter.Search -> { - makeRequest("/lineWebtoon/webtoon/searchWebtoon?query=${filter.query.urlEncoded()}&startIndex=${offset + 1}&pageSize=20") + makeRequest("/lineWebtoon/webtoon/searchWebtoon?query=${filter.query.urlEncoded()}") .getJSONObject("webtoonSearch") .getJSONArray("titleList") .mapJSON { jo -> @@ -163,38 +211,8 @@ internal abstract class WebtoonsParser( is MangaListFilter.Advanced -> { val genre = filter.tags.oneOrThrowIfMany()?.key ?: "ALL" - val genres = makeRequest("/lineWebtoon/webtoon/genreList.json") - .getJSONObject("genreList") - .getJSONArray("genres") - .mapJSON { jo -> parseTag(jo) } - .associateBy { tag -> tag.key } - - val result = makeRequest("/lineWebtoon/webtoon/titleList.json") - .getJSONObject("titleList") - .getJSONArray("titles") - .mapJSON { jo -> - val titleNo = jo.getLong("titleNo") - Manga( - id = generateUid(titleNo), - title = jo.getString("title"), - altTitle = null, - url = titleNo.toString(), - publicUrl = "https://$domain/$languageCode/originals/a/list?title_no=$titleNo", - rating = jo.getFloatOrDefault("starScoreAverage", -10f) / 10f, - isNsfw = jo.getBooleanOrDefault("ageGradeNotice", isNsfwSource), - coverUrl = jo.getString("thumbnail").toAbsoluteUrl(staticDomain), - largeCoverUrl = jo.getStringOrNull("thumbnailVertical")?.toAbsoluteUrl(staticDomain), - tags = setOfNotNull(genres[jo.getString("representGenre")]), - author = jo.getStringOrNull("writingAuthorName"), - description = jo.getString("synopsis"), - // I don't think the API provides this info - state = null, - source = source, - date = jo.getLong("lastEpisodeRegisterYmdt"), - readCount = jo.getLong("readCount"), - likeCount = jo.getLong("likeitCount") - ) - } + val genres = getAllGenreList() + val result = getAllTitleList() val sortedResult = when (filter.sortOrder) { SortOrder.UPDATED -> result.sortedBy { it.date } @@ -212,32 +230,7 @@ internal abstract class WebtoonsParser( } null -> { - makeRequest("/lineWebtoon/webtoon/titleList.json") - .getJSONObject("titleList") - .getJSONArray("titles") - .mapJSON { jo -> - val titleNo = jo.getLong("titleNo") - Manga( - id = generateUid(titleNo), - title = jo.getString("title"), - altTitle = null, - url = titleNo.toString(), - publicUrl = "https://$domain/$languageCode/originals/a/list?title_no=$titleNo", - rating = jo.getFloatOrDefault("starScoreAverage", -10f) / 10f, - isNsfw = jo.getBooleanOrDefault("ageGradeNotice", isNsfwSource), - coverUrl = jo.getString("thumbnail").toAbsoluteUrl(staticDomain), - largeCoverUrl = jo.getStringOrNull("thumbnailVertical")?.toAbsoluteUrl(staticDomain), - tags = emptySet(), - author = jo.getStringOrNull("writingAuthorName"), - description = jo.getString("synopsis"), - // I don't think the API provides this info - state = null, - source = source, - date = jo.getLong("lastEpisodeRegisterYmdt"), - readCount = jo.getLong("readCount"), - likeCount = jo.getLong("likeitCount") - ) - } + getAllTitleList() } } From 9e984510d43fdbb25cb8cb5d573613ba9b6b3a14 Mon Sep 17 00:00:00 2001 From: Naga <94557604+NagaYZ@users.noreply.github.com> Date: Sun, 25 Feb 2024 03:30:00 +0100 Subject: [PATCH 05/18] added caching for title list to speed up loading --- .../parsers/site/all/WebtoonsParser.kt | 37 ++++++++++--------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt index ab90edf15..dc85ce294 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt @@ -131,13 +131,6 @@ internal abstract class WebtoonsParser( } } - private val allTitleCache = SuspendLazy { - makeRequest("/lineWebtoon/webtoon/titleList.json?") - .getJSONObject("titleList") - .getJSONArray("titles") - .toJSONList() - } - private val allGenreCache = SuspendLazy { makeRequest("/lineWebtoon/webtoon/genreList.json") .getJSONObject("genreList") @@ -146,14 +139,11 @@ internal abstract class WebtoonsParser( .associateBy { tag -> tag.key } } - private suspend fun getAllGenreList(): Map{ - return allGenreCache.get() - } - - private suspend fun getAllTitleList(): List { - val genres = getAllGenreList() - - return allTitleCache.get().map { jo -> + private val allTitleCache = SuspendLazy { + makeRequest("/lineWebtoon/webtoon/titleList.json?") + .getJSONObject("titleList") + .getJSONArray("titles") + .mapJSON { jo -> val titleNo = jo.getLong("titleNo") Manga( id = generateUid(titleNo), @@ -165,7 +155,7 @@ internal abstract class WebtoonsParser( author = jo.getStringOrNull("writingAuthorName"), isNsfw = jo.getBooleanOrDefault("ageGradeNotice", isNsfwSource), rating = jo.getFloatOrDefault("starScoreAverage", -10f) / 10f, - tags = setOfNotNull(genres[jo.getString("representGenre")]), + tags = setOfNotNull(allGenreCache.get()[jo.getString("representGenre")]), description = jo.getString("synopsis"), state = null, source = source, @@ -176,6 +166,17 @@ internal abstract class WebtoonsParser( } } + + + private suspend fun getAllGenreList(): Map{ + return allGenreCache.get() + } +/* + private suspend fun getAllTitleList(): List { + val genres = getAllGenreList() + + } +*/ override suspend fun getList(offset: Int, filter: MangaListFilter?): List { if (offset > 0) { return emptyList() @@ -212,7 +213,7 @@ internal abstract class WebtoonsParser( val genre = filter.tags.oneOrThrowIfMany()?.key ?: "ALL" val genres = getAllGenreList() - val result = getAllTitleList() + val result = allTitleCache.get() val sortedResult = when (filter.sortOrder) { SortOrder.UPDATED -> result.sortedBy { it.date } @@ -230,7 +231,7 @@ internal abstract class WebtoonsParser( } null -> { - getAllTitleList() + allTitleCache.get() } } From b716c354386730a4d9ee91cc1211387e73d3e29e Mon Sep 17 00:00:00 2001 From: Naga <94557604+NagaYZ@users.noreply.github.com> Date: Sun, 25 Feb 2024 12:30:11 +0100 Subject: [PATCH 06/18] trying to speed up loading --- .../org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt index dc85ce294..c3de6a0b1 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt @@ -224,14 +224,14 @@ internal abstract class WebtoonsParser( } if (genre != "ALL") { - sortedResult.filter { it.tags.contains(genres[genre]) } + sortedResult.filter { it.tags.contains(genres[genre]) }.subList(offset*30, (offset*30) + 30) } else { - sortedResult + sortedResult.subList(offset*30, (offset*30) + 30) } } null -> { - allTitleCache.get() + allTitleCache.get().subList(offset*30, (offset*30) + 30) } } From 86c981ae8e356702d8e2c5714eb0ee94ca3034cb Mon Sep 17 00:00:00 2001 From: Naga <94557604+NagaYZ@users.noreply.github.com> Date: Sun, 25 Feb 2024 12:33:11 +0100 Subject: [PATCH 07/18] fixup! trying to speed up loading --- .../org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt index c3de6a0b1..216cc48b3 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt @@ -178,9 +178,6 @@ internal abstract class WebtoonsParser( } */ override suspend fun getList(offset: Int, filter: MangaListFilter?): List { - if (offset > 0) { - return emptyList() - } val manga = when (filter) { From 8e5ae510677d6d78c4a0fe4171c7d8498b8ac3a2 Mon Sep 17 00:00:00 2001 From: Naga <94557604+NagaYZ@users.noreply.github.com> Date: Sun, 25 Feb 2024 12:50:49 +0100 Subject: [PATCH 08/18] fixed offset --- .../kotatsu/parsers/site/all/WebtoonsParser.kt | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt index 216cc48b3..f9661fc43 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt @@ -171,12 +171,12 @@ internal abstract class WebtoonsParser( private suspend fun getAllGenreList(): Map{ return allGenreCache.get() } -/* + private suspend fun getAllTitleList(): List { - val genres = getAllGenreList() + return allTitleCache.get() } -*/ + override suspend fun getList(offset: Int, filter: MangaListFilter?): List { val manga = @@ -210,7 +210,7 @@ internal abstract class WebtoonsParser( val genre = filter.tags.oneOrThrowIfMany()?.key ?: "ALL" val genres = getAllGenreList() - val result = allTitleCache.get() + val result = getAllTitleList().subList(offset, offset + 20) val sortedResult = when (filter.sortOrder) { SortOrder.UPDATED -> result.sortedBy { it.date } @@ -221,14 +221,14 @@ internal abstract class WebtoonsParser( } if (genre != "ALL") { - sortedResult.filter { it.tags.contains(genres[genre]) }.subList(offset*30, (offset*30) + 30) + sortedResult.filter { it.tags.contains(genres[genre]) } } else { - sortedResult.subList(offset*30, (offset*30) + 30) + sortedResult } } null -> { - allTitleCache.get().subList(offset*30, (offset*30) + 30) + getAllTitleList().subList(offset, offset + 20) } } From 24a25e3dbff817b616759666351c2f5cccf134f9 Mon Sep 17 00:00:00 2001 From: Naga <94557604+NagaYZ@users.noreply.github.com> Date: Sun, 25 Feb 2024 13:00:31 +0100 Subject: [PATCH 09/18] fixup! fixed offset --- .../org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt index f9661fc43..ca6e01608 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt @@ -210,7 +210,7 @@ internal abstract class WebtoonsParser( val genre = filter.tags.oneOrThrowIfMany()?.key ?: "ALL" val genres = getAllGenreList() - val result = getAllTitleList().subList(offset, offset + 20) + val result = getAllTitleList() val sortedResult = when (filter.sortOrder) { SortOrder.UPDATED -> result.sortedBy { it.date } @@ -221,9 +221,9 @@ internal abstract class WebtoonsParser( } if (genre != "ALL") { - sortedResult.filter { it.tags.contains(genres[genre]) } + sortedResult.filter { it.tags.contains(genres[genre]) }.subList(offset, offset + 20) } else { - sortedResult + sortedResult.subList(offset, offset + 20) } } From de58333dc4c169732732039ab8a15712cdbffd41 Mon Sep 17 00:00:00 2001 From: Naga <94557604+NagaYZ@users.noreply.github.com> Date: Sun, 25 Feb 2024 13:12:09 +0100 Subject: [PATCH 10/18] fixup! fixup! fixed offset --- .../parsers/site/all/WebtoonsParser.kt | 220 +++++++++--------- 1 file changed, 107 insertions(+), 113 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt index ca6e01608..4fe4e3822 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt @@ -12,10 +12,34 @@ import org.koitharu.kotatsu.parsers.MangaSourceParser import org.koitharu.kotatsu.parsers.config.ConfigKey import org.koitharu.kotatsu.parsers.exception.NotFoundException import org.koitharu.kotatsu.parsers.exception.ParseException -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.util.* -import org.koitharu.kotatsu.parsers.util.json.* -import java.util.* +import org.koitharu.kotatsu.parsers.model.ContentType +import org.koitharu.kotatsu.parsers.model.Manga +import org.koitharu.kotatsu.parsers.model.MangaChapter +import org.koitharu.kotatsu.parsers.model.MangaListFilter +import org.koitharu.kotatsu.parsers.model.MangaPage +import org.koitharu.kotatsu.parsers.model.MangaSource +import org.koitharu.kotatsu.parsers.model.MangaTag +import org.koitharu.kotatsu.parsers.model.RATING_UNKNOWN +import org.koitharu.kotatsu.parsers.model.SortOrder +import org.koitharu.kotatsu.parsers.util.SuspendLazy +import org.koitharu.kotatsu.parsers.util.domain +import org.koitharu.kotatsu.parsers.util.generateUid +import org.koitharu.kotatsu.parsers.util.json.getBooleanOrDefault +import org.koitharu.kotatsu.parsers.util.json.getFloatOrDefault +import org.koitharu.kotatsu.parsers.util.json.getIntOrDefault +import org.koitharu.kotatsu.parsers.util.json.getStringOrNull +import org.koitharu.kotatsu.parsers.util.json.mapJSON +import org.koitharu.kotatsu.parsers.util.json.mapJSONIndexed +import org.koitharu.kotatsu.parsers.util.json.mapJSONToSet +import org.koitharu.kotatsu.parsers.util.json.toJSONList +import org.koitharu.kotatsu.parsers.util.mapChapters +import org.koitharu.kotatsu.parsers.util.oneOrThrowIfMany +import org.koitharu.kotatsu.parsers.util.parseJson +import org.koitharu.kotatsu.parsers.util.splitTwoParts +import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl +import org.koitharu.kotatsu.parsers.util.urlEncoded +import java.util.Calendar +import java.util.EnumSet import javax.crypto.Mac import javax.crypto.spec.SecretKeySpec @@ -45,12 +69,10 @@ internal abstract class WebtoonsParser( SortOrder.POPULARITY, // views SortOrder.RATING, // star rating //SortOrder.LIKE, // likes - SortOrder.UPDATED + SortOrder.UPDATED, ) override val headers: Headers - get() = Headers.Builder() - .add("User-Agent", "nApps (Android 12;; linewebtoon; 3.1.0)") - .build() + get() = Headers.Builder().add("User-Agent", "nApps (Android 12;; linewebtoon; 3.1.0)").build() override suspend fun getPageUrl(page: MangaPage): String { return page.url.toAbsoluteUrl(staticDomain) @@ -69,22 +91,14 @@ internal abstract class WebtoonsParser( url = "/lineWebtoon/webtoon/episodeList.json?v=5&titleNo=$titleNo&startIndex=0&pageSize=30", ) - val totalEpisodeCount = firstResult - .getJSONObject("episodeList") - .getInt("totalServiceEpisodeCount") + val totalEpisodeCount = firstResult.getJSONObject("episodeList").getInt("totalServiceEpisodeCount") - val episodes = firstResult - .getJSONObject("episodeList") - .getJSONArray("episode") - .toJSONList() - .toMutableList() + val episodes = firstResult.getJSONObject("episodeList").getJSONArray("episode").toJSONList().toMutableList() while (episodes.count() < totalEpisodeCount) { val page = makeRequest( url = "/lineWebtoon/webtoon/episodeList.json?v=5&titleNo=$titleNo&startIndex=${episodes.count()}&pageSize=30", - ).getJSONObject("episodeList") - .getJSONArray("episode") - .toJSONList() + ).getJSONObject("episodeList").getJSONArray("episode").toJSONList() episodes.addAll(page) } @@ -107,8 +121,7 @@ internal abstract class WebtoonsParser( val titleNo = manga.url.toLong() val chaptersDeferred = async { getChapters(titleNo) } - makeRequest("/lineWebtoon/webtoon/titleInfo.json?titleNo=${titleNo}&anyServiceStatus=false") - .getJSONObject("titleInfo") + makeRequest("/lineWebtoon/webtoon/titleInfo.json?titleNo=${titleNo}&anyServiceStatus=false").getJSONObject("titleInfo") .let { jo -> Manga( id = generateUid(titleNo), @@ -132,43 +145,36 @@ internal abstract class WebtoonsParser( } private val allGenreCache = SuspendLazy { - makeRequest("/lineWebtoon/webtoon/genreList.json") - .getJSONObject("genreList") - .getJSONArray("genres") - .mapJSON { jo -> parseTag(jo) } - .associateBy { tag -> tag.key } + makeRequest("/lineWebtoon/webtoon/genreList.json").getJSONObject("genreList").getJSONArray("genres") + .mapJSON { jo -> parseTag(jo) }.associateBy { tag -> tag.key } } private val allTitleCache = SuspendLazy { - makeRequest("/lineWebtoon/webtoon/titleList.json?") - .getJSONObject("titleList") - .getJSONArray("titles") + makeRequest("/lineWebtoon/webtoon/titleList.json?").getJSONObject("titleList").getJSONArray("titles") .mapJSON { jo -> - val titleNo = jo.getLong("titleNo") - Manga( - id = generateUid(titleNo), - url = titleNo.toString(), - publicUrl = "https://$domain/$languageCode/originals/a/list?title_no=$titleNo", - title = jo.getString("title"), - coverUrl = jo.getString("thumbnail").toAbsoluteUrl(staticDomain), - altTitle = null, - author = jo.getStringOrNull("writingAuthorName"), - isNsfw = jo.getBooleanOrDefault("ageGradeNotice", isNsfwSource), - rating = jo.getFloatOrDefault("starScoreAverage", -10f) / 10f, - tags = setOfNotNull(allGenreCache.get()[jo.getString("representGenre")]), - description = jo.getString("synopsis"), - state = null, - source = source, - date = jo.getLong("lastEpisodeRegisterYmdt"), - readCount = jo.getLong("readCount"), - likeCount = jo.getLong("likeitCount") - ) - } + val titleNo = jo.getLong("titleNo") + Manga( + id = generateUid(titleNo), + url = titleNo.toString(), + publicUrl = "https://$domain/$languageCode/originals/a/list?title_no=$titleNo", + title = jo.getString("title"), + coverUrl = jo.getString("thumbnail").toAbsoluteUrl(staticDomain), + altTitle = null, + author = jo.getStringOrNull("writingAuthorName"), + isNsfw = jo.getBooleanOrDefault("ageGradeNotice", isNsfwSource), + rating = jo.getFloatOrDefault("starScoreAverage", -10f) / 10f, + tags = setOfNotNull(allGenreCache.get()[jo.getString("representGenre")]), + description = jo.getString("synopsis"), + state = null, + source = source, + date = jo.getLong("lastEpisodeRegisterYmdt"), + readCount = jo.getLong("readCount"), + likeCount = jo.getLong("likeitCount"), + ) + } } - - - private suspend fun getAllGenreList(): Map{ + private suspend fun getAllGenreList(): Map { return allGenreCache.get() } @@ -179,68 +185,63 @@ internal abstract class WebtoonsParser( override suspend fun getList(offset: Int, filter: MangaListFilter?): List { - val manga = - when (filter) { - is MangaListFilter.Search -> { - makeRequest("/lineWebtoon/webtoon/searchWebtoon?query=${filter.query.urlEncoded()}") - .getJSONObject("webtoonSearch") - .getJSONArray("titleList") - .mapJSON { jo -> - val titleNo = jo.getLong("titleNo") - Manga( - id = generateUid(titleNo), - title = jo.getString("title"), - altTitle = null, - url = titleNo.toString(), - publicUrl = "https://$domain/$languageCode/originals/a/list?title_no=$titleNo", - rating = RATING_UNKNOWN, - isNsfw = isNsfwSource, - coverUrl = jo.getString("thumbnail").toAbsoluteUrl(staticDomain), - largeCoverUrl = null, - tags = emptySet(), - author = jo.getStringOrNull("writingAuthorName"), - description = null, - state = null, - source = source, - ) - } - } - - is MangaListFilter.Advanced -> { - val genre = filter.tags.oneOrThrowIfMany()?.key ?: "ALL" + val manga = when (filter) { + is MangaListFilter.Search -> { + makeRequest("/lineWebtoon/webtoon/searchWebtoon?query=${filter.query.urlEncoded()}").getJSONObject("webtoonSearch") + .getJSONArray("titleList").mapJSON { jo -> + val titleNo = jo.getLong("titleNo") + Manga( + id = generateUid(titleNo), + title = jo.getString("title"), + altTitle = null, + url = titleNo.toString(), + publicUrl = "https://$domain/$languageCode/originals/a/list?title_no=$titleNo", + rating = RATING_UNKNOWN, + isNsfw = isNsfwSource, + coverUrl = jo.getString("thumbnail").toAbsoluteUrl(staticDomain), + largeCoverUrl = null, + tags = emptySet(), + author = jo.getStringOrNull("writingAuthorName"), + description = null, + state = null, + source = source, + ) + } + } - val genres = getAllGenreList() - val result = getAllTitleList() + is MangaListFilter.Advanced -> { + val genre = filter.tags.oneOrThrowIfMany()?.key ?: "ALL" - val sortedResult = when (filter.sortOrder) { - SortOrder.UPDATED -> result.sortedBy { it.date } - SortOrder.POPULARITY -> result.sortedByDescending { it.readCount } - SortOrder.RATING -> result.sortedByDescending { it.rating } - //SortOrder.LIKE -> result.sortedBy { it.likeCount } - else -> throw IllegalArgumentException("Unsupported sort order: ${filter.sortOrder}") - } + val genres = getAllGenreList() + val result = getAllTitleList() - if (genre != "ALL") { - sortedResult.filter { it.tags.contains(genres[genre]) }.subList(offset, offset + 20) - } else { - sortedResult.subList(offset, offset + 20) - } + val sortedResult = when (filter.sortOrder) { + SortOrder.UPDATED -> result.sortedBy { it.date } + SortOrder.POPULARITY -> result.sortedByDescending { it.readCount } + SortOrder.RATING -> result.sortedByDescending { it.rating } + //SortOrder.LIKE -> result.sortedBy { it.likeCount } + else -> throw IllegalArgumentException("Unsupported sort order: ${filter.sortOrder}") } - null -> { - getAllTitleList().subList(offset, offset + 20) + if (genre != "ALL") { + sortedResult.filter { it.tags.contains(genres[genre]) } + } else { + sortedResult } } - return manga + null -> { + getAllTitleList() + } + } + return manga.subList(offset, (offset + 20).coerceAtMost(manga.size)) } override suspend fun getPages(chapter: MangaChapter): List { val (titleNo, episodeNo) = requireNotNull(chapter.url.splitTwoParts('-')) - return makeRequest("/lineWebtoon/webtoon/episodeInfo.json?v=4&titleNo=$titleNo&episodeNo=$episodeNo") - .getJSONObject("episodeInfo") - .getJSONArray("imageInfo") - .mapJSONIndexed { i, jo -> + return makeRequest("/lineWebtoon/webtoon/episodeInfo.json?v=4&titleNo=$titleNo&episodeNo=$episodeNo").getJSONObject( + "episodeInfo", + ).getJSONArray("imageInfo").mapJSONIndexed { i, jo -> MangaPage( id = generateUid("$titleNo-$episodeNo-$i"), url = jo.getString("url"), @@ -259,9 +260,7 @@ internal abstract class WebtoonsParser( } override suspend fun getAvailableTags(): Set { - return makeRequest("/lineWebtoon/webtoon/genreList.json") - .getJSONObject("genreList") - .getJSONArray("genres") + return makeRequest("/lineWebtoon/webtoon/genreList.json").getJSONObject("genreList").getJSONArray("genres") .mapJSONToSet { jo -> parseTag(jo) } } @@ -281,13 +280,11 @@ internal abstract class WebtoonsParser( private fun finalizeUrl(url: String): HttpUrl { val httpUrl = url.toAbsoluteUrl(apiDomain).toHttpUrl() - val builder = httpUrl.newBuilder() - .addQueryParameter("serviceZone", "GLOBAL") + val builder = httpUrl.newBuilder().addQueryParameter("serviceZone", "GLOBAL") if (httpUrl.queryParameter("v") == null) { builder.addQueryParameter("v", "1") } - builder.addQueryParameter("language", languageCode) - .addQueryParameter("locale", "languageCode") + builder.addQueryParameter("language", languageCode).addQueryParameter("locale", "languageCode") .addQueryParameter("platform", "APP_ANDROID") signer.makeEncryptUrl(builder) return builder.build() @@ -296,7 +293,6 @@ internal abstract class WebtoonsParser( @MangaSourceParser("WEBTOONS_EN", "Webtoons English", "en", type = ContentType.MANGA) class English(context: MangaLoaderContext) : WebtoonsParser(context, MangaSource.WEBTOONS_EN) - private inner class WebtoonsUrlSigner(private val secret: String) { private val mac = Mac.getInstance("HmacSHA1").apply { @@ -315,9 +311,7 @@ internal abstract class WebtoonsParser( fun makeEncryptUrl(urlBuilder: HttpUrl.Builder) { val msgPad = Calendar.getInstance().timeInMillis.toString() val digest = getMessageDigest(getMessage(urlBuilder.build().toString(), msgPad)) - urlBuilder - .addQueryParameter("msgpad", msgPad) - .addQueryParameter("md", digest) + urlBuilder.addQueryParameter("msgpad", msgPad).addQueryParameter("md", digest) // .addEncodedQueryParameter("md", digest.urlEncoded()) } } From a22523480004127a75d4f4268356f1898bbb18f2 Mon Sep 17 00:00:00 2001 From: Naga <94557604+NagaYZ@users.noreply.github.com> Date: Sun, 25 Feb 2024 14:11:30 +0100 Subject: [PATCH 11/18] fixed chapter date --- .../kotatsu/parsers/site/all/WebtoonsParser.kt | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt index 4fe4e3822..fcf2c1e45 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt @@ -102,18 +102,18 @@ internal abstract class WebtoonsParser( episodes.addAll(page) } - return episodes.mapChapters { i, jo -> MangaChapter( id = generateUid("$titleNo-$i"), name = jo.getString("episodeTitle"), number = jo.getInt("episodeSeq"), url = "$titleNo-${jo.get("episodeNo")}", - uploadDate = jo.getLong("modifyYmdt"), + uploadDate = jo.getLong("registerYmdt"), branch = null, scanlator = null, source = source, ) + }.sortedBy { it.number } } @@ -136,8 +136,9 @@ internal abstract class WebtoonsParser( tags = setOf(parseTag(jo.getJSONObject("genreInfo"))), author = jo.getStringOrNull("writingAuthorName"), description = jo.getString("synopsis"), - // I don't think the API provides this info + // I don't think the API provides this info, state = null, + date = jo.getLong("lastEpisodeRegisterYmdt"), chapters = chaptersDeferred.await(), source = source, ) @@ -185,7 +186,7 @@ internal abstract class WebtoonsParser( override suspend fun getList(offset: Int, filter: MangaListFilter?): List { - val manga = when (filter) { + val webtoons = when (filter) { is MangaListFilter.Search -> { makeRequest("/lineWebtoon/webtoon/searchWebtoon?query=${filter.query.urlEncoded()}").getJSONObject("webtoonSearch") .getJSONArray("titleList").mapJSON { jo -> @@ -204,6 +205,7 @@ internal abstract class WebtoonsParser( author = jo.getStringOrNull("writingAuthorName"), description = null, state = null, + date = jo.getLong("lastEpisodeRegisterYmdt"), source = source, ) } @@ -230,11 +232,10 @@ internal abstract class WebtoonsParser( } } - null -> { - getAllTitleList() - } + else -> getAllTitleList() + } - return manga.subList(offset, (offset + 20).coerceAtMost(manga.size)) + return webtoons.subList(offset, (offset + 20).coerceAtMost(webtoons.size)) } override suspend fun getPages(chapter: MangaChapter): List { From 91648756c97b3f2e754d24b843f7dacd326fdb4d Mon Sep 17 00:00:00 2001 From: Naga <94557604+NagaYZ@users.noreply.github.com> Date: Sun, 25 Feb 2024 14:16:24 +0100 Subject: [PATCH 12/18] added more language --- .../parsers/site/all/WebtoonsParser.kt | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt index fcf2c1e45..3636df733 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt @@ -294,6 +294,26 @@ internal abstract class WebtoonsParser( @MangaSourceParser("WEBTOONS_EN", "Webtoons English", "en", type = ContentType.MANGA) class English(context: MangaLoaderContext) : WebtoonsParser(context, MangaSource.WEBTOONS_EN) + @MangaSourceParser("WEBTOONS_ID", "Webtoons Indonesia", "id", type = ContentType.MANGA) + class Indonesian(context: MangaLoaderContext) : WebtoonsParser(context, MangaSource.WEBTOONS_ID) + + @MangaSourceParser("WEBTOONS_ES", "Webtoons Spanish", "es", type = ContentType.MANGA) + class Spanish(context: MangaLoaderContext) : WebtoonsParser(context, MangaSource.WEBTOONS_ES) + + @MangaSourceParser("WEBTOONS_FR", "Webtoons French", "fr", type = ContentType.MANGA) + class French(context: MangaLoaderContext) : WebtoonsParser(context, MangaSource.WEBTOONS_FR) + + @MangaSourceParser("WEBTOONS_TH", "Webtoons Thai", "th", type = ContentType.MANGA) + class Thai(context: MangaLoaderContext) : WebtoonsParser(context, MangaSource.WEBTOONS_TH) + + @MangaSourceParser("WEBTOONS_ZH", "Webtoons Chinese", "zh", type = ContentType.MANGA) + class Chinese(context: MangaLoaderContext) : LineWebtoonsParser(context, MangaSource.WEBTOONS_ZH) + + @MangaSourceParser("WEBTOONS_DE", "Webtoons German", "de", type = ContentType.MANGA) + class German(context: MangaLoaderContext) : LineWebtoonsParser(context, MangaSource.WEBTOONS_DE) + + + private inner class WebtoonsUrlSigner(private val secret: String) { private val mac = Mac.getInstance("HmacSHA1").apply { From a65e41e7962f9a7affbbf9caed9449699f44995a Mon Sep 17 00:00:00 2001 From: Naga <94557604+NagaYZ@users.noreply.github.com> Date: Sun, 25 Feb 2024 14:24:14 +0100 Subject: [PATCH 13/18] fix --- .../org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt index 3636df733..bc878278e 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt @@ -205,7 +205,6 @@ internal abstract class WebtoonsParser( author = jo.getStringOrNull("writingAuthorName"), description = null, state = null, - date = jo.getLong("lastEpisodeRegisterYmdt"), source = source, ) } From ea45e91aa7be64702427f37645a57cbbe8edb263 Mon Sep 17 00:00:00 2001 From: Naga <94557604+NagaYZ@users.noreply.github.com> Date: Sun, 25 Feb 2024 14:47:06 +0100 Subject: [PATCH 14/18] trying to improve speed --- .../parsers/site/all/WebtoonsParser.kt | 51 ++++++++++++++++--- 1 file changed, 45 insertions(+), 6 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt index bc878278e..f976dcb74 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt @@ -1,10 +1,14 @@ package org.koitharu.kotatsu.parsers.site.all +import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.runBlocking import okhttp3.Headers import okhttp3.HttpUrl import okhttp3.HttpUrl.Companion.toHttpUrl +import org.json.JSONArray import org.json.JSONObject import org.koitharu.kotatsu.parsers.MangaLoaderContext import org.koitharu.kotatsu.parsers.MangaParser @@ -30,8 +34,6 @@ import org.koitharu.kotatsu.parsers.util.json.getIntOrDefault import org.koitharu.kotatsu.parsers.util.json.getStringOrNull import org.koitharu.kotatsu.parsers.util.json.mapJSON import org.koitharu.kotatsu.parsers.util.json.mapJSONIndexed -import org.koitharu.kotatsu.parsers.util.json.mapJSONToSet -import org.koitharu.kotatsu.parsers.util.json.toJSONList import org.koitharu.kotatsu.parsers.util.mapChapters import org.koitharu.kotatsu.parsers.util.oneOrThrowIfMany import org.koitharu.kotatsu.parsers.util.parseJson @@ -117,9 +119,47 @@ internal abstract class WebtoonsParser( }.sortedBy { it.number } } + private suspend fun fetchEpisodes(titleNo: Long): List = runBlocking { + val firstResult = makeRequest("/lineWebtoon/webtoon/episodeList.json?v=5&titleNo=$titleNo&startIndex=0&pageSize=30") + + val totalEpisodeCount = firstResult.getJSONObject("episodeList").getInt("totalServiceEpisodeCount") + val episodes = firstResult.getJSONObject("episodeList").getJSONArray("episode").toJSONList().toMutableList() + + val deferredPages = (episodes.count() until totalEpisodeCount step 30).map { startIndex -> + async(Dispatchers.IO) { + val page = makeRequest("/lineWebtoon/webtoon/episodeList.json?v=5&titleNo=$titleNo&startIndex=$startIndex&pageSize=30") + page.getJSONObject("episodeList").getJSONArray("episode").toJSONList() + } + } + + val additionalEpisodes = deferredPages.awaitAll().flatten() + episodes.addAll(additionalEpisodes) + + episodes.mapChapters { i, jo -> + MangaChapter( + id = generateUid("$titleNo-$i"), + name = jo.getString("episodeTitle"), + number = jo.getInt("episodeSeq"), + url = "$titleNo-${jo.get("episodeNo")}", + uploadDate = jo.getLong("registerYmdt"), + branch = null, + scanlator = null, + source = source, + ) + }.sortedBy { it.number } + } + + private fun JSONArray.toJSONList(): List { + val list = mutableListOf() + for (i in 0 until length()) { + list.add(getJSONObject(i)) + } + return list + } + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { val titleNo = manga.url.toLong() - val chaptersDeferred = async { getChapters(titleNo) } + val chaptersDeferred = fetchEpisodes(titleNo) makeRequest("/lineWebtoon/webtoon/titleInfo.json?titleNo=${titleNo}&anyServiceStatus=false").getJSONObject("titleInfo") .let { jo -> @@ -139,7 +179,7 @@ internal abstract class WebtoonsParser( // I don't think the API provides this info, state = null, date = jo.getLong("lastEpisodeRegisterYmdt"), - chapters = chaptersDeferred.await(), + chapters = chaptersDeferred, source = source, ) } @@ -260,8 +300,7 @@ internal abstract class WebtoonsParser( } override suspend fun getAvailableTags(): Set { - return makeRequest("/lineWebtoon/webtoon/genreList.json").getJSONObject("genreList").getJSONArray("genres") - .mapJSONToSet { jo -> parseTag(jo) } + return getAllGenreList().values.toSet() } private suspend fun makeRequest(url: String): JSONObject { From 39eeaacdcbede80a998c8e2d941b715932f1c460 Mon Sep 17 00:00:00 2001 From: Naga <94557604+NagaYZ@users.noreply.github.com> Date: Sun, 25 Feb 2024 16:32:31 +0100 Subject: [PATCH 15/18] fixup! trying to improve speed --- .../parsers/site/all/WebtoonsParser.kt | 31 +++++++++---------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt index f976dcb74..b5466dba5 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt @@ -4,7 +4,6 @@ import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.async import kotlinx.coroutines.awaitAll import kotlinx.coroutines.coroutineScope -import kotlinx.coroutines.runBlocking import okhttp3.Headers import okhttp3.HttpUrl import okhttp3.HttpUrl.Companion.toHttpUrl @@ -87,7 +86,7 @@ internal abstract class WebtoonsParser( "zh" -> "zh-hant" else -> tag } - +/* private suspend fun getChapters(titleNo: Long): List { val firstResult = makeRequest( url = "/lineWebtoon/webtoon/episodeList.json?v=5&titleNo=$titleNo&startIndex=0&pageSize=30", @@ -118,23 +117,25 @@ internal abstract class WebtoonsParser( }.sortedBy { it.number } } - - private suspend fun fetchEpisodes(titleNo: Long): List = runBlocking { +*/ + private suspend fun fetchEpisodes(titleNo: Long): List = coroutineScope { val firstResult = makeRequest("/lineWebtoon/webtoon/episodeList.json?v=5&titleNo=$titleNo&startIndex=0&pageSize=30") val totalEpisodeCount = firstResult.getJSONObject("episodeList").getInt("totalServiceEpisodeCount") val episodes = firstResult.getJSONObject("episodeList").getJSONArray("episode").toJSONList().toMutableList() - val deferredPages = (episodes.count() until totalEpisodeCount step 30).map { startIndex -> + val additionalEpisodes = (episodes.size until totalEpisodeCount step 30).map { startIndex -> async(Dispatchers.IO) { - val page = makeRequest("/lineWebtoon/webtoon/episodeList.json?v=5&titleNo=$titleNo&startIndex=$startIndex&pageSize=30") - page.getJSONObject("episodeList").getJSONArray("episode").toJSONList() + makeRequest("/lineWebtoon/webtoon/episodeList.json?v=5&titleNo=$titleNo&startIndex=$startIndex&pageSize=30") + .getJSONObject("episodeList") + .getJSONArray("episode") + .toJSONList() } - } + }.awaitAll().flatten() - val additionalEpisodes = deferredPages.awaitAll().flatten() episodes.addAll(additionalEpisodes) + // Optimize object creation and sorting episodes.mapChapters { i, jo -> MangaChapter( id = generateUid("$titleNo-$i"), @@ -144,11 +145,10 @@ internal abstract class WebtoonsParser( uploadDate = jo.getLong("registerYmdt"), branch = null, scanlator = null, - source = source, + source = source ) - }.sortedBy { it.number } + }.sortedBy(MangaChapter::number) } - private fun JSONArray.toJSONList(): List { val list = mutableListOf() for (i in 0 until length()) { @@ -156,11 +156,10 @@ internal abstract class WebtoonsParser( } return list } - override suspend fun getDetails(manga: Manga): Manga = coroutineScope { val titleNo = manga.url.toLong() - val chaptersDeferred = fetchEpisodes(titleNo) - + val chaptersDeferred = async { fetchEpisodes(titleNo) } + val chapters = chaptersDeferred.await() makeRequest("/lineWebtoon/webtoon/titleInfo.json?titleNo=${titleNo}&anyServiceStatus=false").getJSONObject("titleInfo") .let { jo -> Manga( @@ -179,7 +178,7 @@ internal abstract class WebtoonsParser( // I don't think the API provides this info, state = null, date = jo.getLong("lastEpisodeRegisterYmdt"), - chapters = chaptersDeferred, + chapters = chapters, source = source, ) } From 2e86c480ecf5c98eb3ec981b96860d83ac436778 Mon Sep 17 00:00:00 2001 From: Naga <94557604+NagaYZ@users.noreply.github.com> Date: Wed, 28 Feb 2024 16:01:36 +0100 Subject: [PATCH 16/18] Update src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt Co-authored-by: Koitharu --- .../org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt index b5466dba5..f0bf9b59c 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt @@ -189,7 +189,7 @@ internal abstract class WebtoonsParser( .mapJSON { jo -> parseTag(jo) }.associateBy { tag -> tag.key } } - private val allTitleCache = SuspendLazy { + private val allTitleCache = SoftSuspendLazy { makeRequest("/lineWebtoon/webtoon/titleList.json?").getJSONObject("titleList").getJSONArray("titles") .mapJSON { jo -> val titleNo = jo.getLong("titleNo") From efc9b3502c755f22fca0c4f7b65c470340e76125 Mon Sep 17 00:00:00 2001 From: Naga <94557604+NagaYZ@users.noreply.github.com> Date: Wed, 28 Feb 2024 18:03:12 +0100 Subject: [PATCH 17/18] fixed suggestions --- .../koitharu/kotatsu/parsers/model/Manga.kt | 21 +- .../parsers/site/all/WebtoonsParser.kt | 188 ++++++++---------- .../kotatsu/parsers/site/ar/TeamXNovel.kt | 2 +- 3 files changed, 91 insertions(+), 120 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/model/Manga.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/model/Manga.kt index 76f030f01..c8bf3b802 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/model/Manga.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/model/Manga.kt @@ -67,14 +67,7 @@ class Manga( * Manga source */ @JvmField val source: MangaSource, - - @JvmField val date: Long? = null, - - @JvmField val readCount: Long? = null, - - @JvmField val likeCount: Long? = null, - - ) { +) { /** * Return if manga has a specified rating @@ -116,10 +109,7 @@ class Manga( largeCoverUrl = largeCoverUrl, description = description, chapters = chapters, - source = source, - date = date, - readCount = readCount, - likeCount = likeCount, + source = source ) override fun equals(other: Any?): Boolean { @@ -143,9 +133,7 @@ class Manga( if (description != other.description) return false if (chapters != other.chapters) return false if (source != other.source) return false - if (date != other.date) return false - if (readCount != other.readCount) return false - if (likeCount != other.likeCount) return false + return true } @@ -165,9 +153,6 @@ class Manga( result = 31 * result + (description?.hashCode() ?: 0) result = 31 * result + (chapters?.hashCode() ?: 0) result = 31 * result + source.hashCode() - result = 31 * result + (date?.hashCode() ?: 0) - result = 31 * result + (readCount?.hashCode() ?: 0) - result = 31 * result + (likeCount?.hashCode() ?: 0) return result } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt index f0bf9b59c..c6f933dbb 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt @@ -1,6 +1,5 @@ package org.koitharu.kotatsu.parsers.site.all -import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.async import kotlinx.coroutines.awaitAll import kotlinx.coroutines.coroutineScope @@ -24,6 +23,7 @@ import org.koitharu.kotatsu.parsers.model.MangaSource import org.koitharu.kotatsu.parsers.model.MangaTag import org.koitharu.kotatsu.parsers.model.RATING_UNKNOWN import org.koitharu.kotatsu.parsers.model.SortOrder +import org.koitharu.kotatsu.parsers.util.SoftSuspendLazy import org.koitharu.kotatsu.parsers.util.SuspendLazy import org.koitharu.kotatsu.parsers.util.domain import org.koitharu.kotatsu.parsers.util.generateUid @@ -86,50 +86,19 @@ internal abstract class WebtoonsParser( "zh" -> "zh-hant" else -> tag } -/* - private suspend fun getChapters(titleNo: Long): List { - val firstResult = makeRequest( - url = "/lineWebtoon/webtoon/episodeList.json?v=5&titleNo=$titleNo&startIndex=0&pageSize=30", - ) - - val totalEpisodeCount = firstResult.getJSONObject("episodeList").getInt("totalServiceEpisodeCount") - - val episodes = firstResult.getJSONObject("episodeList").getJSONArray("episode").toJSONList().toMutableList() - while (episodes.count() < totalEpisodeCount) { - val page = makeRequest( - url = "/lineWebtoon/webtoon/episodeList.json?v=5&titleNo=$titleNo&startIndex=${episodes.count()}&pageSize=30", - ).getJSONObject("episodeList").getJSONArray("episode").toJSONList() - - episodes.addAll(page) - } - return episodes.mapChapters { i, jo -> - MangaChapter( - id = generateUid("$titleNo-$i"), - name = jo.getString("episodeTitle"), - number = jo.getInt("episodeSeq"), - url = "$titleNo-${jo.get("episodeNo")}", - uploadDate = jo.getLong("registerYmdt"), - branch = null, - scanlator = null, - source = source, - ) - - }.sortedBy { it.number } - } -*/ private suspend fun fetchEpisodes(titleNo: Long): List = coroutineScope { - val firstResult = makeRequest("/lineWebtoon/webtoon/episodeList.json?v=5&titleNo=$titleNo&startIndex=0&pageSize=30") + val firstResult = + makeRequest("/lineWebtoon/webtoon/episodeList.json?v=5&titleNo=$titleNo&startIndex=0&pageSize=30") val totalEpisodeCount = firstResult.getJSONObject("episodeList").getInt("totalServiceEpisodeCount") val episodes = firstResult.getJSONObject("episodeList").getJSONArray("episode").toJSONList().toMutableList() val additionalEpisodes = (episodes.size until totalEpisodeCount step 30).map { startIndex -> - async(Dispatchers.IO) { - makeRequest("/lineWebtoon/webtoon/episodeList.json?v=5&titleNo=$titleNo&startIndex=$startIndex&pageSize=30") - .getJSONObject("episodeList") - .getJSONArray("episode") - .toJSONList() + async { + makeRequest("/lineWebtoon/webtoon/episodeList.json?v=5&titleNo=$titleNo&startIndex=$startIndex&pageSize=30").getJSONObject( + "episodeList", + ).getJSONArray("episode").toJSONList() } }.awaitAll().flatten() @@ -145,10 +114,11 @@ internal abstract class WebtoonsParser( uploadDate = jo.getLong("registerYmdt"), branch = null, scanlator = null, - source = source + source = source, ) }.sortedBy(MangaChapter::number) } + private fun JSONArray.toJSONList(): List { val list = mutableListOf() for (i in 0 until length()) { @@ -156,31 +126,36 @@ internal abstract class WebtoonsParser( } return list } + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { val titleNo = manga.url.toLong() val chaptersDeferred = async { fetchEpisodes(titleNo) } val chapters = chaptersDeferred.await() makeRequest("/lineWebtoon/webtoon/titleInfo.json?titleNo=${titleNo}&anyServiceStatus=false").getJSONObject("titleInfo") .let { jo -> - Manga( - id = generateUid(titleNo), - title = jo.getString("title"), - altTitle = null, - url = "$titleNo", - publicUrl = "https://$domain/$languageCode/originals/a/list?title_no=${titleNo}", - rating = jo.getFloatOrDefault("starScoreAverage", -10f) / 10f, - isNsfw = jo.getBooleanOrDefault("ageGradeNotice", isNsfwSource), - coverUrl = jo.getString("thumbnail").toAbsoluteUrl(staticDomain), - largeCoverUrl = jo.getStringOrNull("thumbnailVertical")?.toAbsoluteUrl(staticDomain), - tags = setOf(parseTag(jo.getJSONObject("genreInfo"))), - author = jo.getStringOrNull("writingAuthorName"), - description = jo.getString("synopsis"), - // I don't think the API provides this info, - state = null, + MangaWebtoon( + Manga( + id = generateUid(titleNo), + title = jo.getString("title"), + altTitle = null, + url = "$titleNo", + publicUrl = "https://$domain/$languageCode/originals/a/list?title_no=${titleNo}", + rating = jo.getFloatOrDefault("starScoreAverage", -10f) / 10f, + isNsfw = jo.getBooleanOrDefault("ageGradeNotice", isNsfwSource), + coverUrl = jo.getString("thumbnail").toAbsoluteUrl(staticDomain), + largeCoverUrl = jo.getStringOrNull("thumbnailVertical")?.toAbsoluteUrl(staticDomain), + tags = setOf(parseTag(jo.getJSONObject("genreInfo"))), + author = jo.getStringOrNull("writingAuthorName"), + description = jo.getString("synopsis"), + // I don't think the API provides this info, + state = null, + chapters = chapters, + source = source, + ), date = jo.getLong("lastEpisodeRegisterYmdt"), - chapters = chapters, - source = source, - ) + readCount = jo.getLong("readCount"), + //likeCount = jo.getLong("likeitCount") + ).manga } } @@ -193,23 +168,25 @@ internal abstract class WebtoonsParser( makeRequest("/lineWebtoon/webtoon/titleList.json?").getJSONObject("titleList").getJSONArray("titles") .mapJSON { jo -> val titleNo = jo.getLong("titleNo") - Manga( - id = generateUid(titleNo), - url = titleNo.toString(), - publicUrl = "https://$domain/$languageCode/originals/a/list?title_no=$titleNo", - title = jo.getString("title"), - coverUrl = jo.getString("thumbnail").toAbsoluteUrl(staticDomain), - altTitle = null, - author = jo.getStringOrNull("writingAuthorName"), - isNsfw = jo.getBooleanOrDefault("ageGradeNotice", isNsfwSource), - rating = jo.getFloatOrDefault("starScoreAverage", -10f) / 10f, - tags = setOfNotNull(allGenreCache.get()[jo.getString("representGenre")]), - description = jo.getString("synopsis"), - state = null, - source = source, + MangaWebtoon( + Manga( + id = generateUid(titleNo), + url = titleNo.toString(), + publicUrl = "https://$domain/$languageCode/originals/a/list?title_no=$titleNo", + title = jo.getString("title"), + coverUrl = jo.getString("thumbnail").toAbsoluteUrl(staticDomain), + altTitle = null, + author = jo.getStringOrNull("writingAuthorName"), + isNsfw = jo.getBooleanOrDefault("ageGradeNotice", isNsfwSource), + rating = jo.getFloatOrDefault("starScoreAverage", -10f) / 10f, + tags = setOfNotNull(allGenreCache.get()[jo.getString("representGenre")]), + description = jo.getString("synopsis"), + state = null, + source = source, + ), date = jo.getLong("lastEpisodeRegisterYmdt"), readCount = jo.getLong("readCount"), - likeCount = jo.getLong("likeitCount"), + //likeCount = jo.getLong("likeitCount"), ) } } @@ -218,7 +195,7 @@ internal abstract class WebtoonsParser( return allGenreCache.get() } - private suspend fun getAllTitleList(): List { + private suspend fun getAllTitleList(): List { return allTitleCache.get() } @@ -230,21 +207,26 @@ internal abstract class WebtoonsParser( makeRequest("/lineWebtoon/webtoon/searchWebtoon?query=${filter.query.urlEncoded()}").getJSONObject("webtoonSearch") .getJSONArray("titleList").mapJSON { jo -> val titleNo = jo.getLong("titleNo") - Manga( - id = generateUid(titleNo), - title = jo.getString("title"), - altTitle = null, - url = titleNo.toString(), - publicUrl = "https://$domain/$languageCode/originals/a/list?title_no=$titleNo", - rating = RATING_UNKNOWN, - isNsfw = isNsfwSource, - coverUrl = jo.getString("thumbnail").toAbsoluteUrl(staticDomain), - largeCoverUrl = null, - tags = emptySet(), - author = jo.getStringOrNull("writingAuthorName"), - description = null, - state = null, - source = source, + MangaWebtoon( + Manga( + id = generateUid(titleNo), + title = jo.getString("title"), + altTitle = null, + url = titleNo.toString(), + publicUrl = "https://$domain/$languageCode/originals/a/list?title_no=$titleNo", + rating = RATING_UNKNOWN, + isNsfw = isNsfwSource, + coverUrl = jo.getString("thumbnail").toAbsoluteUrl(staticDomain), + largeCoverUrl = null, + tags = emptySet(), + author = jo.getStringOrNull("writingAuthorName"), + description = null, + state = null, + source = source, + ), + date = jo.getLong("lastEpisodeRegisterYmdt"), + readCount = jo.getLong("readCount"), + //likeCount = jo.getLong("likeitCount"), ) } } @@ -258,13 +240,13 @@ internal abstract class WebtoonsParser( val sortedResult = when (filter.sortOrder) { SortOrder.UPDATED -> result.sortedBy { it.date } SortOrder.POPULARITY -> result.sortedByDescending { it.readCount } - SortOrder.RATING -> result.sortedByDescending { it.rating } - //SortOrder.LIKE -> result.sortedBy { it.likeCount } + SortOrder.RATING -> result.sortedByDescending { it.manga.rating } + //SortOrder.LIKE -> result.sortedBy { it.likeitCount } else -> throw IllegalArgumentException("Unsupported sort order: ${filter.sortOrder}") } if (genre != "ALL") { - sortedResult.filter { it.tags.contains(genres[genre]) } + sortedResult.filter { it.manga.tags.contains(genres[genre]) } } else { sortedResult } @@ -273,7 +255,7 @@ internal abstract class WebtoonsParser( else -> getAllTitleList() } - return webtoons.subList(offset, (offset + 20).coerceAtMost(webtoons.size)) + return webtoons.map { it.manga }.subList(offset, (offset + 20).coerceAtMost(webtoons.size)) } override suspend fun getPages(chapter: MangaChapter): List { @@ -281,13 +263,13 @@ internal abstract class WebtoonsParser( return makeRequest("/lineWebtoon/webtoon/episodeInfo.json?v=4&titleNo=$titleNo&episodeNo=$episodeNo").getJSONObject( "episodeInfo", ).getJSONArray("imageInfo").mapJSONIndexed { i, jo -> - MangaPage( - id = generateUid("$titleNo-$episodeNo-$i"), - url = jo.getString("url"), - preview = null, - source = source, - ) - } + MangaPage( + id = generateUid("$titleNo-$episodeNo-$i"), + url = jo.getString("url"), + preview = null, + source = source, + ) + } } private fun parseTag(jo: JSONObject): MangaTag { @@ -349,8 +331,6 @@ internal abstract class WebtoonsParser( @MangaSourceParser("WEBTOONS_DE", "Webtoons German", "de", type = ContentType.MANGA) class German(context: MangaLoaderContext) : LineWebtoonsParser(context, MangaSource.WEBTOONS_DE) - - private inner class WebtoonsUrlSigner(private val secret: String) { private val mac = Mac.getInstance("HmacSHA1").apply { @@ -373,4 +353,10 @@ internal abstract class WebtoonsParser( // .addEncodedQueryParameter("md", digest.urlEncoded()) } } + + private inner class MangaWebtoon( + val manga: Manga, + @JvmField val date: Long? = null, + @JvmField val readCount: Long? = null, + ) } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ar/TeamXNovel.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ar/TeamXNovel.kt index 9db6249f7..48be190a8 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ar/TeamXNovel.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/ar/TeamXNovel.kt @@ -20,7 +20,7 @@ internal class TeamXNovel(context: MangaLoaderContext) : PagedMangaParser(contex override val availableStates: Set = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED, MangaState.ABANDONED) - override val configKeyDomain = ConfigKey.Domain("teamxnovel.com") + override val configKeyDomain = ConfigKey.Domain("team11x11.com") override val isMultipleTagsSupported = false override suspend fun getListPage(page: Int, filter: MangaListFilter?): List { From 368b61c21a685ad9a9a67a63be3f2a600ee7091a Mon Sep 17 00:00:00 2001 From: Naga <94557604+NagaYZ@users.noreply.github.com> Date: Wed, 28 Feb 2024 18:13:51 +0100 Subject: [PATCH 18/18] fixed suggestions --- .../koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt index c6f933dbb..eb51e1dfb 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/WebtoonsParser.kt @@ -154,7 +154,7 @@ internal abstract class WebtoonsParser( ), date = jo.getLong("lastEpisodeRegisterYmdt"), readCount = jo.getLong("readCount"), - //likeCount = jo.getLong("likeitCount") + //likeCount = jo.getLong("likeitCount"), ).manga } } @@ -223,11 +223,7 @@ internal abstract class WebtoonsParser( description = null, state = null, source = source, - ), - date = jo.getLong("lastEpisodeRegisterYmdt"), - readCount = jo.getLong("readCount"), - //likeCount = jo.getLong("likeitCount"), - ) + )) } }